From 63a58d6829b88fd9b09b0082b160422aab2a3983 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 07:14:57 +0000 Subject: [PATCH 01/18] test(git): reproduce a new write branch created on a stale parent Pins today's behavior before the fix: a warm worker that trusts its checkout of the parent creates the write branch on that checkout even after the parent moved, a no-diff live write creates the write branch anyway, and a branch somebody created concurrently is pushed over. Co-Authored-By: Claude Opus 5.5 --- internal/git/branch_worker_new_branch_test.go | 492 ++++++++++++++++++ internal/git/git_atomic_push_test.go | 69 +++ test/e2e/new_branch_parent_e2e_test.go | 260 +++++++++ test/e2e/repo_assertions_test.go | 11 +- 4 files changed, 830 insertions(+), 2 deletions(-) create mode 100644 internal/git/branch_worker_new_branch_test.go create mode 100644 test/e2e/new_branch_parent_e2e_test.go diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go new file mode 100644 index 00000000..72c427b8 --- /dev/null +++ b/internal/git/branch_worker_new_branch_test.go @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// A write branch the remote does not carry yet is created from its parent, the remote's default +// branch. These tests pin the two halves of that contract on ONE warm worker, which is the case +// the cold-worker test (TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain) cannot +// reach: a worker that already trusts its checkout of the parent plans without fetching, so only +// the push can notice that the parent moved. +// +// - Publication checks the parent: the first commit on the new branch descends from the parent's +// tip as the push session advertised it, not from the checkout the plan ran on. +// - Standby is quiet: a cycle that commits nothing leaves the write branch absent on the remote. + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/config" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +const ( + newBranchTarget = "target-a" + newBranchFolder = "live" +) + +// newBranchFixture is a remote whose main is seeded and whose write branch `feature` is absent, +// a worker for `feature`, and a second clone that moves main from outside. +type newBranchFixture struct { + t *testing.T + worker *BranchWorker + server *git.Repository + seed *git.Repository + seedTree *git.Worktree + seedPath string + remote string +} + +func newNewBranchFixture(t *testing.T, seedFiles map[string]string) *newBranchFixture { + t.Helper() + ctx := context.Background() + tempDir := t.TempDir() + remotePath := filepath.Join(tempDir, "remote.git") + remoteURL := "file://" + remotePath + server := createBareRepo(t, remotePath) + + seedPath := filepath.Join(tempDir, "seed") + seed, seedTree := initLocalRepo(t, seedPath, remoteURL, "main") + f := &newBranchFixture{ + t: t, server: server, seed: seed, seedTree: seedTree, seedPath: seedPath, remote: remoteURL, + } + files := map[string]string{"README.md": "seed\n"} + for name, content := range seedFiles { + files[name] = content + } + f.pushToMain(files) + + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configv1alpha3.AddToScheme(scheme)) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + provider := &configv1alpha3.GitProvider{Spec: configv1alpha3.GitProviderSpec{URL: remoteURL}} + provider.Name = "test-repo" + provider.Namespace = "default" + require.NoError(t, k8sClient.Create(ctx, provider)) + + f.worker = NewBranchWorker( + k8sClient, logr.Discard(), "test-repo", "default", "feature", + RepoIdentity{URL: remoteURL}, nil, BranchWorkerLimits{}, + ) + f.worker.ctx = ctx + f.worker.mapper = configMapMapper() + t.Cleanup(func() { _ = os.RemoveAll(f.worker.repoRootPath()) }) + createGitTargetWithPruneMode(t, f.worker, newBranchTarget, newBranchFolder, configv1alpha3.PruneAlways) + return f +} + +// pushToMain commits files to main from outside the worker and returns the new tip. +func (f *newBranchFixture) pushToMain(files map[string]string) plumbing.Hash { + f.t.Helper() + for name, content := range files { + full := filepath.Join(f.seedPath, name) + require.NoError(f.t, os.MkdirAll(filepath.Dir(full), 0o750)) + require.NoError(f.t, os.WriteFile(full, []byte(content), 0o600)) + _, err := f.seedTree.Add(name) + require.NoError(f.t, err) + } + hash, err := f.seedTree.Commit("outside", &git.CommitOptions{ + Author: &object.Signature{Name: "Outside", Email: "outside@example.com", When: time.Now()}, + }) + require.NoError(f.t, err) + require.NoError(f.t, f.seed.Push(&git.PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/main:refs/heads/main"}, + })) + return hash +} + +// mainTip is where main is on the remote. +func (f *newBranchFixture) mainTip() plumbing.Hash { + f.t.Helper() + ref, err := f.server.Reference(plumbing.NewBranchReferenceName("main"), true) + require.NoError(f.t, err) + return ref.Hash() +} + +// featureOnRemote reports where the write branch is on the remote, if anywhere. +func (f *newBranchFixture) featureOnRemote() (plumbing.Hash, bool) { + f.t.Helper() + ref, err := f.server.Reference(plumbing.NewBranchReferenceName("feature"), true) + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return plumbing.ZeroHash, false + } + require.NoError(f.t, err) + return ref.Hash(), true +} + +// warmOnParent is the trusted fetch/reset a new worker performs: HEAD on main, base trusted. +func (f *newBranchFixture) warmOnParent() plumbing.Hash { + f.t.Helper() + require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + require.True(f.t, f.worker.baseTrusted()) + branch, hash := f.localHead() + require.Equal(f.t, plumbing.NewBranchReferenceName("main"), branch, "an absent write branch checks out its parent") + return hash +} + +// noopResync runs a completed resync that finds nothing to change. +func (f *newBranchFixture) noopResync(loop *branchWorkerEventLoop, desired ...manifestanalyzer.DesiredResource) { + f.t.Helper() + req := &ResyncRequest{ + Desired: desired, + ResourceVersion: "1", + GitTargetName: newBranchTarget, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + loop.handleQueueItem(WorkItem{Resync: req}) + result := <-req.Result + require.NoError(f.t, result.Err) + require.Zero(f.t, result.Stats.Created+result.Stats.Updated+result.Stats.Deleted, "the resync must be a no-op") + require.Empty(f.t, loop.pendingWrites, "a no-op resync retains nothing") +} + +func (f *newBranchFixture) localHead() (plumbing.ReferenceName, plumbing.Hash) { + f.t.Helper() + repo, err := git.PlainOpen(f.worker.repoPath()) + require.NoError(f.t, err) + branch, hash, err := GetCurrentBranch(repo) + require.NoError(f.t, err) + return branch, hash +} + +// liveWrite sends one live ConfigMap event through the loop's commit window. +func liveWrite(loop *branchWorkerEventLoop, name string) { + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent(name, "alice", newBranchTarget)}, + CommitMode: CommitModePerEvent, + }}) +} + +// treeHas reports whether the commit's tree carries the file. +func treeHas(t *testing.T, repo *git.Repository, commit plumbing.Hash, file string) bool { + t.Helper() + c, err := repo.CommitObject(commit) + require.NoError(t, err) + _, err = c.File(file) + return err == nil +} + +func cmFile(name string) string { return newBranchFolder + "/default/configmaps/" + name + ".yaml" } + +// cmDocument is the document the worker writes for configMapTargetEvent(name, ...). +func cmDocument(name string) string { + return "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " + name + + "\n namespace: default\ndata:\n key: " + name + "\n" +} + +// TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip is the defect: a warm +// worker trusts its checkout of the parent at A, the parent moves to B, and a live write must +// still create the write branch on B. +// +// The two local states differ in what the cycle records as its push root, which is what decides +// whether the push primitive's own parent check is exercised. Both are inspected rather than +// assumed. +func TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip(t *testing.T) { + cases := []struct { + name string + setup func(f *newBranchFixture, loop *branchWorkerEventLoop) + }{ + { + name: "HEAD on the parent after a trusted fetch", + setup: func(f *newBranchFixture, _ *branchWorkerEventLoop) { f.warmOnParent() }, + }, + { + name: "a local write branch left by a no-op resync", + setup: func(f *newBranchFixture, loop *branchWorkerEventLoop) { + f.noopResync(loop) + branch, _ := f.localHead() + require.Equal(t, plumbing.NewBranchReferenceName("feature"), branch, + "a no-op resync prepares the local write branch before it learns there is nothing to do") + _, onRemote := f.featureOnRemote() + require.False(t, onRemote) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + hashA := f.mainTip() + tc.setup(f, loop) + _, local := f.localHead() + require.Equal(t, hashA, local, "the worker checked out A") + require.True(t, f.worker.baseTrusted(), "and trusts it, so the next plan does not fetch") + + hashB := f.pushToMain(map[string]string{"LATEST.md": "from main at B\n"}) + + // Hold the push so the cycle's recorded root can be read before it is sent. + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + assert.Equal(t, plumbing.NewBranchReferenceName("main"), f.worker.pushCycleRootBranch, + "the cycle is rooted on the parent the write branch is created from") + assert.Equal(t, hashA, f.worker.pushCycleRootHash, "planned on the stale checkout, as designed") + + loop.pushPending() + require.Empty(t, loop.pendingWrites, "the push succeeded") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "the write created the branch") + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + require.Len(t, commit.ParentHashes, 1) + assert.Equal(t, hashB, commit.ParentHashes[0], "the new branch starts from the parent's current tip") + assert.True(t, treeHas(t, f.server, tip, "LATEST.md"), "and carries B's change") + assert.True(t, treeHas(t, f.server, tip, cmFile("cm1")), "and the captured edit") + }) + } +} + +// TestBranchWorker_InSyncSnapshotsOnAbsentBranchCreateNothing is standby across parent movement: +// two in-sync snapshots, with an unrelated change to main between them, push nothing at all. +func TestBranchWorker_InSyncSnapshotsOnAbsentBranchCreateNothing(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + f.noopResync(loop) + f.pushToMain(map[string]string{"UNRELATED.md": "outside the folder\n"}) + f.noopResync(loop) + + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "an in-sync target creates no branch") + assert.True(t, loop.lastPushAt.IsZero(), "and attempts no push") + assert.Zero(t, fetchCount(t, reader, f.worker, fetchReasonContention)) +} + +// TestBranchWorker_ResyncOnAbsentBranchDiffsAgainstTheMovedParent is the other side: once main +// moves so the cluster differs, the snapshot diffs against the new main and creates the branch +// from it. +func TestBranchWorker_ResyncOnAbsentBranchDiffsAgainstTheMovedParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + f.noopResync(loop) + moved := f.pushToMain(map[string]string{cmFile("stale"): cmDocument("stale")}) + + req := &ResyncRequest{ + ResourceVersion: "2", + GitTargetName: newBranchTarget, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + loop.handleQueueItem(WorkItem{Resync: req}) + result := <-req.Result + require.NoError(t, result.Err) + assert.Equal(t, 1, result.Stats.Deleted, "the document main gained is not in the cluster") + loop.pushPending() + require.Empty(t, loop.pendingWrites) + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{moved}, commit.ParentHashes) + assert.False(t, treeHas(t, f.server, tip, cmFile("stale"))) +} + +// TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby is "keep normal deployments +// quiet". A live write the parent already holds commits nothing, and its push must not create the +// write branch merely to record that nothing changed. It still reaches the remote: "already +// present" is a claim only the advertisement can settle. +func TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby(t *testing.T) { + t.Run("the parent already holds the document", func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + liveWrite(loop, "cm1") + + require.Empty(t, loop.pendingWrites, "the no-diff write was published") + assert.False(t, loop.lastPushAt.IsZero(), "it reached the remote to confirm") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "and the write branch stays absent") + observed, known := f.worker.LastRemoteObservation() + require.True(t, known) + assert.Empty(t, observed.Commit, "the push observed no write branch") + }) + + t.Run("the parent gained the document after the plan", func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + require.False(t, loop.pendingWrites[0].CommitSHA.IsZero(), "planned against A, the write is a commit") + + // The reconciler applied main, and main now holds exactly what the cluster holds. + f.pushToMain(map[string]string{cmFile("cm1"): cmDocument("cm1")}) + loop.pushPending() + + require.Empty(t, loop.pendingWrites) + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "replayed against the new parent, nothing is left to publish") + assert.Equal(t, int64(1), fetchCount(t, reader, f.worker, fetchReasonContention), + "the publication check is what caught the moved parent") + }) +} + +// TestBranchWorker_NewBranchPushCostsNoExtraFetchWhenTheParentHolds pins the price of the check: +// it is read from the advertisement the push session opens anyway. +func TestBranchWorker_NewBranchPushCostsNoExtraFetchWhenTheParentHolds(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + hashA := f.warmOnParent() + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{hashA}, commit.ParentHashes) + for _, reason := range []string{fetchReasonPublication, fetchReasonContention, fetchReasonPushFailureProbe} { + assert.Zero(t, fetchCount(t, reader, f.worker, reason), "no %s fetch", reason) + } +} + +// TestBranchWorker_ParentMovesWhileTheWindowIsOpen covers the window between planning and +// publication from the other end: the parent moves while the commit window is still collecting. +func TestBranchWorker_ParentMovesWhileTheWindowIsOpen(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + defer loop.stopTimers() + f.warmOnParent() + + liveWrite(loop, "cm1") + require.NotNil(t, loop.openWindow, "the window is still collecting") + hashB := f.pushToMain(map[string]string{"LATEST.md": "from main at B\n"}) + + require.True(t, loop.finalizeOpenWindow()) + loop.maybeSchedulePush() + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{hashB}, commit.ParentHashes) +} + +// TestBranchWorker_DeletedWriteBranchIsRecreatedFromTheParent is the return to standby after a +// review: the write branch is merged and deleted, main advances, and the next edit through the +// SAME worker recreates the branch on the new main with only the new edit on top. +func TestBranchWorker_DeletedWriteBranchIsRecreatedFromTheParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + liveWrite(loop, "cm1") + _, onRemote := f.featureOnRemote() + require.True(t, onRemote) + + // Merged by content and deleted, the way a squash merge leaves it. + merged := f.pushToMain(map[string]string{cmFile("cm1"): cmDocument("cm1"), "MERGED.md": "squashed\n"}) + require.NoError(t, f.server.Storer.RemoveReference(plumbing.NewBranchReferenceName("feature"))) + + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + require.Empty(t, loop.pendingWrites) + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "the next edit recreates the branch") + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{merged}, commit.ParentHashes, "on the parent's current tip") + assert.True(t, treeHas(t, f.server, tip, "MERGED.md")) + assert.True(t, treeHas(t, f.server, tip, cmFile("cm2"))) +} + +// TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent covers the two ways the check can +// fail to settle: a parent that keeps moving exhausts the bounded retries, and a remote that cannot +// be reached answers nothing. Either way the work stays retained and the write branch uncreated. +func TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent(t *testing.T) { + t.Run("the parent keeps moving", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + // Every reset onto the remote is immediately outrun by another outside commit, so each + // attempt is checked again and each one finds the parent moved. + original := syncToRemoteFn + t.Cleanup(func() { syncToRemoteFn = original }) + moves := 0 + syncToRemoteFn = func( + ctx context.Context, repo *git.Repository, branch plumbing.ReferenceName, auth []gitclient.Option, + ) (*PullReport, error) { + report, err := original(ctx, repo, branch, auth) + moves++ + f.pushToMain(map[string]string{"MOVING.md": time.Now().String() + string(rune('a'+moves))}) + return report, err + } + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + f.pushToMain(map[string]string{"FIRST.md": "moved before the first attempt\n"}) + loop.pushPending() + + assert.Len(t, loop.pendingWrites, 1, "the work is retained") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "and nothing unchecked was published") + assert.Positive(t, moves) + }) + + t.Run("the remote cannot be reached", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + original := pushAtomicFn + t.Cleanup(func() { pushAtomicFn = original }) + pushAtomicFn = func( + context.Context, *git.Repository, plumbing.Hash, plumbing.ReferenceName, []gitclient.Option, + ) (PushOutcome, error) { + return PushOutcome{}, errors.New("connection refused") + } + + liveWrite(loop, "cm1") + + assert.Len(t, loop.pendingWrites, 1, "the work is retained for retry") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote) + }) +} diff --git a/internal/git/git_atomic_push_test.go b/internal/git/git_atomic_push_test.go index 433e089f..244cc2fb 100644 --- a/internal/git/git_atomic_push_test.go +++ b/internal/git/git_atomic_push_test.go @@ -97,6 +97,75 @@ func TestAtomicPush_PushToOther(t *testing.T) { require.Equal(t, createdCommit, ref.Hash()) } +// TestAtomicPush_PushToOther_DetectsMovedRoot covers the push that creates a branch. Its root is +// the parent it was built from, and a parent that moved since the checkout must refuse the push: +// otherwise the new branch is born on a stale tree. +func TestAtomicPush_PushToOther_DetectsMovedRoot(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + serverPath := filepath.Join(tempDir, "server") + remoteURL := "file://" + serverPath + + localPath := filepath.Join(tempDir, "local") + remoteRepo := createBareRepo(t, serverPath) + commitA := simulateClientCommitOnDisk(t, remoteURL, "main", "README.md", "A") + + localRepo, worktree := initLocalRepo(t, localPath, remoteURL, "feature") + commitFileChange(t, worktree, localPath, "README.md", "ours, built on A") + + commitB := simulateClientCommitOnDisk(t, remoteURL, "main", "OTHER.md", "B") + + mainBranch := plumbing.NewBranchReferenceName("main") + featureBranch := plumbing.NewBranchReferenceName("feature") + outcome, err := PushAtomic(ctx, localRepo, commitA, mainBranch, nil) + require.Error(t, err) + assert.Empty(t, outcome.Kind, "an error means nothing was observed") + + var moved *RemoteMovedError + require.ErrorAs(t, err, &moved) + assert.Equal(t, mainBranch, moved.Branch) + assert.Equal(t, commitA, moved.Expected) + assert.Equal(t, commitB, moved.Advertised) + assert.False(t, moved.Missing) + + _, err = remoteRepo.Reference(featureBranch, true) + require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "the new branch must not be created") +} + +// TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch covers a branch somebody else created +// between our checkout and our push. We believed it absent, so we must not push over their commit. +func TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + serverPath := filepath.Join(tempDir, "server") + remoteURL := "file://" + serverPath + + localPath := filepath.Join(tempDir, "local") + remoteRepo := createBareRepo(t, serverPath) + commitA := simulateClientCommitOnDisk(t, remoteURL, "main", "README.md", "A") + + localRepo, worktree := initLocalRepo(t, localPath, remoteURL, "feature") + commitFileChange(t, worktree, localPath, "README.md", "ours, built on A") + + theirs := simulateClientCommitOnDisk(t, remoteURL, "feature", "THEIRS.md", "theirs") + + mainBranch := plumbing.NewBranchReferenceName("main") + featureBranch := plumbing.NewBranchReferenceName("feature") + outcome, err := PushAtomic(ctx, localRepo, commitA, mainBranch, nil) + require.Error(t, err) + assert.Empty(t, outcome.Kind, "an error means nothing was observed") + + var moved *RemoteMovedError + require.ErrorAs(t, err, &moved) + assert.Equal(t, featureBranch, moved.Branch) + assert.Equal(t, plumbing.ZeroHash, moved.Expected, "we expected the branch to be absent") + assert.Equal(t, theirs, moved.Advertised) + + ref, err := remoteRepo.Reference(featureBranch, true) + require.NoError(t, err) + assert.Equal(t, theirs, ref.Hash(), "their commit must be untouched") +} + // TestAtomicPush_DetectsMissingBranch tests that push detects when remote branch doesn't exist anymore. func TestAtomicPush_DetectsMissingBranch(t *testing.T) { ctx := context.Background() diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go new file mode 100644 index 00000000..2dd20594 --- /dev/null +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A GitTarget whose write branch does not exist yet stays on standby, and its first commit lands +// on the parent branch's CURRENT tip. +// +// The write branch is created from the remote's default branch (its parent) only when a write has +// something to commit. Until then nothing is pushed, including for a live event the parent +// already holds. When a write does commit, publication checks the parent against the push +// session's advertisement, so a worker that checked out the parent long ago still creates the +// branch on the parent's tip rather than on its own stale checkout. +// +// The stale checkout is the whole point, so the spec builds it deliberately: the target's own +// initial snapshot clones main at A and trusts it before B is pushed. A cold worker would fetch on +// its first write and land on B without exercising the check at all. +// +// What fixed it is read off the fetch counter: the moved parent must be recovered by a +// `contention` fetch (the rejected push), with no `refresh` fetch in between. That takes the +// periodic refresher out of the proof without redeploying the controller with it disabled, which +// would disturb every other spec on the leg. +var _ = Describe("New write branch starts from its parent", Label("manager"), Ordered, func() { + const ( + providerName = "parent-provider" + destName = "parent-dest" + ruleName = "parent-rule" + gitPath = "apps/parent" + selectorLabel = "e2e.configbutler.ai/parent-branch" + ) + var ( + testNs string + repo *RepoArtifacts + writeBranch string + ) + + applyConfigMap := func(name string) { + GinkgoHelper() + manifest := fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s + labels: + %s: "yes" +data: + key: %s +`, name, testNs, selectorLabel, name) + out, err := kubectlRunWithStdin(testNs, manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply ConfigMap %q: %s", name, out) + } + + // mirroredDocument is the document the controller writes for applyConfigMap(name). + mirroredDocument := func(name string) string { + return fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + labels: + %s: "yes" + name: %s + namespace: %s +data: + key: %s +`, selectorLabel, name, testNs, name) + } + + branchSeries := func(metric, extra string) string { + return fmt.Sprintf(`sum(%s{provider_namespace=%q,branch=%q%s}) or vector(0)`, + metric, testNs, writeBranch, extra) + } + fetches := func(reason string) float64 { + GinkgoHelper() + v, err := queryPrometheus(branchSeries("gitopsreverser_git_fetches_total", fmt.Sprintf(",reason=%q", reason))) + Expect(err).NotTo(HaveOccurred()) + return v + } + pushes := func() float64 { + GinkgoHelper() + v, err := queryPrometheus(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`)) + Expect(err).NotTo(HaveOccurred()) + return v + } + + BeforeAll(func() { + ensurePrometheusClient() + testNs = testNamespaceFor("manager-parent-branch") + _, _ = kubectlRun("create", "namespace", testNs) + writeBranch = fmt.Sprintf("reverser-e2e-%d", GinkgoRandomSeed()) + + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-parent-branch-%d", GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + provider := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: GitProvider +metadata: + name: %s + namespace: %s +spec: + url: %s + allowedBranches: ["main", %q] + secretRef: + name: %s +`, providerName, testNs, repo.RepoURLHTTP, writeBranch, repo.GitSecretHTTP) + out, err := kubectlRunWithStdin(testNs, provider, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply GitProvider: %s", out) + verifyResourceStatus("gitprovider", providerName, testNs, "True", "Succeeded", "") + }) + + AfterAll(func() { + cleanupWatchRule(ruleName, testNs) + cleanupGitTarget(destName, testNs) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", providerName, "--ignore-not-found=true") + cleanupNamespace(testNs) + }) + + It("stays on standby, then creates the write branch on the parent's current tip", func() { + By("seeding main with commit A, whose folder already holds a document the cluster will match") + commitFilesToMainFromOutside(repo, testNs, "e2e: seed A", map[string]string{ + "README.md": "seed\n", + path.Join(gitPath, testNs, "configmaps", "standby-present.yaml"): mirroredDocument("standby-present"), + }) + + By("creating a GitTarget on an absent write branch, and a WatchRule that selects nothing yet") + createGitTarget(destName, testNs, providerName, gitPath, writeBranch) + rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + gitTargetRef: + name: %s + rules: + - apiGroups: [""] + resources: ["configmaps"] + objectSelector: + matchLabels: + %s: "yes" +`, ruleName, testNs, destName, selectorLabel) + out, err := kubectlRunWithStdin(testNs, rule, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply WatchRule: %s", out) + verifyResourceStatus("watchrule", ruleName, testNs, "True", "Succeeded", "") + waitForWatchRuleStreamsRunning(ruleName, testNs) + + // The initial snapshot is an in-sync no-op that still clones: this is the worker checking + // out A and trusting it, which is the stale state the rest of the spec depends on. + waitForMetricWithTimeout(branchSeries("gitopsreverser_git_fetches_total", ""), + func(v float64) bool { return v > 0 }, + "the worker for the write branch checked out the parent", 2*time.Minute) + + By("an idle, in-sync target creates no branch") + Consistently(func(g Gomega) { + g.Expect(remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch)).To(BeEmpty(), + "idle means no branch and no commit") + }, 15*time.Second, 3*time.Second).Should(Succeed()) + + By("a live event the parent already holds publishes nothing") + pushesBefore := pushes() + applyConfigMap("standby-present") + // The no-diff window is still pushed, because only the remote can confirm "already + // present". Wait for that push cycle to finish before judging the branch. + waitForMetricWithTimeout(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`), + func(v float64) bool { return v > pushesBefore }, + "the no-diff window reached the remote", 2*time.Minute) + Expect(remoteBranchHeadOf(Default, repo.CheckoutDir, writeBranch)).To(BeEmpty(), + "a write that commits nothing must not create the write branch") + + By("main moves to B from outside, touching the target's folder") + contentionBefore, refreshBefore := fetches("contention"), fetches("refresh") + hashB := commitFilesToMainFromOutside(repo, testNs, "e2e: B moves main", map[string]string{ + path.Join(gitPath, "NOTES.md"): "added on main at B\n", + }) + + By("a live edit arrives with no resync in between") + applyConfigMap("standby-edit") + + By("the write branch is created on B and carries the edit") + editPath := path.Join(gitPath, testNs, "configmaps", "standby-edit.yaml") + Eventually(func(g Gomega) { + tip := remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch) + g.Expect(tip).NotTo(BeEmpty(), "the edit creates the write branch") + _, fetchErr := gitRun(repo.CheckoutDir, "fetch", "origin", writeBranch) + g.Expect(fetchErr).NotTo(HaveOccurred()) + parent, revErr := gitRun(repo.CheckoutDir, "rev-parse", tip+"^") + g.Expect(revErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(parent)).To(Equal(hashB), "the first commit must sit on the parent's current tip") + for _, file := range []string{path.Join(gitPath, "NOTES.md"), editPath} { + _, showErr := gitRun(repo.CheckoutDir, "cat-file", "-e", tip+":"+file) + g.Expect(showErr).NotTo(HaveOccurred(), "%s must be on the new branch", file) + } + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + + By("and the publication check, not the refresher, is what found B") + waitForMetricWithTimeout(branchSeries("gitopsreverser_git_fetches_total", `,reason="contention"`), + func(v float64) bool { return v > contentionBefore }, + "the rejected push reset onto the moved parent", time.Minute) + Expect(fetches("refresh")).To(Equal(refreshBefore), + "no refresh fetch may stand in for the publication check") + }) +}) + +// commitFilesToMainFromOutside commits files to main and pushes it directly, the way another +// writer would, and returns the new tip. It starts main when the repository is still empty, and +// retries because losing a push race with the controller says nothing about the behavior under test. +func commitFilesToMainFromOutside(repo *RepoArtifacts, namespace, message string, files map[string]string) string { + GinkgoHelper() + configureRepoOriginWithCredentials(repo, namespace) + + Eventually(func() error { + if _, err := gitRun(repo.CheckoutDir, "fetch", "origin", "main"); err == nil { + if out, err := gitRun(repo.CheckoutDir, "checkout", "-B", "main", "origin/main"); err != nil { + return fmt.Errorf("checkout: %w: %s", err, out) + } + if out, err := gitRun(repo.CheckoutDir, "reset", "--hard", "origin/main"); err != nil { + return fmt.Errorf("reset: %w: %s", err, out) + } + } else { + if out, err := gitRun(repo.CheckoutDir, "checkout", "--orphan", "main"); err != nil { + return fmt.Errorf("orphan: %w: %s", err, out) + } + _, _ = gitRun(repo.CheckoutDir, "rm", "-rf", ".") + } + for name, content := range files { + full := filepath.Join(repo.CheckoutDir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + return err + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + return err + } + if out, err := gitRun(repo.CheckoutDir, "add", name); err != nil { + return fmt.Errorf("add: %w: %s", err, out) + } + } + if out, err := gitRun(repo.CheckoutDir, "commit", "-m", message); err != nil { + return fmt.Errorf("commit: %w: %s", err, out) + } + if out, err := gitRun(repo.CheckoutDir, "push", "origin", "HEAD:main"); err != nil { + return fmt.Errorf("push: %w: %s", err, out) + } + return nil + }, seedPushTimeout, seedPushInterval).Should(Succeed(), "the outside writer kept losing the push race") + + head, err := gitRun(repo.CheckoutDir, "rev-parse", "HEAD") + Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(head) +} diff --git a/test/e2e/repo_assertions_test.go b/test/e2e/repo_assertions_test.go index f720cc2f..11e09e7d 100644 --- a/test/e2e/repo_assertions_test.go +++ b/test/e2e/repo_assertions_test.go @@ -45,8 +45,15 @@ func pullLatestRepoState(g Gomega, checkoutDir string) { // commit, and UNCHANGED (== the prior SHA) for an already-established branch. func remoteBranchHead(g Gomega, checkoutDir string) string { GinkgoHelper() + return remoteBranchHeadOf(g, checkoutDir, "main") +} + +// remoteBranchHeadOf is remoteBranchHead for any branch: the SHA at its tip on the remote, or "" +// when the remote does not carry it. +func remoteBranchHeadOf(g Gomega, checkoutDir, branch string) string { + GinkgoHelper() - output, err := gitRun(checkoutDir, "ls-remote", "--heads", "origin", "main") + output, err := gitRun(checkoutDir, "ls-remote", "--heads", "origin", branch) g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("git ls-remote must succeed even when the branch is absent. Output: %s", output)) @@ -54,7 +61,7 @@ func remoteBranchHead(g Gomega, checkoutDir string) string { if len(fields) == 0 { return "" // branch does not exist yet — nothing committed } - return fields[0] // ls-remote prints "\trefs/heads/main" + return fields[0] // ls-remote prints "\trefs/heads/" } func recentCommitDiagnostics(checkoutDir, pathspec string) string { From 6ac4ba3936a84d40458cd40c071e49b5060b4630 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 07:22:35 +0000 Subject: [PATCH 02/18] fix(git): check the parent before creating a write branch, and create none for a no-op A push that creates the write branch is rooted on the parent it was built from, but validatePushState compared that root only when the pushed branch already existed. A warm worker that trusted its checkout of the parent therefore created the branch on that checkout, however far the parent had moved. The root is now compared either way, and a branch somebody created since we based our work is reported as a moved remote instead of being pushed over; the worker replays onto it. A cycle that commits nothing no longer creates the write branch merely to record that nothing changed: the push settles as NoBranch once the parent is confirmed. The worker remembers the parent a local write branch was created from, so a no-op cycle that leaves HEAD on that local branch still roots the next cycle on the parent. The refresher follows the parent of an absent write branch from the same advertisement, so the folder it re-reads is the tree the next commit builds on. Publication does not depend on it. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 23 +++++- internal/git/branch_worker.go | 63 +++++++++++++-- internal/git/branch_worker_new_branch_test.go | 30 +++++++- internal/git/git_atomic_push.go | 73 ++++++++++++------ internal/git/git_atomic_push_test.go | 10 +-- internal/git/git_roundtrip_ledger_test.go | 9 ++- internal/git/refresh.go | 67 +++++++++++++--- internal/git/refresh_test.go | 77 ++++++++++++++++++- test/e2e/new_branch_parent_e2e_test.go | 76 ++++++++++-------- 9 files changed, 339 insertions(+), 89 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 6f1b8c4f..cb125f10 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,7 +71,11 @@ that a write failed part-way and is cleared **only by a reset** (a third, `repla retained writes whose local commits a reset discarded before the replay could rebuild them). Collapsing them would let one kind of doubt clear another. Either flag makes the next cycle fetch. The failure direction is safe by construction: a stale "untrusted" costs one fetch, while a stale "trusted" -is caught by the compare-and-swap on the next push. **The corollary is the rule to keep in your head +is caught by the compare-and-swap on the next push. For a new write branch, the worker checks the +parent's advertised tip before sending the push. A moved parent triggers fetch, reset, and replay. +The server's compare-and-swap protects creation of the write branch with `Old = zero`; it does not +lock the parent. The parent can still move after the advertisement, so what is guaranteed is the +parent's tip as that advertisement showed it. **The corollary is the rule to keep in your head when adding code here:** anything that resolves, commits, or concludes without reaching a push advertisement must either fetch, or refuse to conclude. See [inbound push notification](design/push-notification-and-reconcile-trigger.md) §3. @@ -561,14 +565,15 @@ instead. ### Remote moved while we were writing -If someone else pushes to the same remote branch before GitOps Reverser's push lands, the operator does -not treat its local clone as authoritative. It keeps the finalized pending writes, fetches the new remote +If someone else pushes to the same remote branch before GitOps Reverser's push lands, or, for a branch +not created yet, if its parent branch moved, the operator does not treat its local clone as +authoritative. It keeps the finalized pending writes, fetches the new remote tip, resets the local clone, replays those writes, and pushes again. ```mermaid flowchart TD LOCAL[Window finalized
pending write retained] --> CHECK[PushAtomic checks remote ref] - CHECK --> MATCH{Remote still at expected SHA?} + CHECK --> MATCH{Write branch, or while it is absent its parent,
still at the expected commit?} MATCH -->|yes| ACCEPT[Push accepted] ACCEPT --> CLEAR[Clear retained pending writes] @@ -1235,6 +1240,16 @@ A branch whose remote has been **deleted** takes the same path. The advertisemen branch at all, which is reported as a moved remote with a zero hash, so the replay re-roots on the remote's default branch and the retry re-creates the branch with the retained writes on top. +**A branch that does not exist yet** is created from the remote's default branch (its parent) only +when a write has something to commit, so an idle or in-sync target never creates it. Until then the +worker compares the folder against the parent. Periodic refresh, ten minutes by default, follows the +parent from the same advertisement and updates the idle view independently of the publication +check. The push records the parent and the commit it started from as its root, and treats a moved +parent like a moved branch: fetch, reset, replay, check again. A branch somebody else created in the +meantime is a moved remote too, so the replay builds on their commit instead of overwriting it. +Proved end to end by the e2e spec "New write branch starts from its parent", whose recovery is +attributed to the publication path rather than the refresher. + **When the cycle fetches at all.** Only the first commit of a cycle may fetch, and only when the base is untrusted or the worktree is dirty (see the ground rule above); a healthy publishing target plans straight onto its own last push. Once writes are retained the guard flips off entirely, because a diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index aaa5f840..3c3b85a7 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -227,6 +227,17 @@ type BranchWorker struct { // Cleared after a successful push. Protected by repoMu. pushCycleRootHash plumbing.Hash + // newBranchParent and newBranchParentHash are the parent the local write branch was created + // from, while the remote does not carry the write branch. They let a cycle that finds HEAD + // already on the local write branch still root itself on the parent: a no-op cycle (a resync + // or a live write with nothing to commit) creates the local branch and leaves HEAD on it, and + // local HEAD alone cannot say that the branch is absent on the remote. + // + // Cleared as soon as the remote is known to carry the branch: a fetch that found it, or a push + // that created or confirmed it. Protected by repoMu. + newBranchParent plumbing.ReferenceName + newBranchParentHash plumbing.Hash + // firsts surfaces the first successful commit and push at default verbosity. firsts branchWorkerLogFirsts @@ -1996,8 +2007,7 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { w.recordRemoteObservation(revision, ObservedByPush) w.Log.V(1).Info("Remote observed by push", "branch", w.Branch, "outcome", string(outcome.Kind), "head", revision) - w.pushCycleRootBranch = "" - w.pushCycleRootHash = plumbing.ZeroHash + w.endPushCycle(outcome.Kind) w.firsts.push.Do(func() { w.Log.Info("First push to remote completed", "branch", w.Branch, @@ -2063,6 +2073,15 @@ func (w *BranchWorker) remoteMovedDuringPush( rootHash plumbing.Hash, auth []gitclient.Option, ) bool { + // A push that creates the write branch is rooted on its parent, so a rejection naming the + // write branch itself is not about the root: somebody created the branch since we based our + // work. That is contention all the same, and the replay builds on their commit. + var moved *RemoteMovedError + if errors.As(pushErr, &moved) && moved.Branch != rootBranch && + moved.Branch == plumbing.NewBranchReferenceName(w.Branch) { + return true + } + remoteHash, known := advertisedRootHash(pushErr, rootBranch) if !known { var fetchErr error @@ -2300,11 +2319,38 @@ func (w *BranchWorker) ensureWriteBranch(repo *gogit.Repository) (plumbing.Refer if err := switchOrCreateBranch(repo, targetBranch, w.Log, w.Branch, baseHash); err != nil { return "", plumbing.ZeroHash, err } + // HEAD was on another branch, which happens only when a fetch found the write branch + // absent and checked out its parent instead. Remember it, so the next cycle on this local + // branch is still rooted on the parent. + w.newBranchParent, w.newBranchParentHash = baseBranch, baseHash + return baseBranch, baseHash, nil + } + + // Already on the local write branch, with nothing committed on top of the parent it was + // created from: the remote still does not carry it, so the parent remains the root. + if w.newBranchParent != "" && baseHash == w.newBranchParentHash { + return w.newBranchParent, w.newBranchParentHash, nil } return baseBranch, baseHash, nil } +// endPushCycle clears the cycle's root after a push the remote answered. Only a push that left the +// branch absent keeps the parent it would be created from. +func (w *BranchWorker) endPushCycle(kind PushOutcomeKind) { + w.pushCycleRootBranch = "" + w.pushCycleRootHash = plumbing.ZeroHash + if kind != PushNoBranch { + w.clearNewBranchParent() + } +} + +// clearNewBranchParent forgets the parent once the remote carries the write branch. +func (w *BranchWorker) clearNewBranchParent() { + w.newBranchParent = "" + w.newBranchParentHash = plumbing.ZeroHash +} + // Push cycle outcomes and retry reasons. // // A push CYCLE is one call to pushPendingCommits, replay retries included, and it ends exactly @@ -2786,12 +2832,17 @@ func (w *BranchWorker) updateBranchMetadataFromPullReport(report *PullReport) { // based on the default branch (or an empty one) IS that state. There is nothing on the remote // left to learn, so a fetch per cycle could only ever return the same answer. // - // What makes it safe is the same thing that makes it safe everywhere else: the - // compare-and-swap. A push onto a branch we believe is absent declares Old = zero, which the - // server rejects if somebody has since created it, and the rejection invalidates the base and - // fetches. So the cost of being wrong is one rejection, not a bad write. + // What makes it safe is the push. For a branch we believe absent, the push is rooted on the + // parent the branch is created from, and validatePushState refuses it if that parent moved or + // if somebody has since created the branch; either rejection invalidates the base and replays + // onto what the advertisement showed. The push then declares Old = zero, so the server also + // refuses a branch created after the advertisement. So the cost of being wrong is one + // rejection, not a bad write. w.setBaseTrusted(true) w.markWorktreeClean() + if report.ExistsOnRemote { + w.clearNewBranchParent() + } // Log if this was an unborn branch if report.HEAD.Unborn { diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go index 72c427b8..c399ff5f 100644 --- a/internal/git/branch_worker_new_branch_test.go +++ b/internal/git/branch_worker_new_branch_test.go @@ -33,7 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" "github.com/ConfigButler/gitops-reverser/internal/telemetry" ) @@ -143,10 +142,9 @@ func (f *newBranchFixture) warmOnParent() plumbing.Hash { } // noopResync runs a completed resync that finds nothing to change. -func (f *newBranchFixture) noopResync(loop *branchWorkerEventLoop, desired ...manifestanalyzer.DesiredResource) { +func (f *newBranchFixture) noopResync(loop *branchWorkerEventLoop) { f.t.Helper() req := &ResyncRequest{ - Desired: desired, ResourceVersion: "1", GitTargetName: newBranchTarget, GitTargetNamespace: "default", @@ -490,3 +488,29 @@ func TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent(t *testing.T) { assert.False(t, onRemote) }) } + +// TestBranchWorker_WriteBranchCreatedConcurrentlyIsBuiltOn is §2.1a through the worker: somebody +// creates the write branch between our plan and our push. Their commit must survive, and ours +// must be replayed on top of it rather than pushed over it. +func TestBranchWorker_WriteBranchCreatedConcurrentlyIsBuiltOn(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + + theirs := simulateClientCommitOnDisk(t, f.remote, "feature", "THEIRS.md", "theirs\n") + loop.pushPending() + require.Empty(t, loop.pendingWrites) + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{theirs}, commit.ParentHashes, "our commit descends from theirs") + assert.True(t, treeHas(t, f.server, tip, "THEIRS.md")) + assert.True(t, treeHas(t, f.server, tip, cmFile("cm1"))) +} diff --git a/internal/git/git_atomic_push.go b/internal/git/git_atomic_push.go index f190ca5c..80604dd7 100644 --- a/internal/git/git_atomic_push.go +++ b/internal/git/git_atomic_push.go @@ -190,8 +190,7 @@ func validatePushState( // Determine the "old" hash for the push command and validate state var oldHash = plumbing.ZeroHash remoteHash, found := refs[branch] - currentRootHash, rootFound := refs[rootBranch] - if !rootFound && !rootHash.IsZero() { + if _, rootFound := refs[rootBranch]; !rootFound && !rootHash.IsZero() { return pushPlan{}, &RemoteMovedError{ Branch: rootBranch, Expected: rootHash, @@ -200,34 +199,64 @@ func validatePushState( } } + if found && localHash == remoteHash { + logger.Info("remote already up2date", "branch", branchName, "hash", localHash) + return pushPlan{settled: PushUpToDate, head: remoteHash}, nil + } + + if err := remoteMovedSinceBase(branch, rootBranch, rootHash, refs); err != nil { + logger.Info("Remote branch not in expected state", "branch", branchName, "root", rootBranch.Short()) + return pushPlan{}, err + } + if found { - // Target branch exists on remote oldHash = remoteHash + } else if nothingToPublish(branch, localHash, rootBranch, rootHash) { + // The advertisement still answered the question the caller asked: this branch is not there. + logger.Info("Nothing to publish; the branch stays absent", "branch", branchName) + return pushPlan{settled: PushNoBranch}, nil + } - // Check if we are already up2date - if localHash == remoteHash { - logger.Info("remote already up2date", "branch", branchName, "hash", localHash) - return pushPlan{settled: PushUpToDate, head: remoteHash}, nil - } + // Old stays zero for a new branch, so the server also refuses it if somebody creates it after + // this advertisement. + return pushPlan{old: oldHash, new: localHash}, nil +} - // Check if the remoteHash is what we based our work on - if rootHash != currentRootHash { - logger.Info("Remote branch not in expected state", "branch", branchName) - return pushPlan{}, &RemoteMovedError{ - Branch: rootBranch, - Expected: rootHash, - Advertised: currentRootHash, - } - } +// nothingToPublish reports, for a branch the remote does not carry, that there is nothing to +// create it with: no local commit at all, or a new branch (rooted on another branch, its parent) +// whose local tip is still the parent's, which the caller has just confirmed. Creating it would +// publish nothing but a name, so it stays absent until a write has something to commit. +func nothingToPublish(branch plumbing.ReferenceName, localHash plumbing.Hash, + rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, +) bool { + if localHash.IsZero() { + return true } + return rootBranch != branch && localHash == rootHash +} - // Nothing on the remote and nothing local to send. The advertisement still answered the - // question the caller asked: this branch is not there. - if !found && localHash.IsZero() { - return pushPlan{settled: PushNoBranch}, nil +// remoteMovedSinceBase reports a remote that is no longer where the local commits were based, or +// nil. Both checks hold whether or not the pushed branch exists, because the case they guard +// hardest is the one where it does not. +func remoteMovedSinceBase( + branch, rootBranch plumbing.ReferenceName, + rootHash plumbing.Hash, + refs map[plumbing.ReferenceName]plumbing.Hash, +) error { + // Somebody created the branch we believed absent. Pushing with Old = their commit would + // overwrite it with a commit that does not descend from it, on any server that accepts a + // non-fast-forward update; the root check below cannot see it, because the root is the parent. + if remoteHash, found := refs[branch]; found && rootBranch != branch { + return &RemoteMovedError{Branch: branch, Expected: plumbing.ZeroHash, Advertised: remoteHash} } - return pushPlan{old: oldHash, new: localHash}, nil + // The commits were built on rootHash. For an existing branch that is the branch itself; for a + // new one it is the parent, which must not have moved either: a branch born on a stale parent + // is born behind it. + if advertised := refs[rootBranch]; advertised != rootHash { + return &RemoteMovedError{Branch: rootBranch, Expected: rootHash, Advertised: advertised} + } + return nil } // performPush executes the packfile creation and push operation. diff --git a/internal/git/git_atomic_push_test.go b/internal/git/git_atomic_push_test.go index 244cc2fb..b3b9e29f 100644 --- a/internal/git/git_atomic_push_test.go +++ b/internal/git/git_atomic_push_test.go @@ -123,10 +123,7 @@ func TestAtomicPush_PushToOther_DetectsMovedRoot(t *testing.T) { var moved *RemoteMovedError require.ErrorAs(t, err, &moved) - assert.Equal(t, mainBranch, moved.Branch) - assert.Equal(t, commitA, moved.Expected) - assert.Equal(t, commitB, moved.Advertised) - assert.False(t, moved.Missing) + assert.Equal(t, RemoteMovedError{Branch: mainBranch, Expected: commitA, Advertised: commitB}, *moved) _, err = remoteRepo.Reference(featureBranch, true) require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "the new branch must not be created") @@ -157,9 +154,8 @@ func TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch(t *testing.T) { var moved *RemoteMovedError require.ErrorAs(t, err, &moved) - assert.Equal(t, featureBranch, moved.Branch) - assert.Equal(t, plumbing.ZeroHash, moved.Expected, "we expected the branch to be absent") - assert.Equal(t, theirs, moved.Advertised) + assert.Equal(t, RemoteMovedError{Branch: featureBranch, Expected: plumbing.ZeroHash, Advertised: theirs}, *moved, + "we expected the branch to be absent") ref, err := remoteRepo.Reference(featureBranch, true) require.NoError(t, err) diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 658768a2..0c2c1400 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -67,6 +67,13 @@ type ledgerFixture struct { // has a commit on main, which is the difference between ledger rows 1 and 2. func newLedgerFixture(t *testing.T, slug string, seeded bool) *ledgerFixture { t.Helper() + return newLedgerFixtureOnBranch(t, slug, seeded, "main") +} + +// newLedgerFixtureOnBranch is newLedgerFixture with a worker for another branch than main, which +// the remote does not carry: the write branch of a target that has not written yet. +func newLedgerFixtureOnBranch(t *testing.T, slug string, seeded bool, branch string) *ledgerFixture { + t.Helper() projectRoot := t.TempDir() repoDir := filepath.Join(projectRoot, "repo.git") @@ -104,7 +111,7 @@ func newLedgerFixture(t *testing.T, slug string, seeded bool) *ledgerFixture { logr.Discard(), providerName, "default", - "main", + branch, RepoIdentity{URL: sim.RepoURL}, nil, BranchWorkerLimits{}, diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 96f6df15..d0492480 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -190,24 +190,27 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr // is what makes it honest for the two targets a clone-first version skipped silently — one // declared but never published to, and one whose GitProvider was recreated against a // different repository. - advertised, err := advertiseRemoteBranch( + advertisement, err := advertiseRemoteBranch( w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), auth) if err != nil { return fmt.Errorf("read the remote advertisement: %w", err) } + advertised := advertisement.branch revision := "" if !advertised.IsZero() { revision = advertised.String() } w.recordRemoteObservation(revision, ObservedByFetch) - // The remote does not carry this branch. That IS the answer, and there is nothing to fetch: a - // fetch would fall back to the default branch and teach us nothing about a branch nobody has - // created — the ordinary state of a target that has not written yet. A branch somebody - // DELETED is caught by the compare-and-swap on the next push. + // The remote does not carry this branch — the ordinary state of a target that has not written + // yet. That IS the observation. What the checkout holds instead is the parent the branch would + // be created from, so that is what is followed, from the same advertisement. This keeps the + // folder's published shape about the tree the next commit builds on; it is not what keeps + // that commit fresh, which the push checks for itself. A branch somebody DELETED is caught by + // the compare-and-swap on the next push. if advertised.IsZero() { w.Log.V(1).Info("Refresh found no such branch on the remote", "branch", w.Branch) - return nil + return l.followParent(advertisement.parent) } // Everything below compares the advertisement against the local checkout, so a missing one is @@ -241,6 +244,31 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr return nil } +// followParent brings the checkout of an absent write branch onto its parent's advertised tip. +// It costs nothing beyond the advertisement already read when the parent has not moved, and it +// writes nothing either way. +func (l *branchWorkerEventLoop) followParent(parent plumbing.Hash) error { + w := l.w + // An empty repository has no parent to follow. + if parent.IsZero() { + return nil + } + repo, err := gogit.PlainOpen(w.repoPath()) + if errors.Is(err, gogit.ErrRepositoryNotExists) { + return nil + } + if err != nil { + return fmt.Errorf("open repository: %w", err) + } + if w.baseTrusted() && parent == localHead(repo) { + w.Log.V(1).Info("Refresh confirmed the parent branch has not moved", "branch", w.Branch) + return nil + } + w.Log.Info("Refresh found the parent branch elsewhere; resetting onto it", + "branch", w.Branch, "parent", parent.String()) + return w.syncWithRemote(w.ctx, fetchReasonRefresh) +} + // rescanLayoutForTarget re-reads the target's folder, publishes what its shape resolves to, and // publishes whether the folder can be written at all. // @@ -315,7 +343,17 @@ func (w *BranchWorker) rescanLayoutForTarget(ctx context.Context, req *RefreshRe w.reportLayout(ctx, batch, worktreeRevision(worktree)) } -// advertiseRemoteBranch asks the remote where a branch is, and transfers nothing else. +// remoteAdvertisement is what one ref advertisement says about a write branch. +type remoteAdvertisement struct { + // branch is where the write branch is; zero when the remote does not carry it. + branch plumbing.Hash + // parent is where the remote's default branch is: the parent a new write branch is created + // from. Zero for an empty repository, or a HEAD that names no branch. + parent plumbing.Hash +} + +// advertiseRemoteBranch asks the remote where a branch and its parent are, and transfers nothing +// else. // // It is built on a storage-less remote, the way CheckRepo is, so it needs no clone on disk. Zero // means the advertisement did not carry the branch, which includes an empty repository — an @@ -324,18 +362,23 @@ func advertiseRemoteBranch( remoteURL string, branch plumbing.ReferenceName, auth []gitclient.Option, -) (plumbing.Hash, error) { +) (remoteAdvertisement, error) { remote := gogit.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{remoteURL}}) refs, err := listRemoteRefs(remote, auth) if err != nil { - return plumbing.ZeroHash, err + return remoteAdvertisement{}, err } + hashes := map[plumbing.ReferenceName]plumbing.Hash{} + var defaultBranch plumbing.ReferenceName for _, ref := range refs { - if ref.Type() == plumbing.HashReference && ref.Name() == branch { - return ref.Hash(), nil + switch { + case ref.Type() == plumbing.HashReference: + hashes[ref.Name()] = ref.Hash() + case ref.Name() == plumbing.HEAD && ref.Type() == plumbing.SymbolicReference: + defaultBranch = ref.Target() } } - return plumbing.ZeroHash, nil + return remoteAdvertisement{branch: hashes[branch], parent: hashes[defaultBranch]}, nil } // localHead is the commit the worktree is on, or the zero hash when the branch is unborn. diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index 3e7bfe0f..2e0aba5d 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -4,7 +4,9 @@ package git import ( "os" + "os/exec" "path/filepath" + "strings" "testing" "time" @@ -43,9 +45,15 @@ func newRefreshHarness(t *testing.T, slug string) *refreshHarness { // newRefreshHarnessOn builds the harness on a remote that may or may not already have the branch. func newRefreshHarnessOn(t *testing.T, slug string, seeded bool) *refreshHarness { + t.Helper() + return newRefreshHarnessForBranch(t, slug, seeded, "main") +} + +// newRefreshHarnessForBranch builds the harness for a worker on another branch than main. +func newRefreshHarnessForBranch(t *testing.T, slug string, seeded bool, branch string) *refreshHarness { t.Helper() h := &refreshHarness{ - ledgerFixture: newLedgerFixture(t, slug, seeded), + ledgerFixture: newLedgerFixtureOnBranch(t, slug, seeded, branch), target: itypes.NewResourceReference("checkout", "shop"), path: "team-a", } @@ -239,6 +247,46 @@ func TestRefresh_AnAbsentBranchCostsOneConnection(t *testing.T) { "no revision IS the observation: a branch does not exist without a commit") } +// TestRefresh_AnAbsentBranchFollowsItsParent: while the write branch does not exist, the checkout +// holds its parent, and a parent somebody moved is followed from the same advertisement, so the +// folder the scan re-reads is the one the next commit builds on. It writes nothing. +func TestRefresh_AnAbsentBranchFollowsItsParent(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-absent-parent-moved", true, "feature") + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + h.contend("OUTSIDE.md", "from-another-writer\n") + moved := revParseMain(t, h.repoDir) + + h.refresh(time.Nanosecond) + + assert.Equal(t, int64(1), fetchCount(t, reader, h.worker, fetchReasonRefresh), + "the parent moved, so the checkout has to be brought onto it") + repo, err := gogit.PlainOpen(h.worker.repoPath()) + require.NoError(t, err) + assert.Equal(t, moved, localHead(repo).String(), "the checkout is on the parent's new tip") + require.NotEmpty(t, h.reported) + assert.Empty(t, h.reported[len(h.reported)-1].Commit, "the write branch is still not on the remote") + assert.NotEmpty(t, h.scanVerdicts, "and the folder was re-read from it") + out, err := exec.Command("git", "-C", h.repoDir, "branch", "--list", "feature").Output() + require.NoError(t, err) + assert.Empty(t, strings.TrimSpace(string(out)), "and following it created no branch") +} + +// TestRefresh_AnAbsentBranchWithAnUnmovedParentCostsOneConnection: following the parent is free +// when it has not moved, because the advertisement that answers for the write branch carries it. +func TestRefresh_AnAbsentBranchWithAnUnmovedParentCostsOneConnection(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-absent-parent-unmoved", true, "feature") + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + + assert.Equal(t, int64(1), h.refresh(time.Nanosecond), "one advertisement answers for both branches") + assert.Zero(t, fetchCount(t, reader, h.worker, fetchReasonRefresh)) +} + // TestRefresh_ObservesTheRemoteWithNoCheckoutYet. A GitTarget that has been declared but has not // published has a worker and no clone, and it still deserves an honest status.remote. // @@ -569,3 +617,30 @@ func TestRefresh_WithNoPathScansNothing(t *testing.T) { assert.Empty(t, layouts) } + +// TestRefresh_ANewBranchDoesNotDependOnARecentRefresh: a refresh that completed just before the +// parent moved leaves the checkout one commit behind, and the next write must still create the +// write branch on the parent's current tip. Publication checks the parent itself; a refresh tick is +// never what makes it fresh. +func TestRefresh_ANewBranchDoesNotDependOnARecentRefresh(t *testing.T) { + h := newRefreshHarnessForBranch(t, "refresh-then-parent-moves", true, "feature") + h.worker.mapper = configMapMapper() + h.createLedgerTarget("live", nil) + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + + h.refresh(time.Nanosecond) + h.contend("LATEST.md", "moved right after the refresh\n") + moved := revParseMain(t, h.repoDir) + + loop := newBranchWorkerEventLoop(h.worker, 0) + defer loop.stopTimers() + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("cm1", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + require.Empty(t, loop.pendingWrites, "the write was published") + + out, err := exec.Command("git", "-C", h.repoDir, "rev-parse", "refs/heads/feature^").Output() + require.NoError(t, err) + assert.Equal(t, moved, strings.TrimSpace(string(out)), "the new branch starts on the parent's current tip") +} diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index 2dd20594..be3f483b 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -196,7 +196,8 @@ spec: g.Expect(fetchErr).NotTo(HaveOccurred()) parent, revErr := gitRun(repo.CheckoutDir, "rev-parse", tip+"^") g.Expect(revErr).NotTo(HaveOccurred()) - g.Expect(strings.TrimSpace(parent)).To(Equal(hashB), "the first commit must sit on the parent's current tip") + g.Expect(strings.TrimSpace(parent)).To(Equal(hashB), + "the first commit must sit on the parent's current tip") for _, file := range []string{path.Join(gitPath, "NOTES.md"), editPath} { _, showErr := gitRun(repo.CheckoutDir, "cat-file", "-e", tip+":"+file) g.Expect(showErr).NotTo(HaveOccurred(), "%s must be on the new branch", file) @@ -220,41 +221,50 @@ func commitFilesToMainFromOutside(repo *RepoArtifacts, namespace, message string configureRepoOriginWithCredentials(repo, namespace) Eventually(func() error { - if _, err := gitRun(repo.CheckoutDir, "fetch", "origin", "main"); err == nil { - if out, err := gitRun(repo.CheckoutDir, "checkout", "-B", "main", "origin/main"); err != nil { - return fmt.Errorf("checkout: %w: %s", err, out) - } - if out, err := gitRun(repo.CheckoutDir, "reset", "--hard", "origin/main"); err != nil { - return fmt.Errorf("reset: %w: %s", err, out) - } - } else { - if out, err := gitRun(repo.CheckoutDir, "checkout", "--orphan", "main"); err != nil { - return fmt.Errorf("orphan: %w: %s", err, out) - } - _, _ = gitRun(repo.CheckoutDir, "rm", "-rf", ".") - } - for name, content := range files { - full := filepath.Join(repo.CheckoutDir, name) - if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { - return err - } - if err := os.WriteFile(full, []byte(content), 0o600); err != nil { - return err - } - if out, err := gitRun(repo.CheckoutDir, "add", name); err != nil { - return fmt.Errorf("add: %w: %s", err, out) - } - } - if out, err := gitRun(repo.CheckoutDir, "commit", "-m", message); err != nil { - return fmt.Errorf("commit: %w: %s", err, out) - } - if out, err := gitRun(repo.CheckoutDir, "push", "origin", "HEAD:main"); err != nil { - return fmt.Errorf("push: %w: %s", err, out) - } - return nil + return attemptCommitFilesToMain(repo, message, files) }, seedPushTimeout, seedPushInterval).Should(Succeed(), "the outside writer kept losing the push race") head, err := gitRun(repo.CheckoutDir, "rev-parse", "HEAD") Expect(err).NotTo(HaveOccurred()) return strings.TrimSpace(head) } + +// attemptCommitFilesToMain is one attempt, from the CURRENT remote tip. It returns errors rather +// than asserting, so Eventually can rebuild on the tip that beat it. +func attemptCommitFilesToMain(repo *RepoArtifacts, message string, files map[string]string) error { + runGit := func(args ...string) error { + if out, err := gitRun(repo.CheckoutDir, args...); err != nil { + return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, out) + } + return nil + } + if _, err := gitRun(repo.CheckoutDir, "fetch", "origin", "main"); err == nil { + if err := runGit("checkout", "-B", "main", "origin/main"); err != nil { + return err + } + if err := runGit("reset", "--hard", "origin/main"); err != nil { + return err + } + } else { + if err := runGit("checkout", "--orphan", "main"); err != nil { + return err + } + _, _ = gitRun(repo.CheckoutDir, "rm", "-rf", ".") + } + for name, content := range files { + full := filepath.Join(repo.CheckoutDir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + return err + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + return err + } + if err := runGit("add", name); err != nil { + return err + } + } + if err := runGit("commit", "-m", message); err != nil { + return err + } + return runGit("push", "origin", "HEAD:main") +} From b0fc1d1a06778943acc86e2e56c24c81e23dd5ff Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 07:37:27 +0000 Subject: [PATCH 03/18] feat(api): add GitTarget spec.parentBranch, the branch a new write branch starts from spec.parentBranch names the branch the write branch is created from, and compared against while the remote does not carry it. Omitted keeps the remote's default branch. It is mutable and only read, so it need not be in allowedBranches. A configured parent must exist for the write branch to be created: a fetch that finds neither reports ParentBranchNotFound and writes nothing, so a typo never starts an orphan branch. Every GitTarget on one write branch must name the same parent, compared as written; the later one reports TargetConflict. A change keeps the branch worker and drops its base trust. Co-Authored-By: Claude Opus 5.5 --- api/v1alpha3/gittarget_types.go | 17 ++ .../crd/bases/configbutler.ai_gittargets.yaml | 12 + docs/architecture.md | 14 +- docs/configuration.md | 31 +++ docs/definitions.md | 9 + hack/crdfields/fields.yaml | 6 + internal/controller/condition_helper.go | 6 + internal/controller/gittarget_controller.go | 159 +++++++++---- .../controller/gittarget_immutability_test.go | 44 ++++ internal/controller/gittarget_layout_test.go | 2 +- .../gittarget_parent_branch_test.go | 180 ++++++++++++++ internal/controller/gittarget_status_test.go | 2 +- .../gittarget_worker_cleanup_test.go | 8 +- internal/git/branch_worker.go | 81 ++++++- internal/git/branch_worker_base_trust_test.go | 2 +- internal/git/branch_worker_new_branch_test.go | 12 +- .../git/branch_worker_parent_branch_test.go | 225 ++++++++++++++++++ .../git/branch_worker_queue_depth_test.go | 6 +- internal/git/branch_worker_split_test.go | 2 + internal/git/git.go | 28 ++- internal/git/git_smart_fetch.go | 60 ++++- internal/git/refresh.go | 20 +- internal/git/refresh_test.go | 39 +++ internal/git/remote_observation.go | 3 + internal/git/remote_report_test.go | 4 +- internal/git/replay_failure_test.go | 2 +- internal/git/worker_manager.go | 10 + internal/git/worker_manager_test.go | 16 +- internal/git/worker_repo_identity_test.go | 12 +- internal/watch/event_router_test.go | 4 +- test/e2e/helpers.go | 22 ++ test/e2e/new_branch_parent_e2e_test.go | 89 ++++--- test/e2e/templates/gittarget.tmpl | 3 + 33 files changed, 1003 insertions(+), 127 deletions(-) create mode 100644 internal/controller/gittarget_parent_branch_test.go create mode 100644 internal/git/branch_worker_parent_branch_test.go diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index bfa6c811..3b12f0d6 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -49,6 +49,23 @@ type GitTargetSpec struct { // +kubebuilder:validation:MinLength=1 Branch string `json:"branch"` + // Why "parent" and not base/source/revision: docs/definitions.md reserves those words, and the + // field is flat because it is one fact. It is mutable because it only says where a branch + // starts; changing it abandons nothing already written, unlike branch and path. It can never + // hold a commit or tag: a pinned starting point would be a sibling field. + + // ParentBranch is the branch the write branch (spec.branch) is created from, and the branch the + // target compares against while the write branch does not exist on the remote. Omitted: the + // branch the remote's HEAD points at (its default branch), or a branch with no history when + // the repository is empty. When set, it must exist on the remote for the write branch to be + // created; otherwise the GitTarget reports ParentBranchNotFound and writes nothing. Once the + // write branch exists, it is used as it is and the parent is not followed. It is only read, so + // it does not need to be in the GitProvider's allowedBranches. All GitTargets on one GitProvider + // and branch must name the same parent branch (omitted counts as its own value). + // +optional + // +kubebuilder:validation:MinLength=1 + ParentBranch string `json:"parentBranch,omitempty"` + // Path within the repository to write resources to, relative to the repository // root. Required and must be non-empty — there is no default, so a GitTarget can // never silently write to the repository root. To deliberately target the diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index dfccbdf4..38729338 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -348,6 +348,18 @@ spec: - Ignore - PushEmptyCommit type: string + parentBranch: + description: |- + ParentBranch is the branch the write branch (spec.branch) is created from, and the branch the + target compares against while the write branch does not exist on the remote. Omitted: the + branch the remote's HEAD points at (its default branch), or a branch with no history when + the repository is empty. When set, it must exist on the remote for the write branch to be + created; otherwise the GitTarget reports ParentBranchNotFound and writes nothing. Once the + write branch exists, it is used as it is and the parent is not followed. It is only read, so + it does not need to be in the GitProvider's allowedBranches. All GitTargets on one GitProvider + and branch must name the same parent branch (omitted counts as its own value). + minLength: 1 + type: string path: description: |- Path within the repository to write resources to, relative to the repository diff --git a/docs/architecture.md b/docs/architecture.md index cb125f10..e7a8fb8f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -359,6 +359,10 @@ Key fields: - `spec.gitProviderRef`: a `GitProvider` in the same namespace, by name. - `spec.clusterProviderRef`: a cluster-scoped source `ClusterProvider`; it defaults to `{name: default}`. - `spec.branch`: immutable branch, validated against `GitProvider.spec.allowedBranches`. +- `spec.parentBranch`: optional, mutable branch the write branch is created from and compared + against while it does not exist. Omitted means the remote's default branch. An explicit parent + must exist when creating the write branch. It does not need to be in `allowedBranches` because + it is only read. - `spec.path`: immutable, required path under the repo (`MinLength=1`; `.` means repo root and must be chosen explicitly). - `spec.encryption`: optional SOPS/age encryption settings for sensitive resources. @@ -372,7 +376,8 @@ resolves the window per open window, since a window is bound to exactly one targ `gitProviderRef`, `clusterProviderRef`, `branch`, and `path` are immutable so a target cannot silently orphan an old materialization or change its source cluster. The controller also rejects path overlaps -between GitTargets sharing a provider and branch. +between GitTargets sharing a provider and branch, and rejects two GitTargets on one branch that name +different parent branches. Status has a kstatus-compatible summary layer plus domain conditions: @@ -1240,9 +1245,10 @@ A branch whose remote has been **deleted** takes the same path. The advertisemen branch at all, which is reported as a moved remote with a zero hash, so the replay re-roots on the remote's default branch and the retry re-creates the branch with the retained writes on top. -**A branch that does not exist yet** is created from the remote's default branch (its parent) only -when a write has something to commit, so an idle or in-sync target never creates it. Until then the -worker compares the folder against the parent. Periodic refresh, ten minutes by default, follows the +**A branch that does not exist yet** is created from its parent (`spec.parentBranch`, or the +remote's default branch when that is omitted) only when a write has something to commit, so an +idle or in-sync target never creates it. Until then the worker compares the folder against the +parent. Periodic refresh, ten minutes by default, follows the parent from the same advertisement and updates the idle view independently of the publication check. The push records the parent and the commit it started from as its root, and treats a moved parent like a moved branch: fetch, reset, replay, check again. A branch somebody else created in the diff --git a/docs/configuration.md b/docs/configuration.md index 31a71c4c..6a43ff29 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -79,6 +79,7 @@ The destination fields are immutable: to move a target, delete it and create a n |---|---|---| | `gitProviderRef` | **required** | The `GitProvider` backing this target, in the same namespace | | `branch` | **required** | Branch to write. Must be in the provider's `allowedBranches` | +| `parentBranch` | the remote's default branch | Branch a `branch` the remote does not have yet is created from. Mutable, and only read, so it need not be allowed. See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) | | `path` | **required** | Folder within the repository. `.` targets the root; empty is rejected | | `clusterProviderRef` | `{"name":"default"}` | Source cluster to mirror from. The default names a `ClusterProvider` called `default` | | `commit.window.idleTimeout` | `5s` | Close a commit window after this much silence. `0s` commits every write on its own. See [the commit window](#the-commit-window-speccommitwindow) | @@ -551,6 +552,8 @@ The important fields are: - `spec.clusterProviderRef`: which `ClusterProvider` supplies resources; omit it to reference the user-created `default` provider - `spec.branch`: which allowed branch to write to +- `spec.parentBranch`: which branch `spec.branch` is created from while it does not exist yet; omit it + for the remote's default branch - `spec.path`: required relative path inside the repository; use `.` only when you deliberately want the repository root - `spec.encryption`: how `Secret` resources should be encrypted before commit @@ -747,6 +750,34 @@ two are always separate objects, and deleting the preview cannot disturb the rea For inspecting a repository without a cluster at all, use the `manifest-analyzer` CLI. +### Starting the write branch from another branch (`spec.parentBranch`) + +A target whose `spec.branch` does not exist on the remote waits on standby: it compares the cluster +against the **parent branch** and creates nothing while they agree. The first write that has +something to commit creates the branch from the parent's current tip, checked when it pushes, so a +target that waited a week does not publish onto a week-old checkout. A write that finds nothing to +change creates no branch at all. + +The parent is the remote's default branch unless you name one: + +```yaml +spec: + gitProviderRef: {name: homelab} + branch: reverser/prod-edits # the write branch: created on the first commit + parentBranch: env/prod # what it is created from + path: apps/checkout +``` + +- An explicit parent must exist when the write branch is created. Otherwise the target reports + `Ready=False` with reason `ParentBranchNotFound`, writes nothing, and recovers once the branch is + pushed. An omitted parent never fails this way, and in an empty repository the first commit + starts a branch with no history. +- Once the write branch exists, it is used as it is: new commits on the parent do not reach it. + Delete the write branch after its changes are merged, and the next edit starts a fresh one. +- `parentBranch` is mutable, and it is only read, so it does not need to be in `allowedBranches`. +- All `GitTarget` objects on one `GitProvider` and branch must name the same parent branch; omitted + counts as its own value. The later-created one reports `TargetConflict`. + `spec.suspend` is **not** this. It is the next section. ### Stopping a target from writing (`spec.suspend`) diff --git a/docs/definitions.md b/docs/definitions.md index 0196be68..02dd6726 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -124,6 +124,15 @@ Observation failure supplies no evidence of absence. The code calls this write-s branch)` tuple. Every write to that branch goes through it. Two words, lowercase in prose, `BranchWorker` as the Go type. +**Write branch.** The branch a `GitTarget` writes to: `spec.branch`. It must be in the +`GitProvider`'s `allowedBranches`. + +**Parent branch.** The branch a write branch is created from, and compared against while the write +branch does not exist on the remote: `spec.parentBranch`, or the remote's default branch when that +is omitted. Always with its noun: "parent" alone already names a parent resource. Not "base", which +names a kustomize read-only base and base trust, and not "source", which names the source cluster. +Once the write branch exists, the parent is not followed. + **Commit window.** The interval a branch worker batches one author's writes to one target over before it commits. Only a write opens one; nothing else does. It closes on its `idleTimeout` of silence or its `maxDuration` after opening, whichever comes first. Windows do not merge across authors, which diff --git a/hack/crdfields/fields.yaml b/hack/crdfields/fields.yaml index 67ed1952..15c90547 100644 --- a/hack/crdfields/fields.yaml +++ b/hack/crdfields/fields.yaml @@ -88,6 +88,12 @@ kinds: doc: The `GitProvider` backing this target, in the same namespace - path: branch doc: Branch to write. Must be in the provider's `allowedBranches` + - path: parentBranch + default: the remote's default branch + doc: >- + Branch a `branch` the remote does not have yet is created from. Mutable, and only read, so + it need not be allowed. See + [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) - path: path doc: Folder within the repository. `.` targets the root; empty is rejected - path: clusterProviderRef diff --git a/internal/controller/condition_helper.go b/internal/controller/condition_helper.go index e523eb31..a6a288de 100644 --- a/internal/controller/condition_helper.go +++ b/internal/controller/condition_helper.go @@ -44,3 +44,9 @@ func conditionIsTrue(conditions []metav1.Condition, conditionType string) bool { func conditionIsFalse(conditions []metav1.Condition, conditionType string) bool { return apimeta.IsStatusConditionFalse(conditions, conditionType) } + +// readyReasonIs reports whether the Ready condition carries the given reason. +func readyReasonIs(conditions []metav1.Condition, reason string) bool { + ready := apimeta.FindStatusCondition(conditions, ConditionTypeReady) + return ready != nil && ready.Reason == reason +} diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 945023cb..709c36c6 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -53,9 +53,12 @@ const ( const ( // GitTargetReasonOK is the healthy reason. It is the shared Succeeded vocabulary rather than // a per-kind spelling; the name is kept for call-site stability. - GitTargetReasonOK = ReasonSucceeded - GitTargetReasonProviderNotFound = "ProviderNotFound" - GitTargetReasonBranchNotAllowed = "BranchNotAllowed" + GitTargetReasonOK = ReasonSucceeded + GitTargetReasonProviderNotFound = "ProviderNotFound" + GitTargetReasonBranchNotAllowed = "BranchNotAllowed" + // GitTargetReasonParentBranchNotFound is the Ready reason for a write branch that cannot be + // created: the remote carries neither it nor the spec.parentBranch it would start from. + GitTargetReasonParentBranchNotFound = "ParentBranchNotFound" GitTargetReasonTargetConflict = "TargetConflict" GitTargetReasonNotChecked = "NotChecked" GitTargetReasonBlocked = "Blocked" @@ -181,6 +184,9 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( st := beginStatus(r.Client, r.Recorder, &target) gitPathWasRefused := conditionIsFalse(target.Status.Conditions, GitTargetConditionGitPathAccepted) + // Writes are dropped while the parent is missing, so its return is recovered the way a refused + // folder is: by a recheck that re-derives what the cluster holds. + parentWasMissing := readyReasonIs(target.Status.Conditions, GitTargetReasonParentBranchNotFound) providerNS := target.Namespace // One read of the GitProvider for everything below it; see getGitProvider. @@ -266,7 +272,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // A standing reconcile request forces the same re-check a refused Git path does: the watch // plane re-anchors the target's streams, which is what makes it re-read the folder rather than // wait for the periodic pass. Taken once per distinct annotation value. - forceRecheck := gitPathWasRefused || r.reconcileRequests.take( + forceRecheck := gitPathWasRefused || parentWasMissing || r.reconcileRequests.take( types.NewResourceReference(target.Name, target.Namespace), reconcileRequestedAt(&target)) observed := r.observeDataPlane(&target, sourceProvider, forceRecheck, log) st.setValue(GitTargetConditionStreamsRunning, observed.axes.Streams) @@ -277,7 +283,8 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( rd := newGitTargetReadiness() convergeAsSuspended(rd, &target) - gitTargetReadinessGates(rd, observed, refs.gitProvider, refs.clusterProvider, refs.sourceCluster) + gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS), + refs.gitProvider, refs.clusterProvider, refs.sourceCluster) st.applyReadiness(rd) if err := st.commit(ctx); err != nil { @@ -762,12 +769,14 @@ func (r *GitTargetReconciler) publishReferenceReadiness( func gitTargetReadinessGates( rd *readiness, observed dataPlaneObservation, - provider, clusterProvider, sourceReach conditionValue, + parent, provider, clusterProvider, sourceReach conditionValue, ) { // Terminal, most specific first. Each of these needs a human: the folder holds content the - // operator will not manage, a watch is refused, or what was written no longer matches live. + // operator will not manage, the write branch has no parent to be created from, a watch is + // refused, or what was written no longer matches live. rd.stalledIf(observed.axes.GitPath.Status == metav1.ConditionFalse, observed.axes.GitPath.Reason, observed.axes.GitPath.Message) + rd.stalledIf(parent.Status == metav1.ConditionFalse, parent.Reason, parent.Message) rd.stalledIf(observed.streams.Blocked > 0, observed.streams.Reason, observed.streams.Message) rd.stalledIf(observed.axes.Render.Status == metav1.ConditionFalse, observed.axes.Render.Reason, observed.axes.Render.Message) @@ -803,6 +812,34 @@ func gitTargetReadinessGates( "Stream declaration has not landed yet; the data-plane surface is not observable") } +// parentBranchReadiness reports whether the write branch can be created: False when the last +// look at the remote found neither the write branch nor the configured spec.parentBranch. +// +// It is read from the branch worker's remote observation, so it costs no round trip and holds +// with periodic refresh disabled: every fetch on the write path records the same observation. +// An omitted parent is never missing; it falls back to the remote's default branch. +func (r *GitTargetReconciler) parentBranchReadiness( + target *configbutleraiv1alpha3.GitTarget, + providerNS string, +) conditionValue { + ok := conditionValue{Status: metav1.ConditionTrue, Reason: GitTargetReasonOK} + if target.Spec.ParentBranch == "" { + return ok + } + observed, seen := r.observeRemote(target, providerNS) + if !seen || observed.MissingParent != target.Spec.ParentBranch { + return ok + } + return conditionValue{ + Status: metav1.ConditionFalse, + Reason: GitTargetReasonParentBranchNotFound, + Message: fmt.Sprintf( + "Branch '%s' does not exist on the remote, and neither does its parent branch '%s' to create "+ + "it from; nothing is written until one of them exists", + target.Spec.Branch, target.Spec.ParentBranch), + } +} + // requestRemoteRefresh asks this target's branch worker to re-prove where the branch is, if // nothing has proved it within the interval. // @@ -902,6 +939,7 @@ func (r *GitTargetReconciler) ensureEventStream( providerNS, target.Spec.Branch, repo, + target.Spec.ParentBranch, ) if err != nil { return nil, fmt.Errorf( @@ -1109,41 +1147,16 @@ func (r *GitTargetReconciler) checkForConflicts( if existing.Namespace != providerNS || existing.Spec.GitProviderRef.Name != target.Spec.GitProviderRef.Name { continue } - if existing.Spec.Branch != target.Spec.Branch || - !git.IsValidTargetPath(existing.Spec.Path) || - !gitTargetPathsOverlap(target.Spec.Path, existing.Spec.Path) { + if existing.Spec.Branch != target.Spec.Branch { continue } - // Two GitTargets on the same provider+branch whose paths are equal or - // nested fight over which documents each one owns. The later-created - // target loses (ties broken deterministically by identity) so every - // materialized folder keeps exactly one owner. - if gitTargetLosesConflict(target, existing) { - var msg string - if normalizeGitTargetPath(target.Spec.Path) == normalizeGitTargetPath(existing.Spec.Path) { - msg = fmt.Sprintf( - "Conflict detected. Another GitTarget '%s/%s' (created at %s) is already using GitProvider '%s/%s', branch '%s', path '%s'. This GitTarget was created later and will not be processed.", - existing.Namespace, - existing.Name, - existing.CreationTimestamp.Format(time.RFC3339), - providerNS, - target.Spec.GitProviderRef.Name, - target.Spec.Branch, - target.Spec.Path, - ) - } else { - msg = fmt.Sprintf( - "Conflict detected. This GitTarget's path '%s' overlaps the path '%s' of GitTarget '%s/%s' (created at %s) on GitProvider '%s/%s', branch '%s' — one path nests inside the other (sibling paths are allowed). This GitTarget was created later and will not be processed.", - target.Spec.Path, - existing.Spec.Path, - existing.Namespace, - existing.Name, - existing.CreationTimestamp.Format(time.RFC3339), - providerNS, - target.Spec.GitProviderRef.Name, - target.Spec.Branch, - ) - } + if lost, msg := losesParentBranchConflict(target, existing, providerNS); lost { + return true, msg, GitTargetReasonTargetConflict, nil + } + if existing.Spec.ParentBranch != target.Spec.ParentBranch { + continue // the other target loses, on its own reconcile + } + if lost, msg := losesPathConflict(target, existing, providerNS); lost { return true, msg, GitTargetReasonTargetConflict, nil } } @@ -1151,6 +1164,72 @@ func (r *GitTargetReconciler) checkForConflicts( return false, "", "", nil } +// losesPathConflict reports whether target loses to existing, on the same write branch, because +// their paths are equal or nested. Two such targets fight over which documents each one owns. The +// later-created target loses (ties broken deterministically by identity) so every materialized +// folder keeps exactly one owner. +func losesPathConflict(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) (bool, string) { + if !git.IsValidTargetPath(existing.Spec.Path) || + !gitTargetPathsOverlap(target.Spec.Path, existing.Spec.Path) || + !gitTargetLosesConflict(target, existing) { + return false, "" + } + if normalizeGitTargetPath(target.Spec.Path) == normalizeGitTargetPath(existing.Spec.Path) { + return true, fmt.Sprintf( + "Conflict detected. Another GitTarget '%s/%s' (created at %s) is already using GitProvider '%s/%s', branch '%s', path '%s'. This GitTarget was created later and will not be processed.", + existing.Namespace, + existing.Name, + existing.CreationTimestamp.Format(time.RFC3339), + providerNS, + target.Spec.GitProviderRef.Name, + target.Spec.Branch, + target.Spec.Path, + ) + } + return true, fmt.Sprintf( + "Conflict detected. This GitTarget's path '%s' overlaps the path '%s' of GitTarget '%s/%s' (created at %s) on GitProvider '%s/%s', branch '%s' — one path nests inside the other (sibling paths are allowed). This GitTarget was created later and will not be processed.", + target.Spec.Path, + existing.Spec.Path, + existing.Namespace, + existing.Name, + existing.CreationTimestamp.Format(time.RFC3339), + providerNS, + target.Spec.GitProviderRef.Name, + target.Spec.Branch, + ) +} + +// losesParentBranchConflict reports whether target loses to existing, on the same write branch, +// because the two name different parent branches. One branch worker owns one checkout per write +// branch, so every target on it has to agree on where the branch starts. Compared as written, with +// omitted as its own value: the answer needs no remote round trip and cannot flip when somebody +// changes the remote's default branch. +func losesParentBranchConflict(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) (bool, string) { + if existing.Spec.ParentBranch == target.Spec.ParentBranch || !gitTargetLosesConflict(target, existing) { + return false, "" + } + return true, parentBranchConflictMessage(target, existing, providerNS) +} + +// parentBranchConflictMessage explains a TargetConflict over spec.parentBranch. +func parentBranchConflictMessage(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) string { + return fmt.Sprintf( + "Conflict detected. GitTarget '%s/%s' (created at %s) writes GitProvider '%s/%s', branch '%s' with parent "+ + "branch %s, and this GitTarget names parent branch %s. All GitTargets on one branch must name the same "+ + "parent branch. This GitTarget was created later and will not be processed.", + existing.Namespace, existing.Name, existing.CreationTimestamp.Format(time.RFC3339), + providerNS, target.Spec.GitProviderRef.Name, target.Spec.Branch, + describeParentBranch(existing.Spec.ParentBranch), describeParentBranch(target.Spec.ParentBranch), + ) +} + +func describeParentBranch(name string) string { + if name == "" { + return "(omitted: the remote's default branch)" + } + return "'" + name + "'" +} + func (r *GitTargetReconciler) ensureEncryptionSecret( ctx context.Context, target *configbutleraiv1alpha3.GitTarget, diff --git a/internal/controller/gittarget_immutability_test.go b/internal/controller/gittarget_immutability_test.go index d73d15f1..2ab6f899 100644 --- a/internal/controller/gittarget_immutability_test.go +++ b/internal/controller/gittarget_immutability_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" @@ -75,6 +76,49 @@ var _ = Describe("GitTarget Destination Immutability", func() { }, "spec.gitProviderRef is immutable") }) + // spec.parentBranch is NOT part of the destination: it says where a write branch starts, so it + // can change on a live target. It can be omitted, but not set to the empty string. + It("allows spec.parentBranch to be set, changed and cleared, but never empty", func() { + ctx := context.Background() + key := types.NamespacedName{Name: "parent-branch-target", Namespace: "default"} + + gitTarget := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "prov-a"}, + Branch: "edits", + Path: "apps", + ParentBranch: "main", + }, + } + Expect(k8sClient.Create(ctx, gitTarget)).Should(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, gitTarget) }) + + for _, parent := range []string{"release", ""} { + Eventually(func(g Gomega) { + current := &configbutleraiv1alpha3.GitTarget{} + g.Expect(k8sClient.Get(ctx, key, current)).To(Succeed()) + current.Spec.ParentBranch = parent + g.Expect(k8sClient.Update(ctx, current)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + } + + empty := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "configbutler.ai/v1alpha3", + "kind": "GitTarget", + "metadata": map[string]interface{}{"name": "empty-parent-target", "namespace": "default"}, + "spec": map[string]interface{}{ + "gitProviderRef": map[string]interface{}{"name": "prov-a"}, + "branch": "edits", + "path": "apps", + "parentBranch": "", + }, + }} + err := k8sClient.Create(ctx, empty) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.parentBranch")) + }) + It("requires a non-empty path: rejects an omitted or empty path but allows an explicit \".\" root", func() { ctx := context.Background() key := types.NamespacedName{Name: "root-policy-target", Namespace: "default"} diff --git a/internal/controller/gittarget_layout_test.go b/internal/controller/gittarget_layout_test.go index 391dde11..bab5a610 100644 --- a/internal/controller/gittarget_layout_test.go +++ b/internal/controller/gittarget_layout_test.go @@ -206,7 +206,7 @@ func TestGitTargetReadiness_StalledFollowsGitPathAccepted(t *testing.T) { }, Render: conditionValue{Status: metav1.ConditionTrue}, }, - }, healthyDependency(), healthyDependency(), healthyDependency()) + }, healthyDependency(), healthyDependency(), healthyDependency(), healthyDependency()) trio := rd.trio() assert.Equal(t, wantStalled.Status, string(trio.Stalled.Status)) diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go new file mode 100644 index 00000000..6549514d --- /dev/null +++ b/internal/controller/gittarget_parent_branch_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/types" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +func parentBranchTarget(name, branch, path, parent string, created time.Time) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "shop", CreationTimestamp: metav1.NewTime(created), + }, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "repo1"}, + Branch: branch, + Path: path, + ParentBranch: parent, + }, + } +} + +func parentBranchScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configbutleraiv1alpha3.AddToScheme(scheme)) + return scheme +} + +// TestCheckForConflicts_ParentBranch: one write branch, one checkout, one parent. Two targets on the +// same branch that name different parents conflict even on sibling paths, and the later one loses. +// Omitted is its own value, so it conflicts with an explicit "main". +func TestCheckForConflicts_ParentBranch(t *testing.T) { + earlier := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + later := earlier.Add(time.Hour) + + cases := []struct { + name string + existing *configbutleraiv1alpha3.GitTarget + target *configbutleraiv1alpha3.GitTarget + wantConflict bool + }{ + { + name: "omitted versus main conflicts", + existing: parentBranchTarget("first", "edits", "apps/a", "", earlier), + target: parentBranchTarget("second", "edits", "apps/b", "main", later), + wantConflict: true, + }, + { + name: "the earlier target wins", + existing: parentBranchTarget("second", "edits", "apps/b", "main", later), + target: parentBranchTarget("first", "edits", "apps/a", "", earlier), + }, + { + name: "the same parent on sibling paths is fine", + existing: parentBranchTarget("first", "edits", "apps/a", "release", earlier), + target: parentBranchTarget("second", "edits", "apps/b", "release", later), + }, + { + name: "another write branch is another checkout", + existing: parentBranchTarget("first", "edits", "apps/a", "", earlier), + target: parentBranchTarget("second", "other", "apps/b", "release", later), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + k8sClient := fake.NewClientBuilder().WithScheme(parentBranchScheme(t)).WithObjects(tc.existing).Build() + r := &GitTargetReconciler{Client: k8sClient} + + conflict, msg, reason, err := r.checkForConflicts(context.Background(), tc.target, "shop") + + require.NoError(t, err) + assert.Equal(t, tc.wantConflict, conflict, msg) + if tc.wantConflict { + assert.Equal(t, GitTargetReasonTargetConflict, reason) + assert.Contains(t, msg, "(omitted: the remote's default branch)") + assert.Contains(t, msg, "'main'") + } + }) + } +} + +func startParentBranchWorkers(t *testing.T) (*GitTargetReconciler, *git.WorkerManager) { + t.Helper() + k8sClient := fake.NewClientBuilder().WithScheme(parentBranchScheme(t)).Build() + workers := git.NewWorkerManager(k8sClient, logr.Discard(), git.BranchWorkerLimits{}, + types.SensitiveResourcePolicy{}) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = workers.Start(ctx) }() + time.Sleep(100 * time.Millisecond) // Start installs the context EnsureWorker starts workers under + router := watch.NewEventRouter(workers, nil, k8sClient, logr.Discard()) + return &GitTargetReconciler{Client: k8sClient, WorkerManager: workers, EventRouter: router}, workers +} + +// TestEnsureEventStream_AParentBranchChangeKeepsTheWorker: changing spec.parentBranch abandons +// nothing written, so it is not a repoint. The worker stays and re-reads the remote. +func TestEnsureEventStream_AParentBranchChangeKeepsTheWorker(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "", time.Now()) + + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + before, ok := workers.GetWorkerForTarget("repo1", "shop", "edits") + require.True(t, ok) + before.SeedRemoteObservationForTest("", time.Now(), "Fetch") + + target.Spec.ParentBranch = "release" + _, err = r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + + after, ok := workers.GetWorkerForTarget("repo1", "shop", "edits") + require.True(t, ok) + assert.Same(t, before, after, "the worker and its clone are kept") + assert.Equal(t, "release", after.ParentBranch()) + assert.False(t, after.BaseTrustedForTest(), "and its checkout of the old parent is no longer trusted") +} + +// TestParentBranchReadiness: a fetch that found neither the write branch nor its configured parent +// stalls the GitTarget as ParentBranchNotFound. An omitted parent never does. +func TestParentBranchReadiness(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) + + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop").Status, + "nothing has looked yet") + + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + worker.RecordMissingParentForTest("release") + + got := r.parentBranchReadiness(target, "shop") + assert.Equal(t, metav1.ConditionFalse, got.Status) + assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) + assert.Contains(t, got.Message, "'release'") + + rd := newGitTargetReadiness() + gitTargetReadinessGates(rd, dataPlaneObservation{axes: gitTargetAxes{ + Streams: conditionValue{Status: metav1.ConditionTrue}, + GitPath: conditionValue{Status: metav1.ConditionTrue}, + Render: conditionValue{Status: metav1.ConditionTrue}, + }}, got, healthyDependency(), healthyDependency(), healthyDependency()) + trio := rd.trio() + assert.Equal(t, metav1.ConditionTrue, trio.Stalled.Status, "it needs somebody to create the branch") + assert.Equal(t, GitTargetReasonParentBranchNotFound, trio.Ready.Reason) + + target.Spec.ParentBranch = "other" + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop").Status, + "the observation is about another parent") + target.Spec.ParentBranch = "" + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop").Status) +} + +func TestReadyReasonIs(t *testing.T) { + conditions := []metav1.Condition{{Type: ConditionTypeReady, Status: metav1.ConditionFalse, + Reason: GitTargetReasonParentBranchNotFound}} + assert.True(t, readyReasonIs(conditions, GitTargetReasonParentBranchNotFound)) + assert.False(t, readyReasonIs(conditions, GitTargetReasonTargetConflict)) + assert.False(t, readyReasonIs(nil, GitTargetReasonParentBranchNotFound)) +} diff --git a/internal/controller/gittarget_status_test.go b/internal/controller/gittarget_status_test.go index 714abe0c..e5a1d656 100644 --- a/internal/controller/gittarget_status_test.go +++ b/internal/controller/gittarget_status_test.go @@ -164,7 +164,7 @@ func TestGitTargetReadinessGates(t *testing.T) { st.setValue(GitTargetConditionRenderMatchesLive, observed.axes.Render) rd := newGitTargetReadiness() - gitTargetReadinessGates(rd, observed, tt.provider, tt.clusterProvider, tt.sourceReach) + gitTargetReadinessGates(rd, observed, conditionValue{}, tt.provider, tt.clusterProvider, tt.sourceReach) st.applyReadiness(rd) ready := findCondition(target.Status.Conditions, GitTargetConditionReady) diff --git a/internal/controller/gittarget_worker_cleanup_test.go b/internal/controller/gittarget_worker_cleanup_test.go index a354e6b8..a79fd7ec 100644 --- a/internal/controller/gittarget_worker_cleanup_test.go +++ b/internal/controller/gittarget_worker_cleanup_test.go @@ -45,7 +45,7 @@ func TestCleanupDeletedGitTarget_StopsAWorkerNothingNeedsAnyMore(t *testing.T) { time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() _, exists := workers.GetWorkerForTarget("repo1", "shop", "main") @@ -81,7 +81,7 @@ func TestCleanupDeletedGitTarget_LeavesAWorkerAnotherGitTargetStillUses(t *testi go func() { _ = workers.Start(ctx) }() time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() @@ -125,7 +125,7 @@ func TestCleanupDeletedGitTarget_RetriesWhenTheSweepCannotRead(t *testing.T) { go func() { _ = workers.Start(ctx) }() time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() @@ -158,7 +158,7 @@ func TestHandleFetchError_RequeuesAFailedCleanup(t *testing.T) { go func() { _ = workers.Start(ctx) }() time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 3c3b85a7..7837a29a 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -56,7 +56,7 @@ var ( //nolint:gochecknoglobals fetchRemoteBranchHashFn = fetchRemoteBranchHash //nolint:gochecknoglobals - syncToRemoteFn = syncToRemote + syncToRemoteFn = syncToRemoteFrom ) // BranchWorker processes events for a single (GitProvider, Branch) combination. @@ -238,6 +238,11 @@ type BranchWorker struct { newBranchParent plumbing.ReferenceName newBranchParentHash plumbing.Hash + // parentBranchName is the GitTarget's spec.parentBranch: the branch a write branch the remote + // does not carry is created from. Empty means the remote's default branch. It is set by the + // reconcile and read by the event loop, hence atomic. + parentBranchName atomic.Pointer[string] + // firsts surfaces the first successful commit and push at default verbosity. firsts branchWorkerLogFirsts @@ -923,7 +928,7 @@ func (w *BranchWorker) prepareBootstrapRepository( repoPath := w.repoPath() w.recordFetch(fetchReasonBootstrap) - pullReport, err := PrepareBranch(ctx, w.repo.URL, repoPath, w.Branch, auth) + pullReport, err := w.prepareBranch(ctx, repoPath, auth) if err != nil { return "", fmt.Errorf("failed to prepare repository: %w", err) } @@ -1858,7 +1863,7 @@ func (w *BranchWorker) ensureBaseForCycle( } w.recordFetch(reason) - pullReport, err := PrepareBranch(w.ctx, w.repo.URL, repoPath, w.Branch, auth) + pullReport, err := w.prepareBranch(w.ctx, repoPath, auth) if err != nil { return fmt.Errorf("prepare repository: %w", err) } @@ -2035,7 +2040,7 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { // changed nothing. w.markReplayRequired() w.recordFetch(fetchReasonContention) - pullReport, syncErr := syncToRemoteFn(w.ctx, repo, plumbing.NewBranchReferenceName(w.Branch), auth) + pullReport, syncErr := w.syncToRemote(w.ctx, repo, auth) if syncErr != nil { w.invalidateBase("sync during replay failed") return fmt.Errorf("sync remote during replay: %w", syncErr) @@ -2185,7 +2190,7 @@ func (w *BranchWorker) refreshRemoteAndRebuildPendingWrites( // false, which is precisely the state no other flag describes. w.markReplayRequired() w.recordFetch(reason) - pullReport, err := syncToRemoteFn(ctx, repo, plumbing.NewBranchReferenceName(w.Branch), auth) + pullReport, err := w.syncToRemote(ctx, repo, auth) if err != nil { w.invalidateBase("sync before replay failed") return fmt.Errorf("sync remote before replay: %w", err) @@ -2345,6 +2350,52 @@ func (w *BranchWorker) endPushCycle(kind PushOutcomeKind) { } } +// ParentBranch is the configured parent branch, or empty for the remote's default branch. +func (w *BranchWorker) ParentBranch() string { + if name := w.parentBranchName.Load(); name != nil { + return *name + } + return "" +} + +// SetParentBranch configures the parent branch. A change invalidates the base, so the next cycle +// fetches and compares against the new parent; the worker and its clone are kept, because +// nothing already written is abandoned. It reports whether the value changed. +func (w *BranchWorker) SetParentBranch(name string) bool { + if w.ParentBranch() == name { + return false + } + w.parentBranchName.Store(&name) + w.invalidateBase("parent branch changed") + return true +} + +// prepareBranch is PrepareBranch for this worker's branch and parent. A configured parent the +// remote does not carry is recorded as an observation, because it is one: the advertisement said +// so, and the GitTarget reports it as ParentBranchNotFound. +func (w *BranchWorker) prepareBranch( + ctx context.Context, repoPath string, auth []gitclient.Option, +) (*PullReport, error) { + report, err := PrepareBranchFrom(ctx, w.repo.URL, repoPath, w.Branch, w.ParentBranch(), auth) + w.noteMissingParent(err) + return report, err +} + +// syncToRemote is syncToRemoteFn for this worker's branch and parent; see prepareBranch. +func (w *BranchWorker) syncToRemote( + ctx context.Context, repo *gogit.Repository, auth []gitclient.Option, +) (*PullReport, error) { + report, err := syncToRemoteFn(ctx, repo, plumbing.NewBranchReferenceName(w.Branch), w.ParentBranch(), auth) + w.noteMissingParent(err) + return report, err +} + +func (w *BranchWorker) noteMissingParent(err error) { + if errors.Is(err, ErrParentBranchNotFound) { + w.recordMissingParent(w.ParentBranch(), ObservedByFetch) + } +} + // clearNewBranchParent forgets the parent once the remote carries the write branch. func (w *BranchWorker) clearNewBranchParent() { w.newBranchParent = "" @@ -2711,9 +2762,17 @@ func (w *BranchWorker) LastRemoteObservation() (RemoteObservation, bool) { // that died mid-upload or an advertisement that never arrived observed nothing, and recording a // guess there is how a stale revision reaches status. func (w *BranchWorker) recordRemoteObservation(revision string, by ObservationSource) { + w.publishObservation(RemoteObservation{Commit: revision, At: time.Now(), By: by, Repo: w.repo}) +} + +// recordMissingParent records that the write branch is absent and its configured parent is too. +func (w *BranchWorker) recordMissingParent(parent string, by ObservationSource) { + w.publishObservation(RemoteObservation{At: time.Now(), By: by, Repo: w.repo, MissingParent: parent}) +} + +func (w *BranchWorker) publishObservation(observed RemoteObservation) { // Stamped with the repository it is about, so that whoever publishes it can tell whether it // still describes the repository the GitTarget points at. See RemoteObservation.Repo. - observed := RemoteObservation{Commit: revision, At: time.Now(), By: by, Repo: w.repo} w.lastObservation.Store(&observed) // Delivered here, at the one point where an observation is made, rather than by each caller // against whichever GitTargets it happened to be holding. The fact is about the BRANCH, so it @@ -2751,7 +2810,7 @@ func (w *BranchWorker) syncWithRemote(ctx context.Context, reason string) error // PrepareBranch handles both initial and update cases w.recordFetch(reason) - report, err := PrepareBranch(ctx, w.repo.URL, repoPath, w.Branch, auth) + report, err := w.prepareBranch(ctx, repoPath, auth) if err != nil { return fmt.Errorf("failed to sync with remote: %w", err) } @@ -2786,7 +2845,7 @@ func (w *BranchWorker) ensureRepositoryInitialized(ctx context.Context) error { } // Use new PrepareBranch abstraction - pullReport, err := PrepareBranch(ctx, w.repo.URL, repoPath, w.Branch, auth) + pullReport, err := w.prepareBranch(ctx, repoPath, auth) if err != nil { return fmt.Errorf("failed to prepare repository: %w", err) } @@ -2875,5 +2934,11 @@ func (w *BranchWorker) SeedRemoteObservationForTest(revision string, at time.Tim w.lastObservation.Store(&observed) } +// RecordMissingParentForTest records the observation a fetch makes when neither the write branch +// nor its configured parent exists, for tests in another package. +func (w *BranchWorker) RecordMissingParentForTest(parent string) { + w.recordMissingParent(parent, ObservedByFetch) +} + // BaseTrustedForTest exposes the flag to tests in another package. func (w *BranchWorker) BaseTrustedForTest() bool { return w.baseTrusted() } diff --git a/internal/git/branch_worker_base_trust_test.go b/internal/git/branch_worker_base_trust_test.go index 437efe52..bcb9bd21 100644 --- a/internal/git/branch_worker_base_trust_test.go +++ b/internal/git/branch_worker_base_trust_test.go @@ -188,7 +188,7 @@ func TestBaseTrust_LostWhenSomebodySaysTheRemoteMoved(t *testing.T) { syncFailed := errors.New("sync failed") original := syncToRemoteFn syncToRemoteFn = func( - _ context.Context, _ *gogit.Repository, _ plumbing.ReferenceName, _ []gitclient.Option, + _ context.Context, _ *gogit.Repository, _ plumbing.ReferenceName, _ string, _ []gitclient.Option, ) (*PullReport, error) { return nil, syncFailed } diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go index c399ff5f..c40357a5 100644 --- a/internal/git/branch_worker_new_branch_test.go +++ b/internal/git/branch_worker_new_branch_test.go @@ -37,8 +37,8 @@ import ( ) const ( - newBranchTarget = "target-a" - newBranchFolder = "live" + newBranchTarget = "standby-target" + newBranchFolder = "standby" ) // newBranchFixture is a remote whose main is seeded and whose write branch `feature` is absent, @@ -88,7 +88,8 @@ func newNewBranchFixture(t *testing.T, seedFiles map[string]string) *newBranchFi f.worker.ctx = ctx f.worker.mapper = configMapMapper() t.Cleanup(func() { _ = os.RemoveAll(f.worker.repoRootPath()) }) - createGitTargetWithPruneMode(t, f.worker, newBranchTarget, newBranchFolder, configv1alpha3.PruneAlways) + createGitTarget(t, f.worker, newBranchTarget, newBranchFolder, + &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneAlways}) return f } @@ -448,9 +449,10 @@ func TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent(t *testing.T) { t.Cleanup(func() { syncToRemoteFn = original }) moves := 0 syncToRemoteFn = func( - ctx context.Context, repo *git.Repository, branch plumbing.ReferenceName, auth []gitclient.Option, + ctx context.Context, repo *git.Repository, branch plumbing.ReferenceName, parent string, + auth []gitclient.Option, ) (*PullReport, error) { - report, err := original(ctx, repo, branch, auth) + report, err := original(ctx, repo, branch, parent, auth) moves++ f.pushToMain(map[string]string{"MOVING.md": time.Now().String() + string(rune('a'+moves))}) return report, err diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go new file mode 100644 index 00000000..b398fc8f --- /dev/null +++ b/internal/git/branch_worker_parent_branch_test.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// spec.parentBranch names the branch a write branch the remote does not carry is created from. +// Omitted keeps the remote's default branch, which the tests in branch_worker_new_branch_test.go +// cover; these cover a configured one: it is followed while the write branch is absent, it must +// exist for the write branch to be created, and it stops mattering once the write branch exists. + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// pushToRelease commits one file to `release` from outside, creating it from main when absent. +func (f *newBranchFixture) pushToRelease(file, content string) plumbing.Hash { + f.t.Helper() + return simulateClientCommitOnDisk(f.t, f.remote, "release", file, content) +} + +func (f *newBranchFixture) commitParent(tip plumbing.Hash) plumbing.Hash { + f.t.Helper() + commit, err := f.server.CommitObject(tip) + require.NoError(f.t, err) + require.Len(f.t, commit.ParentHashes, 1) + return commit.ParentHashes[0] +} + +func TestSmartFetchFrom_ConfiguredParent(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + remoteURL := "file://" + filepath.Join(tempDir, "remote.git") + createBareRepo(t, filepath.Join(tempDir, "remote.git")) + simulateClientCommitOnDisk(t, remoteURL, "main", "README.md", "main\n") + simulateClientCommitOnDisk(t, remoteURL, "release", "RELEASE.md", "release\n") + simulateClientCommitOnDisk(t, remoteURL, "existing", "EXISTING.md", "existing\n") + + cases := []struct { + name, target, parent string + want plumbing.ReferenceName + wantErr error + }{ + {name: "omitted falls back to the default branch", target: "feature", want: "refs/heads/main"}, + {name: "a configured parent replaces the default", target: "feature", parent: "release", + want: "refs/heads/release"}, + {name: "a missing configured parent is refused", target: "feature", parent: "typo", + wantErr: ErrParentBranchNotFound}, + {name: "an existing target ignores a missing parent", target: "existing", parent: "typo", + want: "refs/heads/existing"}, + {name: "a parent equal to an absent target is refused", target: "feature", parent: "feature", + wantErr: ErrParentBranchNotFound}, + {name: "a parent equal to an existing target is that branch", target: "existing", parent: "existing", + want: "refs/heads/existing"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + repo, _ := initLocalRepo(t, filepath.Join(t.TempDir(), "local"), remoteURL, "") + got, err := SmartFetchFrom(ctx, repo, plumbing.NewBranchReferenceName(tc.target), tc.parent, nil) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } + + t.Run("an empty repository refuses a configured parent", func(t *testing.T) { + emptyURL := "file://" + filepath.Join(t.TempDir(), "empty.git") + createBareRepo(t, emptyURL[len("file://"):]) + repo, _ := initLocalRepo(t, filepath.Join(t.TempDir(), "local"), emptyURL, "") + _, err := SmartFetchFrom(ctx, repo, plumbing.NewBranchReferenceName("main"), "main", nil) + require.ErrorIs(t, err, ErrParentBranchNotFound) + + got, err := SmartFetchFrom(ctx, repo, plumbing.NewBranchReferenceName("main"), "", nil) + require.NoError(t, err) + assert.Empty(t, got, "an omitted parent keeps the empty-repository bootstrap") + }) +} + +// TestBranchWorker_NewBranchStartsFromTheConfiguredParent: the write branch is created on the +// configured parent's tip as publication saw it, even when the parent moved after the checkout. +func TestBranchWorker_NewBranchStartsFromTheConfiguredParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release at R1\n") + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + branch, _ := f.localHead() + require.Equal( + t, + plumbing.NewBranchReferenceName("release"), + branch, + "the absent write branch checks out its parent", + ) + r2 := f.pushToRelease("RELEASE-2.md", "release at R2\n") + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, r2, f.commitParent(tip), "created on the configured parent's current tip") + assert.True(t, treeHas(t, f.server, tip, "RELEASE-2.md")) +} + +// TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists: a configured parent the +// remote does not carry refuses the write, starts no orphan branch, and is reported; once it is +// pushed the next write creates the branch on it. +func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + assert.Empty(t, loop.pendingWrites, "nothing was committed") + assert.True(t, loop.lastPushAt.IsZero(), "and nothing was pushed") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "no orphan branch") + observed, known := f.worker.LastRemoteObservation() + require.True(t, known) + assert.Equal(t, "release", observed.MissingParent) + + r1 := f.pushToRelease("RELEASE.md", "now it exists\n") + liveWrite(loop, "cm2") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "it recovers once the parent exists") + assert.Equal(t, r1, f.commitParent(tip)) + observed, _ = f.worker.LastRemoteObservation() + assert.Empty(t, observed.MissingParent) +} + +// TestBranchWorker_ParentBranchEqualToTheWriteBranch: legal when the branch exists, and then it is +// simply that branch; refused when it does not, because nothing names where to start it. +func TestBranchWorker_ParentBranchEqualToTheWriteBranch(t *testing.T) { + t.Run("the branch exists", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + existing := simulateClientCommitOnDisk(t, f.remote, "feature", "FEATURE.md", "feature\n") + f.worker.SetParentBranch("feature") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + tip, _ := f.featureOnRemote() + assert.Equal(t, existing, f.commitParent(tip)) + }) + + t.Run("the branch is absent", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.worker.SetParentBranch("feature") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote) + observed, _ := f.worker.LastRemoteObservation() + assert.Equal(t, "feature", observed.MissingParent) + }) + + t.Run("an empty repository", func(t *testing.T) { + worker, server, _ := setupCommitPushSplitWorkerOnEmptyRemote(t) + worker.mapper = configMapMapper() + createGitTarget(t, worker, newBranchTarget, newBranchFolder, + &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneAlways}) + worker.SetParentBranch("main") + loop := newBranchWorkerEventLoop(worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + _, err := server.Reference(plumbing.NewBranchReferenceName("main"), true) + require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "an explicit parent never bootstraps") + }) +} + +// TestBranchWorker_AnExistingWriteBranchDoesNotFollowItsParent is the rule kept from Flux and Argo +// CD: the parent decides where the branch starts, and nothing after that. +func TestBranchWorker_AnExistingWriteBranchDoesNotFollowItsParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release at R1\n") + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + first, onRemote := f.featureOnRemote() + require.True(t, onRemote) + + f.pushToRelease("RELEASE-2.md", "release at R2\n") + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + tip, _ := f.featureOnRemote() + assert.Equal(t, first, f.commitParent(tip), "the write branch builds on itself") + assert.False(t, treeHas(t, f.server, tip, "RELEASE-2.md")) +} + +// TestBranchWorker_SetParentBranchInvalidatesTheBase: a changed parent is re-read on the next +// cycle; an unchanged one costs nothing. +func TestBranchWorker_SetParentBranchInvalidatesTheBase(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.warmOnParent() + + assert.False(t, f.worker.SetParentBranch(""), "unchanged") + assert.True(t, f.worker.baseTrusted()) + + assert.True(t, f.worker.SetParentBranch("release")) + assert.Equal(t, "release", f.worker.ParentBranch()) + assert.False(t, f.worker.baseTrusted(), "the checkout holds the old parent") +} diff --git a/internal/git/branch_worker_queue_depth_test.go b/internal/git/branch_worker_queue_depth_test.go index 30f6ce2e..f4ba4e0b 100644 --- a/internal/git/branch_worker_queue_depth_test.go +++ b/internal/git/branch_worker_queue_depth_test.go @@ -130,7 +130,7 @@ func TestEnsureWorker_WaitsForTheWorkerItReplacesToStop(t *testing.T) { ensured := make(chan struct{}) go func() { - _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", RepoIdentity{}) + _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", RepoIdentity{}, "") close(ensured) }() @@ -178,7 +178,7 @@ func TestReconcileWorkers_DoesNotStopAWorkerEnsureWorkerJustHandedOut(t *testing // The SAME repository the retiring worker is about, deliberately: that is what makes // an unguarded EnsureWorker take its "already there" path, which is the regression // this test is here to catch. - _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", retiring.repo) + _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", retiring.repo, "") }() // Long enough for an unguarded EnsureWorker to finish inside the window; a guarded one is // blocked on lifecycleMu and finishes after the sweep instead. @@ -243,7 +243,7 @@ func TestReconcileWorkers_DoesNotStopAWorkerCreatedAfterItsSnapshot(t *testing.T go func() { defer close(ensured) _ = manager.EnsureWorker( - context.Background(), "newcomer-provider", "test-ns", "main", RepoIdentity{}) + context.Background(), "newcomer-provider", "test-ns", "main", RepoIdentity{}, "") }() time.Sleep(200 * time.Millisecond) }) diff --git a/internal/git/branch_worker_split_test.go b/internal/git/branch_worker_split_test.go index 8d7ee5c6..3bcb70cf 100644 --- a/internal/git/branch_worker_split_test.go +++ b/internal/git/branch_worker_split_test.go @@ -615,6 +615,7 @@ func TestBranchWorker_TransientPushFailure_RetriesSameLocalCommits(t *testing.T) _ context.Context, _ *git.Repository, _ plumbing.ReferenceName, + _ string, _ []gitclient.Option, ) (*PullReport, error) { syncCalled = true @@ -694,6 +695,7 @@ func TestBranchWorker_PushFollowedByFetchFailure_TreatsAsTransient(t *testing.T) _ context.Context, _ *git.Repository, _ plumbing.ReferenceName, + _ string, _ []gitclient.Option, ) (*PullReport, error) { syncCalled = true diff --git a/internal/git/git.go b/internal/git/git.go index 3e4f1529..d46ca9af 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -98,9 +98,20 @@ func PrepareBranch( ctx context.Context, repoURL, repoPath, targetBranchName string, auth []gitclient.Option, +) (*PullReport, error) { + return PrepareBranchFrom(ctx, repoURL, repoPath, targetBranchName, "", auth) +} + +// PrepareBranchFrom is PrepareBranch with an explicit parent branch for a target branch the remote +// does not carry; empty means the remote's default branch. See SmartFetchFrom. +func PrepareBranchFrom( + ctx context.Context, + repoURL, repoPath, targetBranchName, parentBranch string, + auth []gitclient.Option, ) (*PullReport, error) { logger := log.FromContext(ctx) - logger.Info("Preparing branch for operations", "url", repoURL, "path", repoPath, "branch", targetBranchName) + logger.Info("Preparing branch for operations", "url", repoURL, "path", repoPath, "branch", targetBranchName, + "parentBranch", parentBranch) // Ensure the directory exists if err := os.MkdirAll(filepath.Dir(repoPath), 0750); err != nil { @@ -136,7 +147,7 @@ func PrepareBranch( } targetBranch := plumbing.NewBranchReferenceName(targetBranchName) - pullReport, err := syncToRemote(ctx, repo, targetBranch, auth) + pullReport, err := syncToRemoteFrom(ctx, repo, targetBranch, parentBranch, auth) if err != nil { return nil, err } @@ -366,13 +377,24 @@ func syncToRemote( repo *git.Repository, branch plumbing.ReferenceName, auth []gitclient.Option, +) (*PullReport, error) { + return syncToRemoteFrom(ctx, repo, branch, "", auth) +} + +// syncToRemoteFrom is syncToRemote with an explicit parent branch; see SmartFetchFrom. +func syncToRemoteFrom( + ctx context.Context, + repo *git.Repository, + branch plumbing.ReferenceName, + parent string, + auth []gitclient.Option, ) (*PullReport, error) { _, currentHash, err := GetCurrentBranch(repo) if err != nil { return nil, fmt.Errorf("unexpected fail to read HEAD: %w", err) } - availableBranch, err := SmartFetch(ctx, repo, branch, auth) + availableBranch, err := SmartFetchFrom(ctx, repo, branch, parent, auth) if err != nil { return nil, fmt.Errorf("failed to fetch: %w", err) } diff --git a/internal/git/git_smart_fetch.go b/internal/git/git_smart_fetch.go index 5458afb1..47447e55 100644 --- a/internal/git/git_smart_fetch.go +++ b/internal/git/git_smart_fetch.go @@ -27,6 +27,26 @@ func SmartFetch( repo *git.Repository, target plumbing.ReferenceName, // e.g. "refs/heads/feature" or "HEAD" auth []gitclient.Option, +) (plumbing.ReferenceName, error) { + return SmartFetchFrom(ctx, repo, target, "", auth) +} + +// ErrParentBranchNotFound reports that a configured parent branch is absent on the remote while +// the target branch is absent too, so there is nothing to create the target branch from. It is +// never reported for an omitted parent: that falls back to the remote's default branch, and to +// an unborn branch in an empty repository. +var ErrParentBranchNotFound = errors.New("parent branch not found on the remote") + +// SmartFetchFrom is SmartFetch with an explicit parent: the branch to fall back to when the target +// branch is absent. Empty means the remote's default branch. A configured parent that is absent is +// an error rather than a fallback, so a typo can never start an orphan branch; it does not matter +// while the target branch exists. +func SmartFetchFrom( + ctx context.Context, + repo *git.Repository, + target plumbing.ReferenceName, + parent string, + auth []gitclient.Option, ) (plumbing.ReferenceName, error) { remoteName := "origin" remote, err := repo.Remote(remoteName) @@ -39,12 +59,17 @@ func SmartFetch( if err != nil { return "", err } - if len(refs) == 0 { + if len(refs) == 0 && parent == "" { return "", nil } // 2. Analyze: Find default branch and check target existence - defaultFull, defaultShort, targetExists := analyzeRemoteRefs(ctx, refs, target.String()) + headFull, headShort, targetExists := analyzeRemoteRefs(ctx, refs, target.String()) + + defaultFull, defaultShort, err := fallbackBranch(refs, parent, headFull, headShort, targetExists) + if err != nil { + return "", err + } // 3. Plan: Build RefSpecs based on analysis refSpecs := buildSmartRefSpecs(remoteName, defaultFull, defaultShort, target, targetExists) @@ -75,12 +100,39 @@ func SmartFetch( } } - // 5. Repair: Fix local symbolic HEAD - repairRemoteSymbolicHead(repo, remoteName, defaultShort) + // 5. Repair: Fix local symbolic HEAD. It mirrors the remote's HEAD, never a configured parent. + repairRemoteSymbolicHead(repo, remoteName, headShort) return result, nil } +// fallbackBranch is the branch to fall back on, and to fetch as the safety net: the configured +// parent, or the remote's default branch when none is configured. A configured parent the remote +// does not carry is an error only while the target is absent too. +func fallbackBranch( + refs []*plumbing.Reference, parent, headFull, headShort string, targetExists bool, +) (string, string, error) { + if parent == "" { + return headFull, headShort, nil + } + full, short := configuredParent(refs, parent) + if full == "" && !targetExists { + return "", "", fmt.Errorf("%w: %q", ErrParentBranchNotFound, parent) + } + return full, short, nil +} + +// configuredParent finds a configured parent branch in the advertisement, or returns empty names. +func configuredParent(refs []*plumbing.Reference, parent string) (string, string) { + full := plumbing.NewBranchReferenceName(parent) + for _, ref := range refs { + if ref.Name() == full && ref.Type() == plumbing.HashReference { + return full.String(), parent + } + } + return "", "" +} + func listRemoteRefs(remote *git.Remote, auth []gitclient.Option) ([]*plumbing.Reference, error) { refs, err := remote.List(&git.ListOptions{ClientOptions: auth}) if errors.Is(err, transport.ErrEmptyRemoteRepository) { diff --git a/internal/git/refresh.go b/internal/git/refresh.go index d0492480..bccb940c 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -190,12 +190,19 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr // is what makes it honest for the two targets a clone-first version skipped silently — one // declared but never published to, and one whose GitProvider was recreated against a // different repository. + parentBranch := w.ParentBranch() advertisement, err := advertiseRemoteBranch( - w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), auth) + w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), parentBranch, auth) if err != nil { return fmt.Errorf("read the remote advertisement: %w", err) } advertised := advertisement.branch + // Neither the branch nor the parent it would be created from: there is nothing to follow, + // and the GitTarget reports the parent as missing until somebody creates it. + if advertised.IsZero() && parentBranch != "" && advertisement.parent.IsZero() { + w.recordMissingParent(parentBranch, ObservedByFetch) + return nil + } revision := "" if !advertised.IsZero() { revision = advertised.String() @@ -347,13 +354,14 @@ func (w *BranchWorker) rescanLayoutForTarget(ctx context.Context, req *RefreshRe type remoteAdvertisement struct { // branch is where the write branch is; zero when the remote does not carry it. branch plumbing.Hash - // parent is where the remote's default branch is: the parent a new write branch is created - // from. Zero for an empty repository, or a HEAD that names no branch. + // parent is where the parent a new write branch is created from is: the configured parent + // branch, or the remote's default branch when none is configured. Zero when the remote does + // not carry it, which includes an empty repository. parent plumbing.Hash } // advertiseRemoteBranch asks the remote where a branch and its parent are, and transfers nothing -// else. +// else. An empty parentBranch means the branch the remote's HEAD names. // // It is built on a storage-less remote, the way CheckRepo is, so it needs no clone on disk. Zero // means the advertisement did not carry the branch, which includes an empty repository — an @@ -361,6 +369,7 @@ type remoteAdvertisement struct { func advertiseRemoteBranch( remoteURL string, branch plumbing.ReferenceName, + parentBranch string, auth []gitclient.Option, ) (remoteAdvertisement, error) { remote := gogit.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{remoteURL}}) @@ -378,6 +387,9 @@ func advertiseRemoteBranch( defaultBranch = ref.Target() } } + if parentBranch != "" { + defaultBranch = plumbing.NewBranchReferenceName(parentBranch) + } return remoteAdvertisement{branch: hashes[branch], parent: hashes[defaultBranch]}, nil } diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index 2e0aba5d..b70f7a08 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -644,3 +644,42 @@ func TestRefresh_ANewBranchDoesNotDependOnARecentRefresh(t *testing.T) { require.NoError(t, err) assert.Equal(t, moved, strings.TrimSpace(string(out)), "the new branch starts on the parent's current tip") } + +// TestRefresh_FollowsTheConfiguredParent: with spec.parentBranch set, the refresher follows that +// branch rather than the remote's default, and a parent changed on a live target is re-read on the +// next refresh. +func TestRefresh_FollowsTheConfiguredParent(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-configured-parent", true, "feature") + release := simulateClientCommitOnDisk(t, h.repoDir, "release", "RELEASE.md", "release\n") + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + + require.True(t, h.worker.SetParentBranch("release"), "changed on a live target") + h.refresh(time.Nanosecond) + + assert.Equal(t, int64(1), fetchCount(t, reader, h.worker, fetchReasonRefresh)) + repo, err := gogit.PlainOpen(h.worker.repoPath()) + require.NoError(t, err) + assert.Equal(t, release, localHead(repo), "the checkout is on the configured parent") + assert.NotEmpty(t, h.scanVerdicts, "and the folder was re-read from it") +} + +// TestRefresh_ReportsAMissingConfiguredParent: one advertisement shows neither the write branch nor +// its configured parent, which is reported and fetches nothing. +func TestRefresh_ReportsAMissingConfiguredParent(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-missing-parent", true, "feature") + h.worker.SetParentBranch("release") + h.reported = nil + + assert.Equal(t, int64(1), h.refresh(time.Nanosecond)) + assert.Zero(t, fetchCount(t, reader, h.worker, fetchReasonRefresh)) + require.NotEmpty(t, h.reported) + last := h.reported[len(h.reported)-1] + assert.Equal(t, "release", last.MissingParent) + assert.Empty(t, last.Commit) +} diff --git a/internal/git/remote_observation.go b/internal/git/remote_observation.go index 465f3b6c..26b82b88 100644 --- a/internal/git/remote_observation.go +++ b/internal/git/remote_observation.go @@ -50,6 +50,9 @@ type RemoteObservation struct { // It is zero for a worker that has no identity of its own: the CLI, and tests that never reach // a remote. A comparison against an unknown identity proves nothing, so it is not made. Repo RepoIdentity + // MissingParent names the configured parent branch when the remote carries neither it nor the + // branch, so the branch cannot be created. Empty otherwise, including for an omitted parent. + MissingParent string } // Age is how long ago the observation was made, against now. diff --git a/internal/git/remote_report_test.go b/internal/git/remote_report_test.go index b17ac9f4..d8ffddeb 100644 --- a/internal/git/remote_report_test.go +++ b/internal/git/remote_report_test.go @@ -56,12 +56,12 @@ func TestRemoteForBranch_SurvivesTheWorkerItWasProvedBy(t *testing.T) { before := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} after := RepoIdentity{ProviderUID: "uid-2", URL: "https://example.invalid/second.git"} - require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) + require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) worker, ok := m.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) worker.recordRemoteObservation("aaaa", ObservedByPush) - require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) observed, known := m.RemoteForBranch(key) require.True(t, known, diff --git a/internal/git/replay_failure_test.go b/internal/git/replay_failure_test.go index 45941630..574a7ace 100644 --- a/internal/git/replay_failure_test.go +++ b/internal/git/replay_failure_test.go @@ -334,7 +334,7 @@ func TestReplayFailure_AResetThatDiesHalfwayIsAlsoHeld(t *testing.T) { original := syncToRemoteFn syncToRemoteFn = func( ctx context.Context, repo *gogit.Repository, - branch plumbing.ReferenceName, auth []gitclient.Option, + branch plumbing.ReferenceName, _ string, auth []gitclient.Option, ) (*PullReport, error) { if _, err := SmartFetch(ctx, repo, branch, auth); err != nil { return nil, err diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 566b6af7..2b149fd1 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -283,12 +283,17 @@ func (m *WorkerManager) RemoteForBranch(key BranchKey) (RemoteObservation, bool) // and the caller can fail. See ReplacementPending; the delivery is the GitTarget reconcile's // worker wiring gate. // +// parentBranch is the GitTarget's spec.parentBranch. A change is not a replacement: it says where +// a write branch the remote does not carry is created from, so it abandons nothing already +// written. The worker keeps its clone and drops its base trust instead. +// // Worker creation/start is protected by the manager lock. func (m *WorkerManager) EnsureWorker( _ context.Context, providerName, providerNamespace string, branch string, repo RepoIdentity, + parentBranch string, ) error { // Held across the whole check-and-create so a replacement cannot start while the worker it // replaces is still stopping; they would share a clone. @@ -307,6 +312,10 @@ func (m *WorkerManager) EnsureWorker( if exists { if existing.repo == repo { + if existing.SetParentBranch(parentBranch) { + m.Log.Info("Parent branch changed; the worker re-reads the remote on its next cycle", + "key", key.String(), "parentBranch", parentBranch) + } return nil } // Both identities, at default verbosity: a comparison that is not stable across steady @@ -338,6 +347,7 @@ func (m *WorkerManager) EnsureWorker( // Inject the resolver before Start: the field is read only by the event-loop // goroutine Start spawns, so setting it here (under m.mu, before that goroutine // exists) is race-free. + worker.SetParentBranch(parentBranch) worker.mapper = m.mapper worker.clusterMapper = m.clusterMapper worker.sshHostKeys = m.sshHostKeys diff --git a/internal/git/worker_manager_test.go b/internal/git/worker_manager_test.go index 5fe9c1e7..3ce329b9 100644 --- a/internal/git/worker_manager_test.go +++ b/internal/git/worker_manager_test.go @@ -68,7 +68,7 @@ func mustEnsureWorker( repo RepoIdentity, ) bool { t.Helper() - require.NoError(t, m.EnsureWorker(ctx, providerName, providerNamespace, branch, repo)) + require.NoError(t, m.EnsureWorker(ctx, providerName, providerNamespace, branch, repo, "")) key := BranchKey{RepoNamespace: providerNamespace, RepoName: providerName, Branch: branch} pending := m.ReplacementPending(key) m.AcknowledgeReplacement(key) @@ -144,7 +144,7 @@ func TestEnsureWorker_CreatesAWorkerWithTheIdentityItWasAskedFor(t *testing.T) { createTargetForRegister(ctx, t, client, "target1", "repo1", "main", "clusters/prod") // Register first target - err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo) + err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "") if err != nil { t.Fatalf("Failed to register target: %v", err) } @@ -237,12 +237,12 @@ func TestWorkerManagerDifferentBranches(t *testing.T) { createTargetForRegister(ctx, t, client, "target-dev", "repo1", "develop", "base/") // Register targets for same repo, different branches - err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo) + err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "") if err != nil { t.Fatalf("Failed to register target-main: %v", err) } - err = manager.EnsureWorker(ctx, "repo1", "gitops-system", "develop", repo) + err = manager.EnsureWorker(ctx, "repo1", "gitops-system", "develop", repo, "") if err != nil { t.Fatalf("Failed to register target-dev: %v", err) } @@ -362,7 +362,7 @@ func TestWorkerManagerConcurrentRegistration(t *testing.T) { done := make(chan bool, 10) for i := range 10 { go func(index int) { - err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo) + err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "") if err != nil { t.Errorf("Failed to ensure worker %d: %v", index, err) } @@ -529,7 +529,7 @@ func TestWorkerLifecycle_BuildsAndReclaimsItsOwnState(t *testing.T) { root := workerStateRoot repo := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "")) worker, ok := manager.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) require.True(t, strings.HasPrefix(worker.repoPath(), root+string(filepath.Separator)), @@ -573,7 +573,7 @@ func TestWorkerLifecycle_AReplacementDoesNotReclaimItsPredecessorsCheckout(t *te before := RepoIdentity{ProviderUID: "uid-1", URL: sameURL} after := RepoIdentity{ProviderUID: "uid-2", URL: sameURL} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) old, ok := manager.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) require.NoError(t, os.MkdirAll(old.repoPath(), 0o750)) @@ -585,7 +585,7 @@ func TestWorkerLifecycle_AReplacementDoesNotReclaimItsPredecessorsCheckout(t *te sentinel := filepath.Join(successor, "HEAD") require.NoError(t, os.WriteFile(sentinel, []byte("ref: refs/heads/main\n"), 0o600)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) replacement, ok := manager.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) require.NotSame(t, old, replacement) diff --git a/internal/git/worker_repo_identity_test.go b/internal/git/worker_repo_identity_test.go index bc23e70c..6fd70ab0 100644 --- a/internal/git/worker_repo_identity_test.go +++ b/internal/git/worker_repo_identity_test.go @@ -352,21 +352,21 @@ func TestEnsureWorker_AnUndeliveredRecoveryIsStillPendingOnTheNextPass(t *testin before := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} after := RepoIdentity{ProviderUID: "uid-2", URL: "https://example.invalid/second.git"} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) require.False(t, manager.ReplacementPending(key)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) require.True(t, manager.ReplacementPending(key)) // The delivery fails, so nothing acknowledges it. Two more reconciles of this target, and of // a sibling on the same branch, still find the recovery waiting. - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) assert.True(t, manager.ReplacementPending(key), "a recovery nobody delivered must survive the calls that come after it") manager.AcknowledgeReplacement(key) assert.False(t, manager.ReplacementPending(key)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) assert.False(t, manager.ReplacementPending(key), "and a delivered one must not be asked for again on every steady tick") } @@ -379,8 +379,8 @@ func TestReconcileWorkers_DropsARecoveryForABranchNothingNeeds(t *testing.T) { before := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} after := RepoIdentity{ProviderUID: "uid-2", URL: "https://example.invalid/second.git"} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) require.True(t, manager.ReplacementPending(key)) // No GitTarget names this branch, so the sweep retires its worker. diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index db2c3946..7a5c6222 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -353,8 +353,8 @@ func TestServiceCommitRequest_RegisteredWorkerResolvesNoOpenWindow(t *testing.T) ensureErr := workerManager.EnsureWorker( ctx, "team-a-provider", "team-a", "main", - git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, - ) + git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, "") + require.NoError(t, ensureErr) router := NewEventRouter(workerManager, nil, client, logr.Discard()) diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 3300d14c..81d74f2a 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -252,6 +252,26 @@ func createGitTargetWithOptions( encryptionSecretName string, generateWhenMissing bool, commit gitTargetCommitOptions, +) { + applyGitTarget(name, namespace, providerName, path, branch, "", encryptionSecretName, generateWhenMissing, commit) +} + +// createGitTargetWithParentBranch is createGitTarget with spec.parentBranch; empty omits it. +func createGitTargetWithParentBranch(name, namespace, providerName, path, branch, parentBranch string) { + applyGitTarget(name, namespace, providerName, path, branch, parentBranch, + e2eEncryptionRefName, false, gitTargetCommitOptions{Window: e2eCommitWindow}) +} + +func applyGitTarget( + name, + namespace, + providerName, + path, + branch, + parentBranch, + encryptionSecretName string, + generateWhenMissing bool, + commit gitTargetCommitOptions, ) { By(fmt.Sprintf("creating GitTarget '%s' in ns '%s' for GitProvider '%s' with path '%s'", name, namespace, providerName, path)) @@ -261,6 +281,7 @@ func createGitTargetWithOptions( Namespace string ProviderName string Branch string + ParentBranch string Path string EncryptionSecretName string GenerateWhenMissing bool @@ -273,6 +294,7 @@ func createGitTargetWithOptions( Namespace: namespace, ProviderName: providerName, Branch: branch, + ParentBranch: parentBranch, Path: path, EncryptionSecretName: encryptionSecretName, GenerateWhenMissing: generateWhenMissing, diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index be3f483b..f335393a 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -31,7 +31,19 @@ import ( // `contention` fetch (the rejected push), with no `refresh` fetch in between. That takes the // periodic refresher out of the proof without redeploying the controller with it disabled, which // would disturb every other spec on the leg. -var _ = Describe("New write branch starts from its parent", Label("manager"), Ordered, func() { +// +// It runs twice: once with spec.parentBranch omitted (the remote's default branch, main), and once +// naming `release`, which the GitProvider does not allow writing to because it is only read. +var _ = Describe("New write branch starts from its parent", Label("manager"), func() { + Context("with spec.parentBranch omitted", Ordered, func() { + describeNewWriteBranchSpec("", "main", "parent-branch") + }) + Context("with spec.parentBranch naming release", Ordered, func() { + describeNewWriteBranchSpec("release", "release", "parent-release") + }) +}) + +func describeNewWriteBranchSpec(parentBranch, parent, slug string) { const ( providerName = "parent-provider" destName = "parent-dest" @@ -94,11 +106,11 @@ data: BeforeAll(func() { ensurePrometheusClient() - testNs = testNamespaceFor("manager-parent-branch") + testNs = testNamespaceFor("manager-" + slug) _, _ = kubectlRun("create", "namespace", testNs) writeBranch = fmt.Sprintf("reverser-e2e-%d", GinkgoRandomSeed()) - repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-parent-branch-%d", GinkgoRandomSeed())) + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-%s-%d", slug, GinkgoRandomSeed())) _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") applySOPSAgeKeyToNamespace(testNs) @@ -127,14 +139,19 @@ spec: }) It("stays on standby, then creates the write branch on the parent's current tip", func() { - By("seeding main with commit A, whose folder already holds a document the cluster will match") - commitFilesToMainFromOutside(repo, testNs, "e2e: seed A", map[string]string{ + By("seeding " + parent + " with commit A, whose folder already holds a document the cluster will match") + if parent != "main" { + commitFilesToBranchFromOutside(repo, testNs, "main", "e2e: seed main", map[string]string{ + "README.md": "main is not the parent here\n", + }) + } + commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: seed A", map[string]string{ "README.md": "seed\n", path.Join(gitPath, testNs, "configmaps", "standby-present.yaml"): mirroredDocument("standby-present"), }) By("creating a GitTarget on an absent write branch, and a WatchRule that selects nothing yet") - createGitTarget(destName, testNs, providerName, gitPath, writeBranch) + createGitTargetWithParentBranch(destName, testNs, providerName, gitPath, writeBranch, parentBranch) rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 kind: WatchRule metadata: @@ -178,10 +195,10 @@ spec: Expect(remoteBranchHeadOf(Default, repo.CheckoutDir, writeBranch)).To(BeEmpty(), "a write that commits nothing must not create the write branch") - By("main moves to B from outside, touching the target's folder") + By(parent + " moves to B from outside, touching the target's folder") contentionBefore, refreshBefore := fetches("contention"), fetches("refresh") - hashB := commitFilesToMainFromOutside(repo, testNs, "e2e: B moves main", map[string]string{ - path.Join(gitPath, "NOTES.md"): "added on main at B\n", + hashB := commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: B moves the parent", map[string]string{ + path.Join(gitPath, "NOTES.md"): "added on the parent at B\n", }) By("a live edit arrives with no resync in between") @@ -211,17 +228,20 @@ spec: Expect(fetches("refresh")).To(Equal(refreshBefore), "no refresh fetch may stand in for the publication check") }) -}) +} -// commitFilesToMainFromOutside commits files to main and pushes it directly, the way another -// writer would, and returns the new tip. It starts main when the repository is still empty, and -// retries because losing a push race with the controller says nothing about the behavior under test. -func commitFilesToMainFromOutside(repo *RepoArtifacts, namespace, message string, files map[string]string) string { +// commitFilesToBranchFromOutside commits files to a branch and pushes it directly, the way another +// writer would, and returns the new tip. A branch the remote does not carry yet starts from main, +// or from nothing in an empty repository. It retries because losing a push race with the +// controller says nothing about the behavior under test. +func commitFilesToBranchFromOutside( + repo *RepoArtifacts, namespace, branch, message string, files map[string]string, +) string { GinkgoHelper() configureRepoOriginWithCredentials(repo, namespace) Eventually(func() error { - return attemptCommitFilesToMain(repo, message, files) + return attemptCommitFilesToBranch(repo, branch, message, files) }, seedPushTimeout, seedPushInterval).Should(Succeed(), "the outside writer kept losing the push race") head, err := gitRun(repo.CheckoutDir, "rev-parse", "HEAD") @@ -229,27 +249,17 @@ func commitFilesToMainFromOutside(repo *RepoArtifacts, namespace, message string return strings.TrimSpace(head) } -// attemptCommitFilesToMain is one attempt, from the CURRENT remote tip. It returns errors rather +// attemptCommitFilesToBranch is one attempt, from the CURRENT remote tip. It returns errors rather // than asserting, so Eventually can rebuild on the tip that beat it. -func attemptCommitFilesToMain(repo *RepoArtifacts, message string, files map[string]string) error { +func attemptCommitFilesToBranch(repo *RepoArtifacts, branch, message string, files map[string]string) error { runGit := func(args ...string) error { if out, err := gitRun(repo.CheckoutDir, args...); err != nil { return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, out) } return nil } - if _, err := gitRun(repo.CheckoutDir, "fetch", "origin", "main"); err == nil { - if err := runGit("checkout", "-B", "main", "origin/main"); err != nil { - return err - } - if err := runGit("reset", "--hard", "origin/main"); err != nil { - return err - } - } else { - if err := runGit("checkout", "--orphan", "main"); err != nil { - return err - } - _, _ = gitRun(repo.CheckoutDir, "rm", "-rf", ".") + if err := checkoutRemoteTipOf(repo, branch, runGit); err != nil { + return err } for name, content := range files { full := filepath.Join(repo.CheckoutDir, name) @@ -266,5 +276,24 @@ func attemptCommitFilesToMain(repo *RepoArtifacts, message string, files map[str if err := runGit("commit", "-m", message); err != nil { return err } - return runGit("push", "origin", "HEAD:main") + return runGit("push", "origin", "HEAD:refs/heads/"+branch) +} + +// checkoutRemoteTipOf checks out branch at its remote tip; a branch the remote does not carry +// starts from main, and from an orphan when main is absent too. +func checkoutRemoteTipOf(repo *RepoArtifacts, branch string, runGit func(...string) error) error { + for _, start := range []string{branch, "main"} { + if _, err := gitRun(repo.CheckoutDir, "fetch", "origin", start); err != nil { + continue + } + if err := runGit("checkout", "-B", branch, "origin/"+start); err != nil { + return err + } + return runGit("reset", "--hard", "origin/"+start) + } + if err := runGit("checkout", "--orphan", branch); err != nil { + return err + } + _, _ = gitRun(repo.CheckoutDir, "rm", "-rf", ".") + return nil } diff --git a/test/e2e/templates/gittarget.tmpl b/test/e2e/templates/gittarget.tmpl index 3511362c..3a5fccc9 100644 --- a/test/e2e/templates/gittarget.tmpl +++ b/test/e2e/templates/gittarget.tmpl @@ -7,6 +7,9 @@ spec: gitProviderRef: name: {{ .ProviderName }} branch: {{ .Branch }} +{{- if .ParentBranch }} + parentBranch: {{ .ParentBranch }} +{{- end }} path: {{ .Path }} commit: window: From 55e3eb5db8ceca27c1bdb6094e7938804c813de1 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 08:13:47 +0000 Subject: [PATCH 04/18] fix(git): rebuild retained writes on a parent change, and scope the parent checks Review fixes for spec.parentBranch: - A parent change now reaches writes retained during the push cooldown. They were planned against the old parent, and an invalidated base does not reach them, so the event loop takes the change and refetches and replays before the next commit or push. A missing new parent then refuses publication instead of creating the branch from the old one. - Parent conflicts apply whatever either target's path is. A target with a path its writer rejects still wires the shared worker and would set its parent; path validity now gates only the overlap check. - ParentBranchNotFound ignores an observation proved against another repository than the one the GitProvider names now. Co-Authored-By: Claude Opus 5.5 --- .coverage-baseline | 2 +- internal/controller/gittarget_controller.go | 25 +++++---- .../gittarget_parent_branch_test.go | 54 +++++++++++++++++-- internal/git/branch_worker.go | 13 ++++- .../git/branch_worker_parent_branch_test.go | 34 ++++++++++++ 5 files changed, 110 insertions(+), 18 deletions(-) diff --git a/.coverage-baseline b/.coverage-baseline index 7a60a910..9336dfa7 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -79.0 +79.1 diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 709c36c6..0b0760a8 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -283,7 +283,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( rd := newGitTargetReadiness() convergeAsSuspended(rd, &target) - gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS), + gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS, repoIdentityOf(gitProvider)), refs.gitProvider, refs.clusterProvider, refs.sourceCluster) st.applyReadiness(rd) @@ -817,17 +817,20 @@ func gitTargetReadinessGates( // // It is read from the branch worker's remote observation, so it costs no round trip and holds // with periodic refresh disabled: every fetch on the write path records the same observation. -// An omitted parent is never missing; it falls back to the remote's default branch. +// An omitted parent is never missing; it falls back to the remote's default branch. An +// observation proved against another repository than the one the GitProvider names now (a +// repoint) says nothing about this one. func (r *GitTargetReconciler) parentBranchReadiness( target *configbutleraiv1alpha3.GitTarget, providerNS string, + repo git.RepoIdentity, ) conditionValue { ok := conditionValue{Status: metav1.ConditionTrue, Reason: GitTargetReasonOK} if target.Spec.ParentBranch == "" { return ok } observed, seen := r.observeRemote(target, providerNS) - if !seen || observed.MissingParent != target.Spec.ParentBranch { + if !seen || observed.MissingParent != target.Spec.ParentBranch || differentRepository(observed.Repo, repo) { return ok } return conditionValue{ @@ -1127,13 +1130,6 @@ func (r *GitTargetReconciler) checkForConflicts( target *configbutleraiv1alpha3.GitTarget, providerNS string, ) (bool, string, string, error) { - // A path the writer would reject (absolute, backslashes, ".." traversal) owns - // nothing, so it must neither block others nor be blocked here — its own write - // path fails it. Only well-formed paths participate in overlap detection. - if !git.IsValidTargetPath(target.Spec.Path) { - return false, "", "", nil - } - var allTargets configbutleraiv1alpha3.GitTargetList if err := r.List(ctx, &allTargets); err != nil { return false, "", "", fmt.Errorf("list GitTargets for conflict validation: %w", err) @@ -1150,6 +1146,8 @@ func (r *GitTargetReconciler) checkForConflicts( if existing.Spec.Branch != target.Spec.Branch { continue } + // Checked whatever either path looks like: a target with a path its writer will reject + // still wires the shared worker, and would set its parent. if lost, msg := losesParentBranchConflict(target, existing, providerNS); lost { return true, msg, GitTargetReasonTargetConflict, nil } @@ -1168,8 +1166,13 @@ func (r *GitTargetReconciler) checkForConflicts( // their paths are equal or nested. Two such targets fight over which documents each one owns. The // later-created target loses (ties broken deterministically by identity) so every materialized // folder keeps exactly one owner. +// +// A path the writer would reject (absolute, backslashes, ".." traversal) owns nothing, so it must +// neither block others nor be blocked here — its own write path fails it. Only well-formed paths +// participate in overlap detection. func losesPathConflict(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) (bool, string) { - if !git.IsValidTargetPath(existing.Spec.Path) || + if !git.IsValidTargetPath(target.Spec.Path) || + !git.IsValidTargetPath(existing.Spec.Path) || !gitTargetPathsOverlap(target.Spec.Path, existing.Spec.Path) || !gitTargetLosesConflict(target, existing) { return false, "" diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index 6549514d..85fa854a 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -141,7 +141,7 @@ func TestParentBranchReadiness(t *testing.T) { repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) - assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop").Status, + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status, "nothing has looked yet") _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) @@ -149,7 +149,7 @@ func TestParentBranchReadiness(t *testing.T) { worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") worker.RecordMissingParentForTest("release") - got := r.parentBranchReadiness(target, "shop") + got := r.parentBranchReadiness(target, "shop", repo) assert.Equal(t, metav1.ConditionFalse, got.Status) assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) assert.Contains(t, got.Message, "'release'") @@ -165,10 +165,10 @@ func TestParentBranchReadiness(t *testing.T) { assert.Equal(t, GitTargetReasonParentBranchNotFound, trio.Ready.Reason) target.Spec.ParentBranch = "other" - assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop").Status, + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status, "the observation is about another parent") target.Spec.ParentBranch = "" - assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop").Status) + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status) } func TestReadyReasonIs(t *testing.T) { @@ -178,3 +178,49 @@ func TestReadyReasonIs(t *testing.T) { assert.False(t, readyReasonIs(conditions, GitTargetReasonTargetConflict)) assert.False(t, readyReasonIs(nil, GitTargetReasonParentBranchNotFound)) } + +// TestCheckForConflicts_ParentBranchAppliesToInvalidPaths: a target whose path its writer will +// reject still wires the shared worker, so it must not escape the parent check in either role and +// repoint the healthy sibling's parent. +func TestCheckForConflicts_ParentBranchAppliesToInvalidPaths(t *testing.T) { + earlier := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + healthy := parentBranchTarget("first", "edits", "apps/a", "main", earlier) + invalid := parentBranchTarget("second", "edits", "../bad", "release", earlier.Add(time.Hour)) + + r, workers := startParentBranchWorkers(t) + require.NoError(t, r.Client.Create(context.Background(), healthy)) + _, err := r.ensureEventStream(context.Background(), healthy, "shop", firstRepo, logr.Discard()) + require.NoError(t, err) + + conflict, _, reason, err := r.checkForConflicts(context.Background(), invalid, "shop") + require.NoError(t, err) + assert.True(t, conflict, "an invalid path must not bypass the parent conflict") + assert.Equal(t, GitTargetReasonTargetConflict, reason) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + assert.Equal(t, "main", worker.ParentBranch()) + + // And as the earlier object, the invalid target still wins the parent. + k8sClient := fake.NewClientBuilder().WithScheme(parentBranchScheme(t)). + WithObjects(parentBranchTarget("older", "edits", "../bad", "release", earlier)).Build() + conflict, _, _, err = (&GitTargetReconciler{Client: k8sClient}).checkForConflicts( + context.Background(), parentBranchTarget("newer", "edits", "apps/a", "main", earlier.Add(time.Hour)), "shop") + require.NoError(t, err) + assert.True(t, conflict) +} + +// TestParentBranchReadiness_IgnoresAnotherRepository: the manager keeps a branch's observation across +// a repoint, and a missing parent in the old repository says nothing about the new one. +func TestParentBranchReadiness_IgnoresAnotherRepository(t *testing.T) { + r, workers := startParentBranchWorkers(t) + target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) + _, err := r.ensureEventStream(context.Background(), target, "shop", firstRepo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + worker.RecordMissingParentForTest("release") + require.Equal(t, metav1.ConditionFalse, r.parentBranchReadiness(target, "shop", firstRepo).Status) + + _, err = r.ensureEventStream(context.Background(), target, "shop", secondRepo, logr.Discard()) + require.NoError(t, err) + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", secondRepo).Status, + "the old repository's missing parent is not evidence about the replacement") +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 7837a29a..eb313e89 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -242,6 +242,10 @@ type BranchWorker struct { // does not carry is created from. Empty means the remote's default branch. It is set by the // reconcile and read by the event loop, hence atomic. parentBranchName atomic.Pointer[string] + // parentChangedState is raised by SetParentBranch and consumed by the event loop: writes it + // retains were planned against the old parent, and invalidating base trust alone does not + // reach them (ensureBaseForCycle ignores trust while writes are retained). + parentChangedState atomic.Bool // firsts surfaces the first successful commit and push at default verbosity. firsts branchWorkerLogFirsts @@ -1407,6 +1411,9 @@ func (l *branchWorkerEventLoop) resetCommitTimer(delay time.Duration) { // The case only arises with retained work. With nothing retained, commitPendingWrites' own guard // resets for us, and it can do that safely because there is nothing to lose. func (l *branchWorkerEventLoop) recoverRetainedWrites() error { + // Taken first, so a change that found nothing retained does not linger: with nothing retained + // the invalidated base makes the next cycle fetch the new parent anyway. + parentChanged := l.w.parentChangedState.Swap(false) if len(l.pendingWrites) == 0 { // Nothing is retained, so there is nothing a stale replay could strand. A replay only // ever marks itself required while writes are retained, but clear it here too so the flag @@ -1415,14 +1422,15 @@ func (l *branchWorkerEventLoop) recoverRetainedWrites() error { return nil } dirty, needsReplay := l.w.worktreeDirty(), l.w.replayRequired() - if !dirty && !needsReplay { + if !dirty && !needsReplay && !parentChanged { return nil } l.w.Log.Info("Rebuilding retained writes onto the remote tip", "pendingWrites", len(l.pendingWrites), "worktreeDirty", dirty, - "replayRequired", needsReplay) + "replayRequired", needsReplay, + "parentBranchChanged", parentChanged) // fetchReasonRecovery, the same series the no-retained-writes case records in // ensureBaseForCycle: this is one event, and which half of it an operator sees must not depend // on whether a push happened to be in cooldown at the time. @@ -2367,6 +2375,7 @@ func (w *BranchWorker) SetParentBranch(name string) bool { } w.parentBranchName.Store(&name) w.invalidateBase("parent branch changed") + w.parentChangedState.Store(true) return true } diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index b398fc8f..a7f60f26 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -223,3 +223,37 @@ func TestBranchWorker_SetParentBranchInvalidatesTheBase(t *testing.T) { assert.Equal(t, "release", f.worker.ParentBranch()) assert.False(t, f.worker.baseTrusted(), "the checkout holds the old parent") } + +// TestBranchWorker_AParentChangeRebuildsRetainedWrites: writes retained during the push cooldown +// were planned against the old parent, and an invalidated base does not reach them. The change is +// taken by the event loop, which refetches and replays before the next commit or push, so the +// branch starts on the new parent, or is not created at all when the new parent is missing. +func TestBranchWorker_AParentChangeRebuildsRetainedWrites(t *testing.T) { + for _, parent := range []string{"release", "missing"} { + t.Run(parent, func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + require.True(t, loop.pendingWrites[0].CommitSHA.IsZero(), "a no-op is retained during the cooldown") + + require.True(t, f.worker.SetParentBranch(parent)) + liveWrite(loop, "cm2") + loop.pushPending() + + tip, exists := f.featureOnRemote() + if parent == "missing" { + assert.False(t, exists, "a missing configured parent prevents branch creation") + assert.NotEmpty(t, loop.pendingWrites, "and the retained work is kept") + return + } + require.True(t, exists) + assert.Equal(t, release, f.commitParent(tip), "the first push uses the configured parent") + }) + } +} From b9482275bce747ae622bb2aae4a357c8d6e8618f Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 08:36:37 +0000 Subject: [PATCH 05/18] feat(api): make spec.parentBranch immutable, and report status.remote.parent spec.parentBranch decides what history a new write branch gets, so it is fixed at creation like branch and path, including adding or removing it. Delete and recreate loses nothing: the destination is unchanged. The worker keeps its parent-change handling for the narrow case where every target on a branch is replaced before the worker sweep. While the write branch is absent, status.remote.parent names the parent it would be created from and the commit it is at, observed together with the branch's absence by every fetch, refresh and push. With spec.parentBranch omitted, it shows which default branch the remote resolved; an empty commit means the parent is missing too. Co-Authored-By: Claude Opus 5.5 --- api/v1alpha3/gittarget_types.go | 28 ++++++++- api/v1alpha3/zz_generated.deepcopy.go | 20 ++++++ .../crd/bases/configbutler.ai_gittargets.yaml | 25 ++++++++ docs/architecture.md | 9 +-- docs/configuration.md | 15 ++++- docs/definitions.md | 4 +- hack/crdfields/fields.yaml | 2 +- .../controller/gittarget_immutability_test.go | 45 ++++++++----- .../gittarget_parent_branch_test.go | 21 +++++++ internal/controller/gittarget_remote.go | 22 ++++++- internal/git/branch_worker.go | 53 ++++++++++++---- .../git/branch_worker_parent_branch_test.go | 63 +++++++++++++++++++ internal/git/git.go | 6 +- internal/git/refresh.go | 33 ++++++++-- internal/git/refresh_test.go | 14 +++++ internal/git/remote_observation.go | 5 ++ internal/git/types.go | 3 + internal/git/worker_manager.go | 7 ++- test/e2e/new_branch_parent_e2e_test.go | 10 ++- 19 files changed, 331 insertions(+), 54 deletions(-) diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 3b12f0d6..9c7c0a33 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -28,6 +28,7 @@ import ( // +kubebuilder:validation:XValidation:rule="!has(oldSelf.gitProviderRef) || self.gitProviderRef == oldSelf.gitProviderRef",message="spec.gitProviderRef is immutable; delete and recreate the GitTarget to change its destination" // +kubebuilder:validation:XValidation:rule="self.branch == oldSelf.branch",message="spec.branch is immutable; delete and recreate the GitTarget to change its destination" // +kubebuilder:validation:XValidation:rule="self.path == oldSelf.path",message="spec.path is immutable; delete and recreate the GitTarget to change its destination" +// +kubebuilder:validation:XValidation:rule="has(self.parentBranch) == has(oldSelf.parentBranch) && (!has(self.parentBranch) || self.parentBranch == oldSelf.parentBranch)",message="spec.parentBranch is immutable; delete and recreate the GitTarget to change it" // // spec.clusterProviderRef names the SOURCE cluster a GitTarget mirrors FROM (see its field doc). It // is immutable — a folder's source cluster is part of what the folder means, like @@ -50,9 +51,10 @@ type GitTargetSpec struct { Branch string `json:"branch"` // Why "parent" and not base/source/revision: docs/definitions.md reserves those words, and the - // field is flat because it is one fact. It is mutable because it only says where a branch - // starts; changing it abandons nothing already written, unlike branch and path. It can never - // hold a commit or tag: a pinned starting point would be a sibling field. + // field is flat because it is one fact. It is immutable like branch and path: it decides what + // history a new write branch gets, and a parent changed under a worker holding writes is the + // one case that needed its own replay. Delete and recreate is cheap, because the destination is + // unchanged. It can never hold a commit or tag: a pinned starting point would be a sibling field. // ParentBranch is the branch the write branch (spec.branch) is created from, and the branch the // target compares against while the write branch does not exist on the remote. Omitted: the @@ -62,6 +64,7 @@ type GitTargetSpec struct { // write branch exists, it is used as it is and the parent is not followed. It is only read, so // it does not need to be in the GitProvider's allowedBranches. All GitTargets on one GitProvider // and branch must name the same parent branch (omitted counts as its own value). + // Immutable, including adding or removing it: delete and recreate the GitTarget to change it. // +optional // +kubebuilder:validation:MinLength=1 ParentBranch string `json:"parentBranch,omitempty"` @@ -474,6 +477,25 @@ type GitTargetRemoteStatus struct { // +optional // +kubebuilder:validation:Enum=Push;Fetch VerifiedBy string `json:"verifiedBy,omitempty"` + + // Parent is what the write branch would be created from, observed with it: present only + // while the remote does not carry the write branch. Once the branch exists its parent is not + // followed, so this is omitted. + // +optional + Parent *GitTargetParentStatus `json:"parent,omitempty"` +} + +// GitTargetParentStatus is the parent branch of a write branch that does not exist yet. +type GitTargetParentStatus struct { + // Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when + // that is omitted. + Branch string `json:"branch"` + + // Commit is the commit hash the parent branch is at on the remote: the commit the write + // branch's first commit would build on. EMPTY means the parent branch is not on the remote; + // the GitTarget then reports ParentBranchNotFound. + // +optional + Commit string `json:"commit,omitempty"` } // Two rules for anything added here. A field earns its place only if a reader cannot get it from diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index bc96bda8..890184c4 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -831,6 +831,21 @@ func (in *GitTargetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitTargetParentStatus) DeepCopyInto(out *GitTargetParentStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetParentStatus. +func (in *GitTargetParentStatus) DeepCopy() *GitTargetParentStatus { + if in == nil { + return nil + } + out := new(GitTargetParentStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitTargetPlacementSpec) DeepCopyInto(out *GitTargetPlacementSpec) { *out = *in @@ -884,6 +899,11 @@ func (in *GitTargetRemoteStatus) DeepCopyInto(out *GitTargetRemoteStatus) { in, out := &in.LastVerifiedAt, &out.LastVerifiedAt *out = (*in).DeepCopy() } + if in.Parent != nil { + in, out := &in.Parent, &out.Parent + *out = new(GitTargetParentStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetRemoteStatus. diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 38729338..0d6585e8 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -358,6 +358,7 @@ spec: write branch exists, it is used as it is and the parent is not followed. It is only read, so it does not need to be in the GitProvider's allowedBranches. All GitTargets on one GitProvider and branch must name the same parent branch (omitted counts as its own value). + Immutable, including adding or removing it: delete and recreate the GitTarget to change it. minLength: 1 type: string path: @@ -533,6 +534,10 @@ spec: - message: spec.path is immutable; delete and recreate the GitTarget to change its destination rule: self.path == oldSelf.path + - message: spec.parentBranch is immutable; delete and recreate the GitTarget + to change it + rule: has(self.parentBranch) == has(oldSelf.parentBranch) && (!has(self.parentBranch) + || self.parentBranch == oldSelf.parentBranch) - message: spec.clusterProviderRef is immutable; delete and recreate the GitTarget to change the cluster it mirrors rule: self.clusterProviderRef == oldSelf.clusterProviderRef @@ -694,6 +699,26 @@ spec: an old value there means the layout has not changed rather than that nothing has looked. format: date-time type: string + parent: + description: |- + Parent is what the write branch would be created from, observed with it: present only + while the remote does not carry the write branch. Once the branch exists its parent is not + followed, so this is omitted. + properties: + branch: + description: |- + Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when + that is omitted. + type: string + commit: + description: |- + Commit is the commit hash the parent branch is at on the remote: the commit the write + branch's first commit would build on. EMPTY means the parent branch is not on the remote; + the GitTarget then reports ParentBranchNotFound. + type: string + required: + - branch + type: object verifiedBy: description: |- VerifiedBy is what proved it: `Push` means the server accepted a ref update of ours, so diff --git a/docs/architecture.md b/docs/architecture.md index e7a8fb8f..023a4fa2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -359,7 +359,7 @@ Key fields: - `spec.gitProviderRef`: a `GitProvider` in the same namespace, by name. - `spec.clusterProviderRef`: a cluster-scoped source `ClusterProvider`; it defaults to `{name: default}`. - `spec.branch`: immutable branch, validated against `GitProvider.spec.allowedBranches`. -- `spec.parentBranch`: optional, mutable branch the write branch is created from and compared +- `spec.parentBranch`: optional, immutable branch the write branch is created from and compared against while it does not exist. Omitted means the remote's default branch. An explicit parent must exist when creating the write branch. It does not need to be in `allowedBranches` because it is only read. @@ -375,9 +375,10 @@ may batch and phrase their commits differently. A branch worker serves a `(provi resolves the window per open window, since a window is bound to exactly one target. `gitProviderRef`, `clusterProviderRef`, `branch`, and `path` are immutable so a target cannot silently -orphan an old materialization or change its source cluster. The controller also rejects path overlaps -between GitTargets sharing a provider and branch, and rejects two GitTargets on one branch that name -different parent branches. +orphan an old materialization or change its source cluster. `parentBranch` is immutable too, +including adding or removing it, because it decides what history a new write branch gets. The +controller also rejects path overlaps between GitTargets sharing a provider and branch, and rejects +two GitTargets on one branch that name different parent branches. Status has a kstatus-compatible summary layer plus domain conditions: diff --git a/docs/configuration.md b/docs/configuration.md index 6a43ff29..773371d8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -79,7 +79,7 @@ The destination fields are immutable: to move a target, delete it and create a n |---|---|---| | `gitProviderRef` | **required** | The `GitProvider` backing this target, in the same namespace | | `branch` | **required** | Branch to write. Must be in the provider's `allowedBranches` | -| `parentBranch` | the remote's default branch | Branch a `branch` the remote does not have yet is created from. Mutable, and only read, so it need not be allowed. See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) | +| `parentBranch` | the remote's default branch | Branch a `branch` the remote does not have yet is created from. Immutable. Only read, so it need not be allowed. See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) | | `path` | **required** | Folder within the repository. `.` targets the root; empty is rejected | | `clusterProviderRef` | `{"name":"default"}` | Source cluster to mirror from. The default names a `ClusterProvider` called `default` | | `commit.window.idleTimeout` | `5s` | Close a commit window after this much silence. `0s` commits every write on its own. See [the commit window](#the-commit-window-speccommitwindow) | @@ -774,7 +774,12 @@ spec: starts a branch with no history. - Once the write branch exists, it is used as it is: new commits on the parent do not reach it. Delete the write branch after its changes are merged, and the next edit starts a fresh one. -- `parentBranch` is mutable, and it is only read, so it does not need to be in `allowedBranches`. +- `parentBranch` is immutable, including adding or removing it: delete and recreate the target to + change it, which loses nothing because the destination is unchanged. It is only read, so it does + not need to be in `allowedBranches`. +- While the write branch is absent, `status.remote.parent` shows the parent and the commit the + first commit would build on. With `parentBranch` omitted, that is how you see which default branch + the remote resolved. - All `GitTarget` objects on one `GitProvider` and branch must name the same parent branch; omitted counts as its own value. The later-created one reports `TargetConflict`. @@ -893,6 +898,9 @@ status: commit: 4f2c1ab9e0... # empty = the branch is not on the remote lastVerifiedAt: "2026-09-23T10:14:02Z" verifiedBy: Push # Push | Fetch + parent: # only while the branch is not on the remote + branch: main # spec.parentBranch, or the remote's default branch + commit: 9a1e07c3d2... # empty = the parent is not on the remote either ``` - `commit` is where the branch is. Empty means the branch is not on the remote at all, which is @@ -904,6 +912,9 @@ status: commit is Reverser's own work. `Fetch` means it went and looked, and this is what was there. A `Fetch` beside a commit none of your publications produced is how a **foreign push** to the branch is read off `kubectl`. +- `parent` appears only while the branch is not on the remote: it names what the branch would be + created from and where that was, observed together with the branch's absence. See + [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch). `kubectl get gittarget -o wide` shows `lastVerifiedAt` as an age, in the `Verified` column. diff --git a/docs/definitions.md b/docs/definitions.md index 02dd6726..0df60e75 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -147,8 +147,8 @@ one. One word for the act, in the API (`window.attach`, `attachTimeout`), the co `status.remote`. **Commit.** A bare Git commit hash, wherever one appears in the API: `status.remote.commit`, -`CommitRequest.status.commit`, `status.placement.resolvedAtCommit`. Not "sha", which names an -algorithm Git is in the middle of changing, and not "revision", which Flux uses for a polymorphic +`status.remote.parent.commit`, `CommitRequest.status.commit`, `status.placement.resolvedAtCommit`. +Not "sha", which names an algorithm Git is in the middle of changing, and not "revision", which Flux uses for a polymorphic identifier that may be a tag, a chart version or a composite like `main@sha1:`. If this project ever needs that composite, `revision` is the word waiting for it. diff --git a/hack/crdfields/fields.yaml b/hack/crdfields/fields.yaml index 15c90547..cdad4efd 100644 --- a/hack/crdfields/fields.yaml +++ b/hack/crdfields/fields.yaml @@ -91,7 +91,7 @@ kinds: - path: parentBranch default: the remote's default branch doc: >- - Branch a `branch` the remote does not have yet is created from. Mutable, and only read, so + Branch a `branch` the remote does not have yet is created from. Immutable. Only read, so it need not be allowed. See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) - path: path diff --git a/internal/controller/gittarget_immutability_test.go b/internal/controller/gittarget_immutability_test.go index 2ab6f899..5a19011b 100644 --- a/internal/controller/gittarget_immutability_test.go +++ b/internal/controller/gittarget_immutability_test.go @@ -76,32 +76,43 @@ var _ = Describe("GitTarget Destination Immutability", func() { }, "spec.gitProviderRef is immutable") }) - // spec.parentBranch is NOT part of the destination: it says where a write branch starts, so it - // can change on a live target. It can be omitted, but not set to the empty string. - It("allows spec.parentBranch to be set, changed and cleared, but never empty", func() { + // spec.parentBranch decides what history a new write branch gets, so it is fixed at creation + // like the destination, including adding or removing it. It can be omitted, but never empty. + It("rejects changing, adding or removing spec.parentBranch, and an empty one", func() { ctx := context.Background() - key := types.NamespacedName{Name: "parent-branch-target", Namespace: "default"} - - gitTarget := &configbutleraiv1alpha3.GitTarget{ - ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, - Spec: configbutleraiv1alpha3.GitTargetSpec{ - GitProviderRef: meta.LocalObjectReference{Name: "prov-a"}, - Branch: "edits", - Path: "apps", - ParentBranch: "main", - }, + withParent := types.NamespacedName{Name: "parent-branch-target", Namespace: "default"} + withoutParent := types.NamespacedName{Name: "no-parent-branch-target", Namespace: "default"} + + for _, tc := range []struct { + key types.NamespacedName + parent string + }{{withParent, "main"}, {withoutParent, ""}} { + gitTarget := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: tc.key.Name, Namespace: tc.key.Namespace}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "prov-a"}, + Branch: "edits", + Path: "apps-" + tc.key.Name, + ParentBranch: tc.parent, + }, + } + Expect(k8sClient.Create(ctx, gitTarget)).Should(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, gitTarget) }) } - Expect(k8sClient.Create(ctx, gitTarget)).Should(Succeed()) - DeferCleanup(func() { _ = k8sClient.Delete(ctx, gitTarget) }) - for _, parent := range []string{"release", ""} { + expectRejected := func(key types.NamespacedName, parent string) { Eventually(func(g Gomega) { current := &configbutleraiv1alpha3.GitTarget{} g.Expect(k8sClient.Get(ctx, key, current)).To(Succeed()) current.Spec.ParentBranch = parent - g.Expect(k8sClient.Update(ctx, current)).To(Succeed()) + err := k8sClient.Update(ctx, current) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("spec.parentBranch is immutable")) }, timeout, interval).Should(Succeed()) } + expectRejected(withParent, "release") // changed + expectRejected(withParent, "") // removed + expectRejected(withoutParent, "release") // added empty := &unstructured.Unstructured{Object: map[string]interface{}{ "apiVersion": "configbutler.ai/v1alpha3", diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index 85fa854a..0d747085 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -224,3 +224,24 @@ func TestParentBranchReadiness_IgnoresAnotherRepository(t *testing.T) { assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", secondRepo).Status, "the old repository's missing parent is not evidence about the replacement") } + +// TestParentStatusOf: status.remote.parent is present only while the write branch is absent and +// a parent was named, and a moved parent is news even when the write branch is still absent. +func TestParentStatusOf(t *testing.T) { + absent := git.RemoteObservation{ParentBranch: "main", ParentCommit: "aaaa"} + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{Branch: "main", Commit: "aaaa"}, + parentStatusOf(absent)) + assert.Nil(t, parentStatusOf(git.RemoteObservation{Commit: "bbbb", ParentBranch: "main"}), + "an existing write branch does not follow its parent") + assert.Nil(t, parentStatusOf(git.RemoteObservation{}), "an empty repository has no parent") + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{Branch: "release"}, + parentStatusOf(git.RemoteObservation{ParentBranch: "release", MissingParent: "release"}), + "a missing parent is named with no commit") + + at := metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + published := &configbutleraiv1alpha3.GitTargetRemoteStatus{LastVerifiedAt: &at, Parent: parentStatusOf(absent)} + moved := published.DeepCopy() + moved.Parent.Commit = "cccc" + assert.True(t, remoteStatusIsNews(published, moved), "the parent moved") + assert.False(t, remoteStatusIsNews(published, published.DeepCopy())) +} diff --git a/internal/controller/gittarget_remote.go b/internal/controller/gittarget_remote.go index 397f4d8b..c3ec79d9 100644 --- a/internal/controller/gittarget_remote.go +++ b/internal/controller/gittarget_remote.go @@ -148,6 +148,7 @@ func (r *GitTargetReconciler) publishRemote( Commit: observed.Commit, LastVerifiedAt: &metav1.Time{Time: observed.At}, VerifiedBy: string(observed.By), + Parent: parentStatusOf(observed), } if !remoteStatusIsNews(target.Status.Remote, next) { return @@ -203,10 +204,29 @@ func remoteStatusIsNews(published, next *configbutleraiv1alpha3.GitTargetRemoteS if published == nil || published.LastVerifiedAt == nil || next.LastVerifiedAt == nil { return true } - if published.Commit != next.Commit { + if published.Commit != next.Commit || !equalParentStatus(published.Parent, next.Parent) { return true } // metav1.Time is second-granular on the wire, so this compares against what was PUBLISHED: a // sub-second difference the API server would round away is not movement. return next.LastVerifiedAt.Time.After(published.LastVerifiedAt.Time) } + +// parentStatusOf is the parent stanza of status.remote: present only while the observation found +// the write branch absent and named a parent to create it from. +func parentStatusOf(observed git.RemoteObservation) *configbutleraiv1alpha3.GitTargetParentStatus { + if observed.Commit != "" || observed.ParentBranch == "" { + return nil + } + return &configbutleraiv1alpha3.GitTargetParentStatus{ + Branch: observed.ParentBranch, + Commit: observed.ParentCommit, + } +} + +func equalParentStatus(a, b *configbutleraiv1alpha3.GitTargetParentStatus) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index eb313e89..f164aff3 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -2013,13 +2013,7 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { w.setBaseTrusted(true) // The half a fetch-only record could not carry: on an active branch the push is the // event that MOVES the revision. - revision := "" - if !outcome.Head.IsZero() { - revision = outcome.Head.String() - } - w.recordRemoteObservation(revision, ObservedByPush) - w.Log.V(1).Info("Remote observed by push", - "branch", w.Branch, "outcome", string(outcome.Kind), "head", revision) + w.recordPushObservation(outcome, rootBranch, rootHash) w.endPushCycle(outcome.Kind) w.firsts.push.Do(func() { w.Log.Info("First push to remote completed", @@ -2348,6 +2342,25 @@ func (w *BranchWorker) ensureWriteBranch(repo *gogit.Repository) (plumbing.Refer return baseBranch, baseHash, nil } +// recordPushObservation records what a push that returned without an error observed. +func (w *BranchWorker) recordPushObservation( + outcome PushOutcome, rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, +) { + revision := "" + if !outcome.Head.IsZero() { + revision = outcome.Head.String() + } + if outcome.Kind == PushNoBranch && rootBranch != plumbing.NewBranchReferenceName(w.Branch) { + // Settled without creating the branch, after confirming its parent against this + // advertisement: the root IS the parent's tip. + w.recordAbsentBranch(rootBranch.Short(), rootHash.String(), ObservedByPush) + } else { + w.recordRemoteObservation(revision, ObservedByPush) + } + w.Log.V(1).Info("Remote observed by push", + "branch", w.Branch, "outcome", string(outcome.Kind), "head", revision) +} + // endPushCycle clears the cycle's root after a push the remote answered. Only a push that left the // branch absent keeps the parent it would be created from. func (w *BranchWorker) endPushCycle(kind PushOutcomeKind) { @@ -2367,8 +2380,9 @@ func (w *BranchWorker) ParentBranch() string { } // SetParentBranch configures the parent branch. A change invalidates the base, so the next cycle -// fetches and compares against the new parent; the worker and its clone are kept, because -// nothing already written is abandoned. It reports whether the value changed. +// fetches and compares against the new parent, and the event loop rebuilds retained writes on it; +// the worker and its clone are kept. spec.parentBranch is immutable, so a change reaches a live +// worker only when the targets on its branch were replaced. It reports whether the value changed. func (w *BranchWorker) SetParentBranch(name string) bool { if w.ParentBranch() == name { return false @@ -2776,7 +2790,20 @@ func (w *BranchWorker) recordRemoteObservation(revision string, by ObservationSo // recordMissingParent records that the write branch is absent and its configured parent is too. func (w *BranchWorker) recordMissingParent(parent string, by ObservationSource) { - w.publishObservation(RemoteObservation{At: time.Now(), By: by, Repo: w.repo, MissingParent: parent}) + w.publishObservation(RemoteObservation{ + At: time.Now(), By: by, Repo: w.repo, MissingParent: parent, ParentBranch: parent, + }) +} + +// recordAbsentBranch records that the write branch is absent, and where the parent it would be +// created from is. An empty parent is an empty repository, which has none. +func (w *BranchWorker) recordAbsentBranch(parentBranch, parentCommit string, by ObservationSource) { + if parentBranch == "" { + parentCommit = "" + } + w.publishObservation(RemoteObservation{ + At: time.Now(), By: by, Repo: w.repo, ParentBranch: parentBranch, ParentCommit: parentCommit, + }) } func (w *BranchWorker) publishObservation(observed RemoteObservation) { @@ -2881,11 +2908,11 @@ func (w *BranchWorker) shouldReadRetainedLocalRepository(repoPath string) bool { func (w *BranchWorker) updateBranchMetadataFromPullReport(report *PullReport) { // A branch the remote does not have is recorded as an observation with no revision, which is // the honest answer rather than a missing one: a git branch without a commit does not exist. - revision := "" if report.ExistsOnRemote { - revision = report.HEAD.Sha + w.recordRemoteObservation(report.HEAD.Sha, ObservedByFetch) + } else { + w.recordAbsentBranch(report.ParentBranch, report.HEAD.Sha, ObservedByFetch) } - w.recordRemoteObservation(revision, ObservedByFetch) // This is the ONE place trust can be gained by a fetch, because it is the one place a fetch // is known to have been followed by a reset: every call site is a PrepareBranch or a diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index a7f60f26..d7c840ec 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -257,3 +257,66 @@ func TestBranchWorker_AParentChangeRebuildsRetainedWrites(t *testing.T) { }) } } + +// TestBranchWorker_ObservesTheParentOfAnAbsentBranch: every way the worker looks at the remote +// records, alongside the write branch's absence, the parent it would be created from, which +// status.remote.parent publishes. With spec.parentBranch omitted that names the remote's default +// branch. Once the write branch exists, there is no parent to report. +func TestBranchWorker_ObservesTheParentOfAnAbsentBranch(t *testing.T) { + observed := func(f *newBranchFixture) RemoteObservation { + t.Helper() + o, known := f.worker.LastRemoteObservation() + require.True(t, known) + return o + } + + t.Run("a fetch names the default branch when the parent is omitted", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + hashA := f.warmOnParent() + o := observed(f) + assert.Equal(t, ObservedByFetch, o.By) + assert.Empty(t, o.Commit) + assert.Equal(t, "main", o.ParentBranch) + assert.Equal(t, hashA.String(), o.ParentCommit) + }) + + t.Run("a push that leaves the branch absent confirms the parent", func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + hashA := f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + o := observed(f) + assert.Equal(t, ObservedByPush, o.By) + assert.Equal(t, "main", o.ParentBranch) + assert.Equal(t, hashA.String(), o.ParentCommit) + + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + o = observed(f) + assert.NotEmpty(t, o.Commit, "the branch exists now") + assert.Empty(t, o.ParentBranch, "and its parent is no longer followed") + }) + + t.Run("a configured parent, present and missing", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + o := observed(f) + assert.Equal(t, "release", o.ParentBranch) + assert.Empty(t, o.ParentCommit, "the parent is not on the remote") + assert.Equal(t, "release", o.MissingParent) + + r1 := f.pushToRelease("RELEASE.md", "release\n") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + o = observed(f) + assert.Equal(t, "release", o.ParentBranch) + assert.Equal(t, r1.String(), o.ParentCommit) + assert.Empty(t, o.MissingParent) + }) +} diff --git a/internal/git/git.go b/internal/git/git.go index d46ca9af..0599bb26 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -406,7 +406,11 @@ func syncToRemoteFrom( } remoteExists := availableBranch.Short() == branch.Short() - return createPullReport(branch.Short(), currentHash, newHash, remoteExists, false), nil + report := createPullReport(branch.Short(), currentHash, newHash, remoteExists, false) + if !remoteExists { + report.ParentBranch = availableBranch.Short() + } + return report, nil } // Failed to fetch from both sources, so let's configure head to be unborn at targetbranch. diff --git a/internal/git/refresh.go b/internal/git/refresh.go index bccb940c..7ce7b293 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -197,17 +197,15 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr return fmt.Errorf("read the remote advertisement: %w", err) } advertised := advertisement.branch - // Neither the branch nor the parent it would be created from: there is nothing to follow, - // and the GitTarget reports the parent as missing until somebody creates it. - if advertised.IsZero() && parentBranch != "" && advertisement.parent.IsZero() { - w.recordMissingParent(parentBranch, ObservedByFetch) + if missingParent := w.recordAdvertisement(advertisement, parentBranch); missingParent { + // Neither the branch nor the parent it would be created from: there is nothing to follow, + // and the GitTarget reports the parent as missing until somebody creates it. return nil } revision := "" if !advertised.IsZero() { revision = advertised.String() } - w.recordRemoteObservation(revision, ObservedByFetch) // The remote does not carry this branch — the ordinary state of a target that has not written // yet. That IS the observation. What the checkout holds instead is the parent the branch would @@ -251,6 +249,25 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr return nil } +// recordAdvertisement records what one advertisement said about the write branch and, while it +// is absent, its parent. It reports a configured parent that is missing as well. +func (w *BranchWorker) recordAdvertisement(advertisement remoteAdvertisement, parentBranch string) bool { + if !advertisement.branch.IsZero() { + w.recordRemoteObservation(advertisement.branch.String(), ObservedByFetch) + return false + } + if advertisement.parent.IsZero() { + if parentBranch != "" { + w.recordMissingParent(parentBranch, ObservedByFetch) + return true + } + w.recordAbsentBranch("", "", ObservedByFetch) // an empty repository: no parent at all + return false + } + w.recordAbsentBranch(advertisement.parentBranch, advertisement.parent.String(), ObservedByFetch) + return false +} + // followParent brings the checkout of an absent write branch onto its parent's advertised tip. // It costs nothing beyond the advertisement already read when the parent has not moved, and it // writes nothing either way. @@ -358,6 +375,8 @@ type remoteAdvertisement struct { // branch, or the remote's default branch when none is configured. Zero when the remote does // not carry it, which includes an empty repository. parent plumbing.Hash + // parentBranch is that parent's short name; empty when the remote's HEAD names no branch. + parentBranch string } // advertiseRemoteBranch asks the remote where a branch and its parent are, and transfers nothing @@ -390,7 +409,9 @@ func advertiseRemoteBranch( if parentBranch != "" { defaultBranch = plumbing.NewBranchReferenceName(parentBranch) } - return remoteAdvertisement{branch: hashes[branch], parent: hashes[defaultBranch]}, nil + return remoteAdvertisement{ + branch: hashes[branch], parent: hashes[defaultBranch], parentBranch: defaultBranch.Short(), + }, nil } // localHead is the commit the worktree is on, or the zero hash when the branch is unborn. diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index b70f7a08..d6186a80 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -683,3 +683,17 @@ func TestRefresh_ReportsAMissingConfiguredParent(t *testing.T) { assert.Equal(t, "release", last.MissingParent) assert.Empty(t, last.Commit) } + +// TestRefresh_ObservesTheParentOfAnAbsentBranch: the advertisement that answers for the write +// branch also names its parent, so the refresher reports both at one connection. +func TestRefresh_ObservesTheParentOfAnAbsentBranch(t *testing.T) { + h := newRefreshHarnessForBranch(t, "refresh-parent-observed", true, "feature") + h.reported = nil + + assert.Equal(t, int64(1), h.refresh(time.Nanosecond)) + require.NotEmpty(t, h.reported) + last := h.reported[len(h.reported)-1] + assert.Empty(t, last.Commit) + assert.Equal(t, "main", last.ParentBranch, "an omitted parent is the branch the remote's HEAD names") + assert.Equal(t, revParseMain(t, h.repoDir), last.ParentCommit) +} diff --git a/internal/git/remote_observation.go b/internal/git/remote_observation.go index 26b82b88..fc4d888f 100644 --- a/internal/git/remote_observation.go +++ b/internal/git/remote_observation.go @@ -53,6 +53,11 @@ type RemoteObservation struct { // MissingParent names the configured parent branch when the remote carries neither it nor the // branch, so the branch cannot be created. Empty otherwise, including for an omitted parent. MissingParent string + // ParentBranch and ParentCommit are the parent an absent branch would be created from, and + // where it was, observed together with the branch's absence. Both are empty while the branch + // exists, and for an empty repository. ParentCommit is empty when the parent is missing. + ParentBranch string + ParentCommit string } // Age is how long ago the observation was made, against now. diff --git a/internal/git/types.go b/internal/git/types.go index ca4444a5..fb5e9146 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -142,6 +142,9 @@ type PullReport struct { ExistsOnRemote bool // Branch exists on remote HEAD BranchInfo IncomingChanges bool // SHA changed, requiring resource-level reconcile + // ParentBranch is the branch checked out instead, when the target branch is absent: the parent + // it would be created from. HEAD.Sha is then the parent's commit. Empty otherwise. + ParentBranch string } // BranchKey uniquely identifies a (GitProvider, Branch) combination. diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 2b149fd1..bbbed976 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -283,9 +283,10 @@ func (m *WorkerManager) RemoteForBranch(key BranchKey) (RemoteObservation, bool) // and the caller can fail. See ReplacementPending; the delivery is the GitTarget reconcile's // worker wiring gate. // -// parentBranch is the GitTarget's spec.parentBranch. A change is not a replacement: it says where -// a write branch the remote does not carry is created from, so it abandons nothing already -// written. The worker keeps its clone and drops its base trust instead. +// parentBranch is the GitTarget's spec.parentBranch. It is immutable on a GitTarget, so a live +// worker sees a different one only when every target on its branch was replaced before the worker +// sweep ran. That is not a replacement: the worker keeps its clone, and SetParentBranch drops its +// base trust and rebuilds any retained writes on the new parent. // // Worker creation/start is protected by the manager lock. func (m *WorkerManager) EnsureWorker( diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index f335393a..2c9ff362 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -145,7 +145,7 @@ spec: "README.md": "main is not the parent here\n", }) } - commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: seed A", map[string]string{ + hashA := commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: seed A", map[string]string{ "README.md": "seed\n", path.Join(gitPath, testNs, "configmaps", "standby-present.yaml"): mirroredDocument("standby-present"), }) @@ -178,6 +178,14 @@ spec: func(v float64) bool { return v > 0 }, "the worker for the write branch checked out the parent", 2*time.Minute) + By("status.remote names the parent the write branch would be created from") + Eventually(func(g Gomega) { + g.Expect(gitTargetRemoteField(g, destName, testNs, "commit")).To(BeEmpty(), "the write branch is absent") + g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.branch")).To(Equal(parent), + "with spec.parentBranch omitted this is the default branch the remote resolved") + g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.commit")).To(Equal(hashA)) + }, 3*time.Minute, 5*time.Second).Should(Succeed()) + By("an idle, in-sync target creates no branch") Consistently(func(g Gomega) { g.Expect(remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch)).To(BeEmpty(), From bfbb7d979a4e66d948782e74da4e73acc985d895 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 08:47:38 +0000 Subject: [PATCH 06/18] feat(api): report whether the parent was found, missing or unborn status.remote.parent.state is Found (the first commit builds on the parent's tip), Missing (spec.parentBranch is not on the remote either, so nothing is written) or Unborn (an empty repository and no parent configured, so the first commit has no history). Without it, a blocked target and a fresh repository both showed an empty commit, or no parent at all. The worker's observation carries the same state instead of a MissingParent name, so the readiness gate and the status read one fact. Co-Authored-By: Claude Opus 5.5 --- api/v1alpha3/gittarget_types.go | 29 ++++++++++++++++--- .../crd/bases/configbutler.ai_gittargets.yaml | 17 ++++++++--- docs/configuration.md | 16 ++++++---- internal/controller/gittarget_controller.go | 3 +- .../gittarget_parent_branch_test.go | 18 +++++++----- internal/controller/gittarget_remote.go | 5 ++-- internal/git/branch_worker.go | 15 ++++++++-- .../git/branch_worker_parent_branch_test.go | 22 ++++++++++---- internal/git/refresh_test.go | 4 ++- internal/git/remote_observation.go | 25 ++++++++++++---- test/e2e/new_branch_parent_e2e_test.go | 1 + 11 files changed, 116 insertions(+), 39 deletions(-) diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 9c7c0a33..30c180d7 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -485,15 +485,36 @@ type GitTargetRemoteStatus struct { Parent *GitTargetParentStatus `json:"parent,omitempty"` } +// GitTargetParentState is what a write branch that does not exist yet would be created from. +// +kubebuilder:validation:Enum=Found;Missing;Unborn +type GitTargetParentState string + +const ( + // GitTargetParentFound means the parent branch exists; the first commit builds on its tip. + GitTargetParentFound GitTargetParentState = "Found" + // GitTargetParentMissing means spec.parentBranch is not on the remote either, so the write branch + // cannot be created; the GitTarget reports ParentBranchNotFound. + GitTargetParentMissing GitTargetParentState = "Missing" + // GitTargetParentUnborn means the repository is empty and no parent is configured, so the first + // commit starts the write branch with no history. + GitTargetParentUnborn GitTargetParentState = "Unborn" +) + // GitTargetParentStatus is the parent branch of a write branch that does not exist yet. type GitTargetParentStatus struct { + // State is what the write branch would be created from: Found (the parent's tip), Missing + // (spec.parentBranch is not on the remote; nothing is written) or Unborn (an empty repository + // and no parent configured; the first commit has no history). + // +required + State GitTargetParentState `json:"state"` + // Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when - // that is omitted. - Branch string `json:"branch"` + // that is omitted. Empty when State is Unborn. + // +optional + Branch string `json:"branch,omitempty"` // Commit is the commit hash the parent branch is at on the remote: the commit the write - // branch's first commit would build on. EMPTY means the parent branch is not on the remote; - // the GitTarget then reports ParentBranchNotFound. + // branch's first commit would build on. Set only when State is Found. // +optional Commit string `json:"commit,omitempty"` } diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 0d6585e8..763abb77 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -708,16 +708,25 @@ spec: branch: description: |- Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when - that is omitted. + that is omitted. Empty when State is Unborn. type: string commit: description: |- Commit is the commit hash the parent branch is at on the remote: the commit the write - branch's first commit would build on. EMPTY means the parent branch is not on the remote; - the GitTarget then reports ParentBranchNotFound. + branch's first commit would build on. Set only when State is Found. + type: string + state: + description: |- + State is what the write branch would be created from: Found (the parent's tip), Missing + (spec.parentBranch is not on the remote; nothing is written) or Unborn (an empty repository + and no parent configured; the first commit has no history). + enum: + - Found + - Missing + - Unborn type: string required: - - branch + - state type: object verifiedBy: description: |- diff --git a/docs/configuration.md b/docs/configuration.md index 773371d8..7e452008 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -777,9 +777,9 @@ spec: - `parentBranch` is immutable, including adding or removing it: delete and recreate the target to change it, which loses nothing because the destination is unchanged. It is only read, so it does not need to be in `allowedBranches`. -- While the write branch is absent, `status.remote.parent` shows the parent and the commit the - first commit would build on. With `parentBranch` omitted, that is how you see which default branch - the remote resolved. +- While the write branch is absent, `status.remote.parent` shows the parent, whether it was + `Found`, `Missing` or `Unborn`, and the commit the first commit would build on. With + `parentBranch` omitted, that is how you see which default branch the remote resolved. - All `GitTarget` objects on one `GitProvider` and branch must name the same parent branch; omitted counts as its own value. The later-created one reports `TargetConflict`. @@ -899,8 +899,9 @@ status: lastVerifiedAt: "2026-09-23T10:14:02Z" verifiedBy: Push # Push | Fetch parent: # only while the branch is not on the remote + state: Found # Found | Missing | Unborn branch: main # spec.parentBranch, or the remote's default branch - commit: 9a1e07c3d2... # empty = the parent is not on the remote either + commit: 9a1e07c3d2... # set only when Found ``` - `commit` is where the branch is. Empty means the branch is not on the remote at all, which is @@ -912,8 +913,11 @@ status: commit is Reverser's own work. `Fetch` means it went and looked, and this is what was there. A `Fetch` beside a commit none of your publications produced is how a **foreign push** to the branch is read off `kubectl`. -- `parent` appears only while the branch is not on the remote: it names what the branch would be - created from and where that was, observed together with the branch's absence. See +- `parent` appears only while the branch is not on the remote: it says what the branch would be + created from, observed together with the branch's absence. `state` is `Found` (the first commit + builds on `commit`, the parent's tip), `Missing` (`spec.parentBranch` is not on the remote + either; the target reports `ParentBranchNotFound` and writes nothing) or `Unborn` (an empty + repository and no parent configured; the first commit starts the branch with no history). See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch). `kubectl get gittarget -o wide` shows `lastVerifiedAt` as an age, in the `Verified` column. diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 0b0760a8..ff177e85 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -830,7 +830,8 @@ func (r *GitTargetReconciler) parentBranchReadiness( return ok } observed, seen := r.observeRemote(target, providerNS) - if !seen || observed.MissingParent != target.Spec.ParentBranch || differentRepository(observed.Repo, repo) { + if !seen || observed.ParentState != git.ParentMissing || observed.ParentBranch != target.Spec.ParentBranch || + differentRepository(observed.Repo, repo) { return ok } return conditionValue{ diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index 0d747085..9b55c16f 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -228,14 +228,18 @@ func TestParentBranchReadiness_IgnoresAnotherRepository(t *testing.T) { // TestParentStatusOf: status.remote.parent is present only while the write branch is absent and // a parent was named, and a moved parent is news even when the write branch is still absent. func TestParentStatusOf(t *testing.T) { - absent := git.RemoteObservation{ParentBranch: "main", ParentCommit: "aaaa"} - assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{Branch: "main", Commit: "aaaa"}, - parentStatusOf(absent)) - assert.Nil(t, parentStatusOf(git.RemoteObservation{Commit: "bbbb", ParentBranch: "main"}), + absent := git.RemoteObservation{ParentState: git.ParentFound, ParentBranch: "main", ParentCommit: "aaaa"} + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{ + State: configbutleraiv1alpha3.GitTargetParentFound, Branch: "main", Commit: "aaaa", + }, parentStatusOf(absent)) + assert.Nil(t, parentStatusOf(git.RemoteObservation{Commit: "bbbb"}), "an existing write branch does not follow its parent") - assert.Nil(t, parentStatusOf(git.RemoteObservation{}), "an empty repository has no parent") - assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{Branch: "release"}, - parentStatusOf(git.RemoteObservation{ParentBranch: "release", MissingParent: "release"}), + assert.Nil(t, parentStatusOf(git.RemoteObservation{}), "an observation that names no parent state") + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{State: configbutleraiv1alpha3.GitTargetParentUnborn}, + parentStatusOf(git.RemoteObservation{ParentState: git.ParentUnborn}), "an empty repository") + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{ + State: configbutleraiv1alpha3.GitTargetParentMissing, Branch: "release", + }, parentStatusOf(git.RemoteObservation{ParentState: git.ParentMissing, ParentBranch: "release"}), "a missing parent is named with no commit") at := metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) diff --git a/internal/controller/gittarget_remote.go b/internal/controller/gittarget_remote.go index c3ec79d9..ee9e45f0 100644 --- a/internal/controller/gittarget_remote.go +++ b/internal/controller/gittarget_remote.go @@ -213,12 +213,13 @@ func remoteStatusIsNews(published, next *configbutleraiv1alpha3.GitTargetRemoteS } // parentStatusOf is the parent stanza of status.remote: present only while the observation found -// the write branch absent and named a parent to create it from. +// the write branch absent, which is when it says what the branch would be created from. func parentStatusOf(observed git.RemoteObservation) *configbutleraiv1alpha3.GitTargetParentStatus { - if observed.Commit != "" || observed.ParentBranch == "" { + if observed.Commit != "" || observed.ParentState == "" { return nil } return &configbutleraiv1alpha3.GitTargetParentStatus{ + State: configbutleraiv1alpha3.GitTargetParentState(observed.ParentState), Branch: observed.ParentBranch, Commit: observed.ParentCommit, } diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index f164aff3..1bca0609 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -2784,25 +2784,34 @@ func (w *BranchWorker) LastRemoteObservation() (RemoteObservation, bool) { // accepted — because both prove the same kind of fact. An error path must NOT call it: a push // that died mid-upload or an advertisement that never arrived observed nothing, and recording a // guess there is how a stale revision reaches status. +// +// An empty revision is a branch the remote does not carry, observed without a parent to name: an +// empty repository, which leaves the first commit unborn. func (w *BranchWorker) recordRemoteObservation(revision string, by ObservationSource) { + if revision == "" { + w.recordAbsentBranch("", "", by) + return + } w.publishObservation(RemoteObservation{Commit: revision, At: time.Now(), By: by, Repo: w.repo}) } // recordMissingParent records that the write branch is absent and its configured parent is too. func (w *BranchWorker) recordMissingParent(parent string, by ObservationSource) { w.publishObservation(RemoteObservation{ - At: time.Now(), By: by, Repo: w.repo, MissingParent: parent, ParentBranch: parent, + At: time.Now(), By: by, Repo: w.repo, ParentState: ParentMissing, ParentBranch: parent, }) } // recordAbsentBranch records that the write branch is absent, and where the parent it would be // created from is. An empty parent is an empty repository, which has none. func (w *BranchWorker) recordAbsentBranch(parentBranch, parentCommit string, by ObservationSource) { + state := ParentFound if parentBranch == "" { - parentCommit = "" + state, parentCommit = ParentUnborn, "" } w.publishObservation(RemoteObservation{ - At: time.Now(), By: by, Repo: w.repo, ParentBranch: parentBranch, ParentCommit: parentCommit, + At: time.Now(), By: by, Repo: w.repo, + ParentState: state, ParentBranch: parentBranch, ParentCommit: parentCommit, }) } diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index d7c840ec..6d56a8bd 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -130,7 +130,7 @@ func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testi assert.False(t, onRemote, "no orphan branch") observed, known := f.worker.LastRemoteObservation() require.True(t, known) - assert.Equal(t, "release", observed.MissingParent) + assert.Equal(t, ParentMissing, observed.ParentState) r1 := f.pushToRelease("RELEASE.md", "now it exists\n") liveWrite(loop, "cm2") @@ -139,7 +139,7 @@ func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testi require.True(t, onRemote, "it recovers once the parent exists") assert.Equal(t, r1, f.commitParent(tip)) observed, _ = f.worker.LastRemoteObservation() - assert.Empty(t, observed.MissingParent) + assert.Empty(t, observed.ParentState, "the write branch exists, so no parent is reported") } // TestBranchWorker_ParentBranchEqualToTheWriteBranch: legal when the branch exists, and then it is @@ -169,7 +169,8 @@ func TestBranchWorker_ParentBranchEqualToTheWriteBranch(t *testing.T) { _, onRemote := f.featureOnRemote() assert.False(t, onRemote) observed, _ := f.worker.LastRemoteObservation() - assert.Equal(t, "feature", observed.MissingParent) + assert.Equal(t, ParentMissing, observed.ParentState) + assert.Equal(t, "feature", observed.ParentBranch) }) t.Run("an empty repository", func(t *testing.T) { @@ -276,10 +277,21 @@ func TestBranchWorker_ObservesTheParentOfAnAbsentBranch(t *testing.T) { o := observed(f) assert.Equal(t, ObservedByFetch, o.By) assert.Empty(t, o.Commit) + assert.Equal(t, ParentFound, o.ParentState) assert.Equal(t, "main", o.ParentBranch) assert.Equal(t, hashA.String(), o.ParentCommit) }) + t.Run("an empty repository is unborn", func(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorkerOnEmptyRemote(t) + require.NoError(t, worker.ensureRepositoryInitialized(worker.ctx)) + o, known := worker.LastRemoteObservation() + require.True(t, known) + assert.Equal(t, ParentUnborn, o.ParentState) + assert.Empty(t, o.ParentBranch) + assert.Empty(t, o.ParentCommit) + }) + t.Run("a push that leaves the branch absent confirms the parent", func(t *testing.T) { f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) hashA := f.warmOnParent() @@ -310,13 +322,13 @@ func TestBranchWorker_ObservesTheParentOfAnAbsentBranch(t *testing.T) { o := observed(f) assert.Equal(t, "release", o.ParentBranch) assert.Empty(t, o.ParentCommit, "the parent is not on the remote") - assert.Equal(t, "release", o.MissingParent) + assert.Equal(t, ParentMissing, o.ParentState) r1 := f.pushToRelease("RELEASE.md", "release\n") require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) o = observed(f) assert.Equal(t, "release", o.ParentBranch) assert.Equal(t, r1.String(), o.ParentCommit) - assert.Empty(t, o.MissingParent) + assert.Equal(t, ParentFound, o.ParentState) }) } diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index d6186a80..9a726c9e 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -680,7 +680,8 @@ func TestRefresh_ReportsAMissingConfiguredParent(t *testing.T) { assert.Zero(t, fetchCount(t, reader, h.worker, fetchReasonRefresh)) require.NotEmpty(t, h.reported) last := h.reported[len(h.reported)-1] - assert.Equal(t, "release", last.MissingParent) + assert.Equal(t, ParentMissing, last.ParentState) + assert.Equal(t, "release", last.ParentBranch) assert.Empty(t, last.Commit) } @@ -694,6 +695,7 @@ func TestRefresh_ObservesTheParentOfAnAbsentBranch(t *testing.T) { require.NotEmpty(t, h.reported) last := h.reported[len(h.reported)-1] assert.Empty(t, last.Commit) + assert.Equal(t, ParentFound, last.ParentState) assert.Equal(t, "main", last.ParentBranch, "an omitted parent is the branch the remote's HEAD names") assert.Equal(t, revParseMain(t, h.repoDir), last.ParentCommit) } diff --git a/internal/git/remote_observation.go b/internal/git/remote_observation.go index fc4d888f..2da85893 100644 --- a/internal/git/remote_observation.go +++ b/internal/git/remote_observation.go @@ -50,15 +50,28 @@ type RemoteObservation struct { // It is zero for a worker that has no identity of its own: the CLI, and tests that never reach // a remote. A comparison against an unknown identity proves nothing, so it is not made. Repo RepoIdentity - // MissingParent names the configured parent branch when the remote carries neither it nor the - // branch, so the branch cannot be created. Empty otherwise, including for an omitted parent. - MissingParent string - // ParentBranch and ParentCommit are the parent an absent branch would be created from, and - // where it was, observed together with the branch's absence. Both are empty while the branch - // exists, and for an empty repository. ParentCommit is empty when the parent is missing. + // ParentState is what the remote offers an absent branch to be created from, observed + // together with its absence. Empty while the branch exists. + ParentState ParentState + // ParentBranch and ParentCommit name that parent and where it is. ParentCommit is empty + // unless ParentState is ParentFound; ParentBranch is empty for ParentUnborn. ParentBranch string ParentCommit string } +// ParentState is what a branch the remote does not carry would be created from. +type ParentState string + +const ( + // ParentFound means the parent branch exists, and the branch's first commit builds on its tip. + ParentFound ParentState = "Found" + // ParentMissing means the configured parent branch is not on the remote either, so the branch + // cannot be created and nothing is written until one of them exists. + ParentMissing ParentState = "Missing" + // ParentUnborn means the repository offers no branch to start from (it is empty) and none was + // configured, so the first commit starts the branch with no history. + ParentUnborn ParentState = "Unborn" +) + // Age is how long ago the observation was made, against now. func (o RemoteObservation) Age(now time.Time) time.Duration { return now.Sub(o.At) } diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index 2c9ff362..2d254a6c 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -181,6 +181,7 @@ spec: By("status.remote names the parent the write branch would be created from") Eventually(func(g Gomega) { g.Expect(gitTargetRemoteField(g, destName, testNs, "commit")).To(BeEmpty(), "the write branch is absent") + g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.state")).To(Equal("Found")) g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.branch")).To(Equal(parent), "with spec.parentBranch omitted this is the default branch the remote resolved") g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.commit")).To(Equal(hashA)) From dcbc8b82c0e1ecf0aee59f1a1bdc95a40ca83bf9 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 09:35:30 +0000 Subject: [PATCH 07/18] fix(git): keep a parent change pending until the rebuild succeeds recoverRetainedWrites took the parent-change flag before the rebuild ran. A rebuild that failed before its reset, reading the GitProvider or the credentials, left nothing remembering that the retained writes sat on the old parent, so the retry published from it, or created the branch although the new parent was missing. Hand the change back when the rebuild fails. Co-Authored-By: Claude Opus 5.5 --- internal/git/branch_worker.go | 10 +++- .../git/branch_worker_parent_branch_test.go | 49 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 1bca0609..5bf54630 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -1434,7 +1434,15 @@ func (l *branchWorkerEventLoop) recoverRetainedWrites() error { // fetchReasonRecovery, the same series the no-retained-writes case records in // ensureBaseForCycle: this is one event, and which half of it an operator sees must not depend // on whether a push happened to be in cooldown at the time. - return l.invalidateAndRefresh("retained writes cannot be trusted", fetchReasonRecovery) + err := l.invalidateAndRefresh("retained writes cannot be trusted", fetchReasonRecovery) + if err != nil && parentChanged { + // The rebuild can fail before its reset marks a replay required (reading the GitProvider or + // the credentials, say), and then nothing else remembers that the retained writes sit on + // the old parent: the next attempt would publish from it. Hand the change back until a + // rebuild succeeds. + l.w.parentChangedState.Store(true) + } + return err } // invalidateAndRefresh drops base trust and, when writes are retained, acts on that invalidation diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index 6d56a8bd..5d68c32b 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -9,6 +9,7 @@ package git import ( "context" + "errors" "path/filepath" "testing" "time" @@ -16,6 +17,8 @@ import ( "github.com/go-git/go-git/v6/plumbing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" ) @@ -259,6 +262,52 @@ func TestBranchWorker_AParentChangeRebuildsRetainedWrites(t *testing.T) { } } +// TestBranchWorker_AParentChangeSurvivesAFailedRebuild: the rebuild a parent change asks for can +// fail before it resets anything, here on reading the GitProvider. The change must still be +// pending on the next attempt; taking it at the first try would leave the retained writes planned +// on the old parent, and the retry would publish from it, or create the branch although the new +// parent is missing. +func TestBranchWorker_AParentChangeSurvivesAFailedRebuild(t *testing.T) { + for _, parent := range []string{"release", "missing"} { + t.Run(parent, func(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + require.True(t, f.worker.SetParentBranch(parent)) + + inner := f.worker.Client.(client.WithWatch) + f.worker.Client = interceptor.NewClient(inner, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, + opts ...client.GetOption) error { + if _, ok := obj.(*configv1alpha3.GitProvider); ok { + return errors.New("transient API failure") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + loop.pushPending() + require.Len(t, loop.pendingWrites, 1, "a failed rebuild keeps the work") + + f.worker.Client = inner + loop.pushPending() + + tip, exists := f.featureOnRemote() + if parent == "missing" { + assert.False(t, exists, "the missing new parent still prevents branch creation on the retry") + return + } + require.True(t, exists) + assert.Equal(t, release, f.commitParent(tip), "the retry starts from the new parent") + }) + } +} + // TestBranchWorker_ObservesTheParentOfAnAbsentBranch: every way the worker looks at the remote // records, alongside the write branch's absence, the parent it would be created from, which // status.remote.parent publishes. With spec.parentBranch omitted that names the remote's default From 015536bb9b79fc7d7a69eda9a27edcaa3544fa3d Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 10:43:33 +0000 Subject: [PATCH 08/18] docs: record where the parent-branch work goes next, and the open #404 follow-ups Four design pages, checked in so the direction is visible: - gittarget-parent-branch.md: the original plan, now implemented in this PR. - gittarget-parent-hardening.md: the hardening pass from the review of this PR. It keeps the fast path pull-free, and covers parent changes on a live worker, an empty repository that gains a ref, an unresolved default branch, and recovery that cannot strand work. - gittarget-parent-empty-repository.md: an empty repository's default branch. It assumes main first; asking the server needs a small go-git change, already tried in an experiment. - gittarget-parent-observation.md: keep the parent visible after the write branch exists. todo.md lists the follow-ups from the #404 review. Co-Authored-By: Claude Opus 5.5 --- docs/design/gittarget-parent-branch.md | 595 ++++++++++++++++ .../gittarget-parent-empty-repository.md | 262 +++++++ docs/design/gittarget-parent-hardening.md | 659 ++++++++++++++++++ docs/design/gittarget-parent-observation.md | 251 +++++++ todo.md | 57 ++ 5 files changed, 1824 insertions(+) create mode 100644 docs/design/gittarget-parent-branch.md create mode 100644 docs/design/gittarget-parent-empty-repository.md create mode 100644 docs/design/gittarget-parent-hardening.md create mode 100644 docs/design/gittarget-parent-observation.md create mode 100644 todo.md diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md new file mode 100644 index 00000000..14919b03 --- /dev/null +++ b/docs/design/gittarget-parent-branch.md @@ -0,0 +1,595 @@ +# GitTarget parent branch: keep a target on standby until a write needs a branch + +> **design**: implemented in #407 (with #408's `spec.parentBranch` folded in). Open follow-ups: +> [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md) and +> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md). +> Date: 2026-10-01. +> Related: [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), +> [`gittarget-configuration-freshness.md`](gittarget-configuration-freshness.md), +> [`../definitions.md`](../definitions.md) + +**The use case.** A `GitTarget` stays on standby while the cluster matches `main`. Its watches are +running, its write branch does not exist on the remote, and normal Git changes can keep flowing +through the cluster's GitOps reconciler. When a captured cluster edit needs a commit, the target +creates its write branch using a freshly checked parent. A target that waited a week must not +publish that edit on a week-old checkout. + +**The proposal.** Let the target name its parent branch. Omitting the field keeps today's default +of the remote's default branch. Setting `release-1.x` or `env/prod` supports the same standby flow +against another branch. The write branch can later serve as a PR head, with the parent as its base; +creating PRs is separate work. + +```yaml +spec: + branch: reverser/prod-edits # the write branch (unchanged, immutable) + parentBranch: main # optional; omitted = the remote's default branch +``` + +**The constraint.** An idle, converged target creates no remote branch and no commit. Publication +validates the parent against the push session's advertisement and replays captured writes if it +moved. If replay leaves no difference, the target stays on standby. Existing commit windows and +push scheduling still apply: activation needs no manual switch or refresh tick, but publication +does not bypass those timers. Here, "no branch" means no branch on the remote; local working refs +may exist. + +**Independent of polling.** We already refresh remote observations with a **ten-minute default** +(`--git-refresh-interval`, Helm `git.refreshInterval`). Build this contract as though that refresher +were absent: it must hold with `--git-refresh-interval=0`, before the first tick, and between ticks. +Periodic refresh improves the idle view of Git; the publication check protects the first write. + +**Before the feature, a defect to fix (§2).** Code review identifies an untested defect: the push is +designed to refuse a parent branch that moved (the parent is recorded as the push root), but +`validatePushState` only makes that comparison when the write branch already exists. So when it does +not, a cycle that records the parent as its root can publish from a stale checkout. The exact worker +setup matters (§2.3); the push primitive's missing check is visible in the code. + +## Use cases and scope + +### Capture an operational edit for review + +The cluster normally follows `main`, and `reverser/prod-edits` is absent. An operator changes a +Deployment during an incident. Reverser captures that change, checks the parent when publishing, +and creates the write branch for review. Any intervening commits to `main` belong in its starting +history. Replay re-plans the captured edit against that new tree, including any changed file layout. + +This records the edit; keeping it active in the cluster still depends on the GitOps reconciler's +existing reconciliation policy. The parent-branch feature does not change that policy. + +### Keep normal deployments quiet + +A deployment advances `main`, and the reconciler applies the same manifests to the cluster. The +resulting watch events can initially look like edits against the cached parent. After checking and +replaying against the newer parent, they produce no difference. Reverser creates neither an empty +review branch nor a commit merely to record that it caught up. + +An idle remote refresh only reads and reports. Parent movement alone must not start a write or +turn the cluster's temporarily older state into a proposed rollback. Explicit resyncs retain their +existing cluster-to-Git semantics; this proposal does not resolve deployment-order races for them. + +### Capture changes against a release or environment branch + +A cluster following `release-1.x` or `env/prod` needs its proposed changes based on that branch. +`parentBranch` expresses this directly. Branching from `main` could otherwise include unrelated +release changes in the eventual PR. An explicit parent that is missing blocks branch creation. + +### Return to standby after review + +After a PR is merged and its write branch is deleted, the same target can use the parent again +when the next edit arrives. This depends on detecting deletion and replaying against the parent, +so it needs a lifecycle test. Automatic deletion, PR management, and deciding whether a surviving +write branch is safe to reset remain separate work (§7). While the write branch exists, new +commits build on it; this proposal does not continuously rebase it onto the parent. + +## 1. What happens today + +The default branch is already the parent branch, implicitly. + +- `SmartFetch` ([`git_smart_fetch.go`](../../internal/git/git_smart_fetch.go)) lists the remote refs. + If `spec.branch` exists it fetches it; otherwise it falls back to the ref the remote `HEAD` points at. + If neither exists (an empty repository) the worker makes `HEAD` unborn and the first commit is a root + commit. +- The worker compares the cluster against whatever it checked out. A resync fetches first + ([`resync_flush.go:132`](../../internal/git/resync_flush.go)), so a resync compares against the + default branch's current tip. +- `ensureWriteBranch` prepares a local `spec.branch` from the checked-out commit before executing + the writes. This can happen even when execution produces no commit. A later push creates the + branch on the remote. +- Once `spec.branch` exists on the remote, the default branch no longer matters. Nothing ever rebases + the write branch onto it. +- The [refresher](../../internal/git/refresh.go) checks aging remote observations on a reconcile + tick. The [refresh setting](../configuration.md#keeping-an-idle-target-fresh---git-refresh-interval) + defaults to ten minutes and sets the observation-age threshold for that check. + An unchanged branch costs one advertisement, and a moved branch also costs a fetch and reset. + Today it returns early when the write branch is absent, so the ten-minute setting does not keep + that target's parent checkout current. Parent following in the refresher is proposed in §2.4. + +## 2. Defect: a live event on an absent write branch commits onto a stale parent + +### 2.1 The path + +1. A worker for `branch: feature`, absent on the remote, finishes a fetch and reset with local + `HEAD` still on `main` at A. `SmartFetch` selected `main` as the fallback. This is the starting + state to establish explicitly; a completed snapshot resync can change it (§2.3). +2. `updateBranchMetadataFromPullReport` + ([`branch_worker.go:2767`](../../internal/git/branch_worker.go)) marks the base **trusted**, on + purpose, including for an absent branch. Its comment gives the reasoning: "for a branch the remote + does not have, a worktree based on the default branch IS that state. There is nothing on the remote + left to learn", and "a push onto a branch we believe is absent declares Old = zero, which the server + rejects if somebody has since created it". +3. Nothing changes in the cluster for a week. Meanwhile `main` moves to commit B. The refresher looks + only at `feature`, finds it absent, and returns without touching trust. +4. A live event arrives. `ensureBaseForCycle` ([`branch_worker.go:1824`](../../internal/git/branch_worker.go)) + sees a trusted, clean base and skips the fetch. The write is planned and committed on top of **A**. + `ensureWriteBranch` records the push-cycle root as `(main, A)` (`pushCycleRootBranch`/`Hash`, + [`branch_worker.go:218`](../../internal/git/branch_worker.go): "for a brand new branch it is the + default branch we based the new branch on. PushAtomic uses this pair to detect concurrent updates"). +5. `PushAtomic` receives that root. But `validatePushState` + ([`git_atomic_push.go:165`](../../internal/git/git_atomic_push.go)) compares the root's recorded + hash with its advertised tip **only inside `if found`**, that is, only when the write branch + already exists on the remote. For a new branch the only root check is "the root still exists". So + `main` at B versus the recorded A is never compared, the push goes out with `Old = zero`, and + `feature` is born on A, a week stale. + +So the design is right and already in place: the parent is recorded as the push root precisely so the +push can refuse a moved parent. The implementation skips that comparison in the one case it was +recorded for. `TestAtomicPush_PushToOther` +([`git_atomic_push_test.go:72`](../../internal/git/git_atomic_push_test.go)) is the only test of a +root that differs from the pushed branch, and its `main` never moves. + +### 2.1a A second hole in the same branch of code + +When the root differs from the pushed branch and the pushed branch **is** found (somebody created +`feature` between our fetch and our push), `oldHash` becomes their commit and the root check compares +`main` against `main`. If `main` did not move, the push sends `Old = `, +`New = `, and nothing checks that ours descends from theirs: their commit is overwritten by a +non-fast-forward update on any server that allows one. The comment in +`updateBranchMetadataFromPullReport` ("a push onto a branch we believe is absent declares Old = zero, +which the server rejects if somebody has since created it") describes behavior this code does not +enforce. Also from reading only. + +### 2.2 Consequences + +- The write was planned against A's folder. If B changed that folder (another writer, a merged PR), + the plan is wrong. At best the new branch lacks B's changes. At worst it re-writes a document B + restructured, and the eventual merge conflicts or quietly undoes B. +- The PR flow is where this surfaces: a freshly created PR branch that is already behind its base. +- A new worker starts untrusted and fetches before planning, so a restart can mask the stale state. + +### 2.3 Why the existing test does not catch it + +`TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain` +([`branch_worker_test.go:668`](../../internal/git/branch_worker_test.go)) checks out A, moves `main` +to B, then commits through a **newly constructed** worker. A new worker starts with the base untrusted, +so it fetches and lands on B. The test proves the cold path; the defect is on the warm path, where the +same worker already trusts its checkout of A. + +The primitive's missing comparison is clear from reading; no reproduction has been executed here. +Step 1 of the plan (§6) proves it with a failing unit test. + +There is a second setup detail: a completed no-op resync calls `ensureWriteBranch` before learning +that there is no difference. It can leave local `HEAD` on `feature`, although `feature` is absent +on the remote. The next cycle then records `(feature, A)`, and the existing missing-root check can +already force a fetch. The original assumption that any warm resync leaves `(main, A)` as the next +push root is therefore too broad. Test both states, and inspect the recorded root before claiming +that the worker or e2e reproduction must fail. + +### 2.3a A no-difference live event likely creates the write branch today + +Verified by reading on 2026-10-01, not executed. This breaks the core constraint more directly than +§2.1 does, so it gets its own red test. A live window that plans no change is **still retained and +pushed**: `finalizeOpenWindowWithReason` appends it "including for a no-diff window", because only +the remote can settle "already present". `commitPendingWrites` has already run `ensureWriteBranch`, +so local `feature` exists at the parent's commit. `runPushCycle` has no zero-commit short cut, and +`validatePushState` returns `PushNoBranch` only when the local hash is zero. With `feature` absent +and the local hash nonzero, the plan is `Old = zero, New = `. The push creates +`feature` on the remote with no commit of its own. If HEAD was already on a local `feature` (§2.3), +the missing-root rejection replays first, and the result is the same branch created at the new +parent tip. + +The "keep normal deployments quiet" use case hits exactly this: the reconciler applies `main`, the +watch sees the apply, and the window finds no difference. §2.4 item 2 is the fix; §6.2 item 4 and +§6.3 step 3b are the tests, and both should be red today. + +### 2.4 Fix + +**Keep publication responsible for freshness.** Use the existing advertisement and replay path: + +1. **`validatePushState` (the fix).** Compare `rootHash` with the root's advertised tip whether or not + the pushed branch exists. And when the root is a different branch from the one being pushed (a new + branch), require the pushed branch to still be absent: if it was found, return `RemoteMovedError` + for the pushed branch instead of pushing over it (§2.1a). Both rejections already flow into the + existing path: `remoteMovedDuringPush` invalidates the base, fetches, rebuilds the retained writes, + and pushes again. No extra connection, since the advertisement is read on the push session anyway. + Keep `Old = zero` for branch creation so the server also rejects a branch created after the + advertisement. The parent comparison is a client-side check: it is not atomic with the server's + update of the write branch. A parent can move during upload, with no fixed time bound on that + gap. The contract is the parent tip observed in the successful attempt's advertisement. +2. **Preserve standby after a no-op.** Live writes still reach the advertisement when they produce + no commit. After validation and any replay, if no pending write produced a commit and the write + branch remains absent, finish without creating it. Today `PushNoBranch` only covers an unborn + local `HEAD`; a local branch pointing at a nonzero parent needs explicit coverage too. Keep the + expected remote absence and resolved parent available across local branch switches and no-op + cycles. Local `HEAD` alone does not describe that relationship (§2.3). +3. **In the refresher (the folder view).** While the write branch is absent, check the parent branch + in the same advertisement. If it moved past the local checkout, fetch, reset, and re-read the + folder, so `status.placement` and refusals describe what the next commit will build on. + Still one connection when nothing moved; a moved parent also costs the fetch path. No commit. + This improves idle observations independently of the publication guarantee. Delaying or + disabling it must not change which parent a new write branch uses. + +Replay means re-planning the retained cluster writes against the fetched tree. It preserves +unrelated content and applies the existing conflict policy for the captured objects; it is not a +Git rebase of a previously published branch. If the remote cannot be checked, retain the pending +work for retry. If the parent keeps moving, use the existing bounded retry behavior and retain the +work rather than publishing an unchecked result. + +Rejected: removing trust for an absent write branch. The comment in step 2 records that an earlier +version did exactly that and was reverted for cost, and it would still leave the window between a +fetch and the push. + +This fix stands on its own and should ship first, without the new field: it is a correctness bug in +today's default-branch behavior. + +## 3. Prior art + +The original survey read both projects from `external-sources/` and the module cache. + +### Flux: `ImageUpdateAutomation` + +`image-automation-controller/api@v1.2.3`, `api/v1/git.go`: `spec.git.checkout.ref` (a full +`GitRepositoryRef`: branch, tag, semver, name or commit) and `spec.git.push.branch`, which "is created +using `.spec.checkout.branch` as the starting point, if it doesn't already exist". Omitted `checkout` +falls back to the `GitRepository`'s ref, and that to `master`. + +The starting point matters only when the push branch is created. `SwitchBranch` +(`external-sources/flux/pkg/git/gogit/client.go:462` in the local checkout): push +branch on the remote, build on it; absent, create it from the checked-out HEAD. There is no rebase. +Once the push branch exists, new commits on `main` are invisible to it; the usual remedy is to delete +the branch after the PR merges. + +### Argo CD: the source hydrator + +`pkg/apis/application/v1alpha1/types.go:428`: `drySource.targetRevision` (what it reads, any +revision), `syncSource.targetBranch` (the starting point), and an optional `hydrateTo.targetBranch` +(a staging branch "an external system would then have to move ... e.g. by pull request"). + +- Free-form revisions appear only where Argo reads. Everything it writes to, or creates a branch from, + is typed `targetBranch`. +- `CheckoutOrNew(target, base)` (`util/git/client.go:1637`): check out `hydrateTo` if it exists, else + create it from `syncSource`. Starting point at creation only, never a rebase. +- No difference, no commit: `WriteForPaths` returns `shouldCommit=false` and the commit server records + a git note instead (`commitserver/commit/commit.go:176`). +- `CheckoutOrOrphan` (`util/git/client.go:154`) creates an orphan branch when the sync branch is + absent: the explicit "empty branch" case of §5. + +### What we take from them + +1. A branch-typed field at a write position (Argo). +2. Both treat the starting point as a creation-time input only. We already compare against the parent + while the write branch is absent, and §7 is where we can go further than either. + +### Git tools that use "parent branch" + +git-town (`git town set-parent`, `git town sync`) and Graphite's stacks use **parent branch** for a +branch relationship that also includes synchronization. Our initial contract is narrower: follow +the parent while the write branch is absent, then build on the write branch. The name must not +promise automatic synchronization of an existing write branch; that remains deferred (§7). + +## 4. Proposal + +### 4.1 API + +```go +// GitTargetSpec + + // ParentBranch is the branch the write branch (spec.branch) is created from, and that the target + // compares against while the write branch does not exist on the remote. It is only read, so it + // does not need to be in the GitProvider's allowedBranches. Omitted: the branch the remote's HEAD + // points at (its default branch). When the write branch is absent, an explicitly set parent + // must exist or branch creation is refused. An existing write branch is used independently. + // +optional + // +kubebuilder:validation:MinLength=1 + ParentBranch string `json:"parentBranch,omitempty"` +``` + +- **Mutable.** It says where a branch starts; changing it abandons nothing already written (unlike + `branch`/`path`, which are the destination). A change takes effect the next time the write branch is + absent. +- **Not in `allowedBranches`.** We never write to it. +- **`parentBranch == branch` is legal** when that branch exists. If it is absent, the explicit + parent requirement blocks creation, including in an empty repository. Omission retains today's + bootstrap behavior. + +### 4.2 Naming + +[`definitions.md`](../definitions.md) binds here. The candidates it rules out: + +| Candidate | Problem | +|---|---| +| `sourceRevision` | "source" is the source *cluster* (`clusterProviderRef`, `rules[].sourceNamespace`) | +| `baseRevision`, `baseRef`, `baseBranch` | "base" already means a kustomize **read-only base** and **base trust** (rule 1: one concept, one word); `revision` is reserved for a polymorphic Flux-style identifier; a `...Ref` field must reference an object (rule 2) | +| `checkout.ref.branch` | verbatim Flux, but a one-field block nested twice, and "checkout" names the action while this field names a relationship | +| `startPoint` | Git's own word (`git switch -c `), but it means creation only, and the field also governs what is followed | + +`parentBranch` is a single fact, so it is flat (rule 2), and it names the standby relationship. The +word "parent" appears in the subresource specs ("parent resource"), but "parent branch" always carries +its noun. In prose: **the parent branch** and **the write branch**. Both go into `definitions.md` +under "Write side" when this ships. + +The cost of the name: it can never hold a commit or tag. That is accepted (§5). + +### 4.3 One parent branch per write branch + +The branch worker is keyed by `(GitProvider namespace, GitProvider name, branch)` and owns one +checkout, so every target on that write branch must agree on its parent. `checkForConflicts` +([`gittarget_controller.go:1087`](../../internal/controller/gittarget_controller.go)) gains a second +clause: same provider and write branch, different `parentBranch` is a `TargetConflict`, and the +later-created target loses, as it does for overlapping paths. + +Values are compared as written, with omitted as its own value, so omitted and `main` conflict even +when the remote's `HEAD` is `main`. The controller decides without a remote round trip, and the answer +must not flip when somebody changes the remote's default branch. + +The worker receives the parent branch alongside `RepoIdentity`, and `EnsureWorker` treats a change +like a GitProvider repoint: invalidate the base, keep the worker. + +### 4.4 Worker behavior + +| Write branch on remote | Parent branch on remote | Behavior | +|---|---|---| +| yes | any | unchanged: fetch and work on the write branch | +| no | yes | compare against the parent branch; create the write branch from its current tip on the first commit (§2.4) | +| no | no, **set explicitly** | **refuse**: `Ready=False`, reason `ParentBranchNotFound`, nothing written | +| no | no, omitted, remote `HEAD` unset | unchanged: unborn `HEAD`, root commit (empty repository) | + +`SmartFetch` takes the parent branch instead of always using remote `HEAD`. When a configured parent +branch is missing it returns a sentinel error rather than `""`, so the worker never starts an orphan +branch because of a typo. + +`status.remote` keeps meaning the write branch (empty `commit` = not created yet). + +## 5. Implicit when omitted, strict when set; deferred explicit forms + +**The principle.** Omitted is lenient: the default branch, and an empty branch with no history when +the repository itself is empty, because that is the first-install case and requiring configuration for +it is friction. Set is strict: it must exist, or the target refuses. Making the empty-repository case +explicit would force everyone to configure today's default. + +**Deferred, additive when someone asks:** + +- **A deliberately empty write branch in a non-empty repository.** A dedicated `cluster-state` branch + with no history from `main`, the `gh-pages` pattern; Argo's `CheckoutOrOrphan`. Legitimate, unasked. + It would have its own field. +- **A pinned commit or tag** ("branch off exactly what is deployed"). Small value: the starting point + only decides where the write branch is born, after which it carries on independently, and a pin + leaves the refresher nothing to follow. Fetching one commit by hash also needs server support + (`uploadpack.allowReachableSHA1InWant` or similar); support varies between servers, which is why + Flux lets `commit` be paired with `branch`. It cannot go in `parentBranch`; it would be a sibling + field. + +## 5a. Handoff: executing this from a fresh context + +Read this section first. + +- **Historical.** This was the handoff for #407 and #408, both implemented. The argument lives in the + PR bodies, and the lasting text in `architecture.md` (§8). +- **Do not disturb the commit-window work** on `feat/commit-window-surface` in the main checkout. Work + in a separate git worktree on a new branch off `origin/main`. Before running `task test-e2e`, ask + the user: the e2e cluster is shared and a second run dies on its per-cluster lock. +- **Two PRs, in this order.** + - **PR 1, `fix(git)`:** §6.1, the §6.3 e2e spec, the §2.4 fix, the §6.2 tests, and the §8.1 to 8.3 + `architecture.md` edits. Also correct the comment in `updateBranchMetadataFromPullReport` that + claims an absent branch is pushed with `Old = zero` and that the server rejects it if somebody + created it. + - **PR 2, `feat(api)`:** the field, §4 and §6.4, plus §8.4. +- **This is the Git write path** (`internal/git/`), which AGENTS.md treats as high risk. Follow the + repo's validation sequence; the coverage ratchet bump (`.coverage-baseline`) is committed with the + change. +- **Show red before green for the defect.** The two §6.1 atomic-push tests should fail before the + fix. Worker tests must distinguish the local states in §2.3; an acceptance test that already + passes through missing-root recovery does not reproduce the primitive defect. Inspect the + recorded root and replay path, and retain that test as lifecycle coverage. + +### End-to-end test pitfalls found while planning §6.3 + +- **The Gitea repository starts empty.** Seed `main` with commit A from outside before creating the + target, or there is no parent to go stale. +- **Every helper hard-codes `main`.** `remoteBranchHead` (`repo_assertions_test.go:46`) runs + `ls-remote --heads origin main`, and `pushKustomizationFromOutside` (`refresh_remote_e2e_test.go:175`) + fetches and pushes `main`. The spec needs a branch parameter for the first, and a "commit file X to + `main` from outside" variant of the second. +- **The spec can pass without exercising the defect.** The stale-parent path needs a **warm** worker + that checked out A and trusts it *before* B is pushed. If the worker has not cloned yet when the + ConfigMap event arrives, it fetches on the cold path, lands on B, and the spec passes without + proving anything. So step 2 must observe that the worker fetched at A: candidates are the per-branch + fetch metric (the `fetchReason*` series in `branch_worker.go`, scraped by the e2e Prometheus) or the + controller log line `Preparing branch for operations` for that branch. Also account for a local + branch created by a no-op resync (§2.3). Do not use `status.remote` as proof of the checked-out + parent: it describes the write branch, and publication has its own cadence. +- **The target's own initial snapshot must not create the branch.** Use a `WatchRule` with a label + selector that matches nothing (the namespace always holds `kube-root-ca.crt`), so the snapshot is + an in-sync no-op that still clones. +- **The GitProvider's `allowedBranches`** must include the write branch. + +## 6. Plan and tests + +Order matters: prove today's behavior, fix the defect on today's default, then add the field. + +### 6.1 Reproduce the defect (unit, fails today) + +At the push primitive, next to `TestAtomicPush_PushToOther`: + +- `TestAtomicPush_PushToOther_DetectsMovedRoot`: local `feature` built on `main` at A, `main` moved to + B on the remote, `feature` absent. Expect `RemoteMovedError{Branch: main, Expected: A, Advertised: B}`. +- `TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch`: `main` unmoved, but `feature` created + on the remote by somebody else. Expect `RemoteMovedError` for `feature`, and their commit untouched. + +At the worker, through the whole cycle, cover both local states: + +`TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip`: one worker, seeded `main` +at A, write branch `feature` absent on the remote. First exercise a trusted fetch/reset that leaves +`HEAD` on `main`; separately exercise a completed no-op resync that leaves a local `feature`. +Assert the setup in each case. Push B to `main` from a second clone, then send a live write through +the **same** worker, with no intervening refresh or resync. Assert the first published commit on +`feature` has parent B and contains B's file. The first variant targets the missing parent check; +the second guards the normal standby lifecycle and can already pass through missing-root recovery. + +### 6.2 Fix (§2.4), and cover the "no commit while idle" side + +All with a seeded `main` and `feature` absent; every existing test of these rules runs on `main` or on +an empty remote: + +| Existing test | Why it does not cover the case | +|---|---| +| `TestResync_WorkerNoopDoesNotRetainOrPush` | write branch is `main` | +| `TestBranchWorker_DoesNotCreateBootstrapOnlyCommit` | empty remote | +| `TestRefresh_AnAbsentBranchCostsOneConnection` | empty remote, nothing to follow | +| `TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain` | cold worker only (§2.3) | +| e2e `commit_request_e2e_test.go:97` ("branch is not created") | empty repository, branch `main` | + +1. An in-sync snapshot creates no `feature`, and no push is attempted. +2. `main` moves (an unrelated file); a second in-sync snapshot still creates no `feature`. +3. `main` moves so the cluster now differs; the resync diffs against the new `main`. +4. `main` gains exactly the document the cluster has; replay leaves no commit and `feature` is + never created on the remote. Also cover a live write that already matches an unmoved parent. + Both still verify the remote before reporting completion. The live-write variant is expected red + today (§2.3a). +5. Push-time check: parent moved between plan and push; the push is treated as rejected, replays, and + `feature` parents on the new tip. Happy path: no extra connection (count with the existing fetch + and connection metrics, as `refresh_test.go` does). +6. Refresher: parent moved while `feature` is absent: advertisement followed by the fetch/reset + path and folder re-read, no commit. Parent unmoved: one connection, no fetch. +7. Publication with periodic refresh disabled, and with the last refresh completed immediately before + `main` advances: both start from the parent checked during publication. Exercise parent movement + while a commit window is open too. A refresh tick is never a precondition for activation. +8. Merge and delete the write branch, advance `main`, then send another edit through the same + worker. The recreated branch contains the new parent history and only the new pending edits. +9. Parent movement during replay retries: each attempt checks again. An unavailable remote or + exhausted retries retain the pending work and leave the remote write branch uncreated. + +### 6.3 e2e: the first commit lands on the latest default branch (required) + +This spec is the acceptance test for the standby lifecycle and is not optional. Write it after 6.1, +before the fix. If it already passes, inspect whether it used missing-root recovery (§2.3); the +primitive tests still have to reproduce the missing comparison. + +There is none today: apart from a negative `allowedBranches` check +(`gitprovider_validation_e2e_test.go:99`), every e2e `GitTarget` writes to `main`. New spec, +`Label("manager")`, no `parentBranch` set: + +1. Seed the Gitea repo's `main` with commit A. The `GitProvider` allows `main` and + `reverser-e2e-`. +2. Create a `WatchRule` whose label selector matches nothing yet, and a `GitTarget` writing to + `reverser-e2e-`. Wait until its initial snapshot has completed, which is what makes the + worker check out A and trust it. +3. Assert the write branch does not exist (`ls-remote`), and still does not after a steady tick: + idle means no branch and no commit. + 3b. **A live event with no difference.** Seed A with the folder already holding a ConfigMap + document, then create that ConfigMap in the cluster with matching content and the selector's + label. Assert the write branch is still absent once the commit window and push cooldown have + passed. This is the "keep normal deployments quiet" case, and §2.3a expects it to fail today. +4. Push commit B to `main` from outside, touching a file inside the target's folder, the way + `pushKustomizationFromOutside` does in `refresh_remote_e2e_test.go`. +5. Create a `ConfigMap` that matches the selector. Avoid a new resync between B and this live event. + Prove that recovery happened on the **publication** path by asserting which fetch fixed it: the + per-branch `GitFetchesTotal` series must gain a `contention` fetch (the rejected push) and no + `refresh` fetch between steps 4 and 6. That removes the refresher from the proof without a + dedicated deployment. The e2e controller runs with `--git-refresh-interval=30s` for every spec, + and redeploying it with `0` for one spec would disturb the rest of its leg. The + refresh-disabled case is cheap at unit level, where §6.2 item 7 covers it. +6. Assert the write branch now exists, its tip's parent is B, B's file is present, and the + `ConfigMap` document is in the folder. + +The bring-up costs one Gitea repository and one target; it fits an existing `manager` leg. + +### 6.4 The field + +- `parentBranch: release` with `release` ≠ default: the new write branch parents on `release`'s + tip checked during publication. +- Configured parent missing: `ParentBranchNotFound`, no orphan branch, no push; recovers once the + branch is pushed. +- `parentBranch == branch`: existing branch works; absent branch refuses, including on an empty + remote. An omitted parent still allows empty-repository bootstrap. +- Once the write branch exists, moving the parent changes nothing (the Flux/Argo rule we keep for now). +- `TargetConflict` for two targets on one write branch with different parents; created-later loses; + omitted versus `main` conflicts. +- Changing `parentBranch` on a live target while the write branch is absent re-reads the folder on the + next refresh. +- CRD: `parentBranch` non-empty when set; mutable. +- e2e: the §6.3 spec repeated with `parentBranch` naming a non-default branch. + +## 7. Next phase: following again after a merge + +Deleting the write branch after merge lets the target return to standby (§6.2). A write branch +left on the remote needs a separate policy: its next change builds on that surviving branch, which +may have fallen behind the parent. With a squash merge, a later PR can also show old commit history. + +Candidate rule: **when the write branch adds nothing to the parent branch, reset it to the parent's +tip** (force-push, or delete it and let the next commit recreate it). "Adds nothing" is easy for a +merge commit or fast-forward: the write tip is an ancestor of the parent tip. A squash merge is +different: the commits never enter `main`'s history, so the test has to be content-based, for example +"the folder's tree at the write tip equals the folder's tree at the parent tip". Open points: compare +the folder only, or every path the branch ever touched; and whether resetting a branch with an open PR +is acceptable on forges that treat it as closing the PR. Settle this with the PR-creation design. + +## 8. Changes to `docs/architecture.md`, applied with the fix + +Not applied yet, because `architecture.md` describes what the code does and today it does not do this. +Each edit lands in the PR that makes it true: 8.1 to 8.3 with the §2.4 fix, 8.4 with the field. + +`architecture.md` says nothing today about where a new write branch starts, and its ground rule +overstates the guarantee for a branch that does not exist yet: "a stale 'trusted' is caught by the +compare-and-swap on the next push" (line 73). The rule to state, briefly: + +> An idle target creates no branch and no commit. When the write branch does not exist yet, the +> target compares against its parent branch. Publication checks that parent against the push +> session's advertisement and replays retained writes when it moved. If replay leaves no commit, +> the write branch stays absent. This works with periodic remote refresh disabled. + +### 8.1 Ground rules (after the "stale trusted" sentence, around line 73) + +> For a new write branch, the worker checks the parent's advertised tip before sending the push. +> A moved parent triggers fetch, reset, and replay. The server's compare-and-swap protects creation +> of the write branch with `Old = zero`; it does not lock the parent. The parent can still move +> after the advertisement, so the guarantee is freshness at that observation, not at server acceptance. + +### 8.2 Local clones and conflict retry (next to the "deleted branch" paragraph, around line 1234) + +> **A branch that does not exist yet** is created from the remote's default branch (its parent) +> only when a write has something to commit, so an idle or in-sync target never creates it. +> Until then the worker compares the folder against the +> parent. Periodic refresh, ten minutes by default, updates the idle view independently of the +> publication check. The push records the parent and the commit it started from as its root, and +> treats a moved parent like a moved branch: fetch, reset, replay, check again. Proved end to end +> by the e2e spec `` with periodic refresh disabled. + +### 8.3 "Remote moved while we were writing" flowchart (around line 570) + +The decision node becomes `Write branch (or, while it is absent, its parent) still at the expected +commit?`, and the prose above it gains half a sentence: "...or, for a branch not created yet, if its +parent branch moved". + +### 8.4 Configuration model, GitTarget key fields (around line 358) + +> - `spec.parentBranch`: optional, mutable branch the write branch is created from and compared +> against while it does not exist. Omitted means the remote's default branch. An explicit parent +> must exist when creating the write branch. It does not need to be in `allowedBranches` because +> it is only read. + +Plus one line beside the existing overlap rule: "...and rejects two GitTargets on one branch that name +different parent branches." Update §8.2's default-branch wording to include the configured parent +in the same feature PR. + +## 9. Open questions + +1. Report the parent branch's observed commit in status (`status.remote.parent.commit`?) so an + operator can see what an absent write branch would be created from. Useful, but it grows a block the + refresher samples on the steady tick. +2. `ParentBranchNotFound` as a `Ready=False` reason versus its own condition. It gates writing, like + `BranchNotAllowed`, so a `Ready` reason looks right; confirm against + [`../spec/status-conditions-guide.md`](../spec/status-conditions-guide.md). +3. `GitProvider.status.branches` lists write branches; should each entry show its parent branch? +4. An omitted parent resolves through remote `HEAD` during fetch. Define when to re-resolve it if + the default branch changes while the old parent still exists: the current push session does + not advertise symbolic `HEAD`, so checking the recorded parent's hash alone cannot detect that + change. Explicit `parentBranch: main` avoids this ambiguity for the primary standby use case. diff --git a/docs/design/gittarget-parent-empty-repository.md b/docs/design/gittarget-parent-empty-repository.md new file mode 100644 index 00000000..9b035be3 --- /dev/null +++ b/docs/design/gittarget-parent-empty-repository.md @@ -0,0 +1,262 @@ +# GitTarget parent branch: an empty repository's default branch + +> **Plan**. The experiment in §2 is done; nothing else is implemented. Dated 2026-10-01. +> +> Siblings: +> +> - [`gittarget-parent-branch.md`](gittarget-parent-branch.md), the original plan; +> - [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md), the hardening pass on #407. +> +> Run it **after** the hardening pass. Its §4.2 and §4.3 define "empty" and `Unborn`, and this +> plan refines that `Unborn` case. + +## 1. The problem + +Take an empty repository whose default branch is `main`, but which has no commits yet. Add a +`GitTarget` with `branch: reverser/edits` and no `parentBranch`. + +Today the first write creates **`reverser/edits` as the repository's first branch**: + +- The commit has no parent, and no `main` exists afterwards. +- What we pushed does not match the branch the repository says is its default. +- On GitHub, the first branch pushed to an empty repository probably becomes the default branch + (verify this, §5 step 0). If so, the operator's review branch silently becomes the repository's + default branch. Even if not, the eventual PR `reverser/edits` → `main` cannot be made: the two + branches have no common history. + +This is not a status detail. It is the wrong branch name on the first commit we make. + +`branch: main`, which is what the quickstart uses, is unaffected: the first commit creates `main`, +which is correct. + +## 2. What we know (experiment, 2026-10-01) + +**The server already tells us the default branch.** Over protocol v2, `ls-refs` with the `unborn` +argument reports the target of an unborn HEAD: + +```text +git -c protocol.version=2 ls-remote --symref https://github.com/ConfigButler/empty.git + (GIT_TRACE_PACKET) ls-remote< unborn HEAD symref-target:refs/heads/main +``` + +- Canonical git 2.39 does the same: a bare repository made with `git init --bare -b trunk` + answers `unborn HEAD symref-target:refs/heads/trunk`. +- Protocol v0 reports nothing. + +**go-git asks the question, then throws the answer away.** At v6.0.0-alpha.5, and unchanged on +upstream main at `fc18716c`: + +- It speaks protocol v2 by default, and sends `unborn` when the server advertises + `ls-refs=unborn` (`internal/transport/v2.go`, `lsRefsSupportsUnborn`). +- But when there are no hash refs, `GetRemoteRefs` returns plain `ErrEmptyRemoteRepository` for an + upload-pack session, and the reply is discarded: + - `plumbing/transport/http/handshake.go` ~252; + - `plumbing/transport/pack_stream.go` ~119. +- `RemoteRefs.Unborn` exists, but it is unreachable for an empty repository. +- So our `CheckRepo` has to answer "no way to know" (`internal/git/git.go:59`). + +**Why our live test passes.** `TestCheckRepo_PublicConnectivityEmpty` +(`internal/git/git_operations_test.go:175`, against `ConfigButler/empty.git`) runs, and is not +disabled. It skips only without network. It asserts that `DefaultBranch` is **empty** and that +the local HEAD is unborn on the *write* branch (`cool-test`). It pins today's behavior rather than +the server's knowledge. + +**A small, backward-compatible go-git patch recovers the answer.** I tested this patch through a +temporary `-modfile` `replace`; nothing in the repository changed. The patch: + +- adds `transport.EmptyRemoteRepositoryError{Unborn plumbing.ReferenceName}`, whose `Is(ErrEmptyRemoteRepository)` + is true; +- returns it from both empty-advertisement paths above. + +Results: + +- `Remote.List` against `ConfigButler/empty.git` returns an error that still matches + `errors.Is(err, ErrEmptyRemoteRepository)`, and `errors.As` yields `Unborn = refs/heads/main`. +- The existing live test still passes unchanged. +- The diff is in §7. + +**Our hermetic harness cannot show this today.** `startGitHTTPServer` +(`internal/git/ado_multiack_test.go` ~125) deliberately strips `Git-Protocol` to force protocol +v0, so that it simulates Azure DevOps. Against it, the patched go-git returns the plain error with +no `Unborn`. + +**Not reported:** any server or path that negotiates v0/v1. Whether Azure DevOps supports v2 is +unknown; the harness assumes v0. SSH to GitHub needs `GIT_PROTOCOL` sent over the session; that is +not verified. + +## 3. Proposed behavior + +Everything below turns on one name, the **default branch D** of an empty repository. There are two +ways to know D, and they ship in two stages. + +### 3.1 Stage 1, which this plan leads with: assume `main` + +**D is `main`**, hardcoded. This needs no go-git change, so it can ship now, alongside or right +after the hardening pass. + +- `main` is what GitHub, GitLab and Azure DevOps create for a new repository, and what `git init` + suggests. `master` is not an acceptable default in 2026, and nothing here falls back to it. +- Nothing is guessed silently. The assumption is visible in `status.remote.parent.branch`, it is + named in the refusal message, and it is documented in `configuration.md`. + +### 3.2 Stage 2: ask the server + +When the server reports its unborn HEAD (§2: protocol v2 `ls-refs` with `unborn`, once go-git +passes it through, §4), **D is what the server said**. This only refines stage 1: the rules below +are the same, and only where D comes from changes. + +### 3.3 The rules, given D + +| Empty repository, and… | Behavior | +|---|---| +| `branch == D`, parent omitted | as today: the first commit creates D as the root. `status.remote.parent = {state: Unborn, branch: D}` | +| `branch != D`, parent omitted | **refuse.** Ready=False/Stalled, reason `ParentBranchUnborn`: "the repository is empty; its default branch 'D' has no commits yet. Push a first commit to 'D', or set `spec.branch: D`". Nothing is written; the work is retained, or remembered per hardening §4.4. When D appears, the normal standby flow creates the write branch from it | +| `parentBranch: P` | the same refusal and reason, naming P. Today this is `Missing`, a less precise message. See the escape hatch below for `P == branch` | + +**The cost of stage 1.** A repository whose real default branch is not `main` (say `trunk`), on a +server that does not report its unborn HEAD (v0/v1; Azure DevOps is unverified), with +`branch: trunk`: + +- Today that bootstraps `trunk` correctly. +- Under stage 1 it is refused, because we believe D is `main`. The message says exactly what to + do: push a first commit to `trunk`, after which everything works. +- Stage 2 removes the refusal for every server that reports its unborn HEAD, which includes + GitHub. + +**Escape hatch, decision for the user:** allow `parentBranch == branch` on an empty repository to +mean "start this branch with no history". + +- This is explicit intent, needs no knowledge of D, and fixes the cost above without a manual + push. +- It reverses one rule from #408. Today, `parentBranch == branch` on an absent branch is refused + even in an empty repository. + +**Why refuse rather than bootstrap D ourselves.** The parent-branch contract is that the parent is +*only read*, and it is not in `allowedBranches`. Creating D ourselves would break that, and would +mean inventing an "initialize repository" commit on a branch we don't own. Refusing is the "clear +failure" this design asks for, and recovery is the existing standby flow. + +**Second decision for the user:** an opt-in that bootstraps instead. + +- The mechanism would be one atomic push of two refs: an empty-tree commit on D, and the content + commit on the write branch on top of it, with D required to be in `allowedBranches`. +- This plan does not include it. If you want it, it is a follow-up. + +**`RemoteObservation` and the API.** For `Unborn`, `ParentBranch` and `status.remote.parent.branch` +now always carry D. Update the doc comments that today say "empty when Unborn": + +- `internal/git/remote_observation.go`; +- `api/v1alpha3/gittarget_types.go` ~498–512; +- `configuration.md`. + +## 4. Getting the go-git change (stage 2 only) + +1. **Upstream first.** Open a go-git issue and a PR with §7: the error type, both paths, and tests + against a v2 `git http-backend` and over `file://`. + - The change is additive, and backward-compatible for every `errors.Is` caller. + - For prior art on our upstream write-ups, see the `file://` CAS write-up in + `external-sources/go-git/` (gitignored). +2. **Our side waits for a go-git release that carries it.** Until then stage 1 applies, so nothing + waits on upstream. +3. **Fallback, if upstream stalls:** a temporary `replace` to a fork tag. + - Only with the user's agreement. We already pin an alpha. + - Do not hand-roll a v2 `ls-refs` client in this repository. + +## 5. Steps + +### 5.1 Stage 1 (now) + +0. **Verify the GitHub claim** in §1 on a *throwaway* repository the user creates, never on + `ConfigButler/empty.git`. The question: does pushing a non-default branch first make it the + default? Record the answer in §1. It sharpens the motivation but does not change the design. +1. **Discovery.** The empty-repository result carries D. For now that is the constant + `DefaultBranchForEmptyRepository = "main"`, with one comment explaining why `main` and why + never `master`. `CheckRepo` returns `DefaultBranch{ShortName: "main", Unborn: true}`. +2. **Worker.** + - The empty-repository path applies the §3.3 table before `makeHeadUnborn`. + - The refusal is a new sentinel, `ErrParentBranchUnborn`. Handle it like + `ErrParentBranchNotFound` on fetch, refresh, publication and the hardening probe, so the same + recovery obligation applies. + - The observation becomes `ParentState=Unborn` with `ParentBranch=D`. +3. **Controller.** `parentBranchReadiness` maps that observation (with a `ParentRequested` match) + to `ParentBranchUnborn`, Stalled, using the §3.3 message. +4. **Tests**, red first: + - empty repository with `branch: main`: a root commit on `main`, as today. This covers the + quickstart path; + - `branch: feature`: refused, nothing pushed, the work retained. Then push `main` externally, + and assert that `feature` is created from it without another cluster edit; + - `parentBranch: release` on an empty repository: the `ParentBranchUnborn` message; + - the escape hatch, if chosen; + - ledger: rows unchanged. A constant costs no connection. +5. **Live test.** `TestCheckRepo_PublicConnectivityEmpty` expects `main` / `Unborn`, and its + `PrepareBranch("cool-test")` half now expects the refusal. +6. **Docs.** + - In `configuration.md`'s parent-branch section: add the empty-repository rows, the `main` + assumption, and the escape hatch, if chosen. + - In UPGRADING: a target writing anything other than `main` into an empty repository now waits + for `main`, or for the escape hatch. + +### 5.2 Stage 2 (once go-git carries the change) + +1. **Harness.** Add a `startRealGitServerV2`, or an option, that keeps `Git-Protocol`, so v2 + reaches `git http-backend`. Leave the ADO/v0 server and the ledger rows untouched. +2. **Discovery.** `listRemoteRefs`/`SmartFetchFrom` and `CheckRepo` use `errors.As` to read the + unborn target. When present it replaces the constant; when absent, the constant stays. +3. **Tests**, on the v2 harness, with a bare repository made by `init -b trunk`: + - `branch: trunk` bootstraps `trunk`; + - `branch: feature` is refused, naming `trunk`. + + On the v0 harness, the stage 1 behavior holds and is pinned. The ledger is unchanged: the + unborn target rides on the `ls-refs` call discovery already makes. + +## 6. Out of scope + +- The opt-in bootstrap (§3.3, a decision for the user). +- Repositories that are non-empty but whose default branch is unresolved. That is hardening §4.3. + +## 7. The go-git patch used in the experiment (against v6.0.0-alpha.5) + +```text +--- a/plumbing/transport/errors.go ++++ b/plumbing/transport/errors.go +@@ import ( + "errors" + ++ "github.com/go-git/go-git/v6/plumbing" ++ + internal "github.com/go-git/go-git/v6/internal/transport" + ) +@@ ++// EmptyRemoteRepositoryError is ErrEmptyRemoteRepository with what the server ++// said about its unborn HEAD (protocol v2 ls-refs "unborn"), when it said it. ++type EmptyRemoteRepositoryError struct { ++ // Unborn is the branch HEAD points at, e.g. refs/heads/main; empty when ++ // the server did not report it (v0/v1, or no ls-refs=unborn support). ++ Unborn plumbing.ReferenceName ++} ++ ++func (e *EmptyRemoteRepositoryError) Error() string { return ErrEmptyRemoteRepository.Error() } ++ ++func (e *EmptyRemoteRepositoryError) Is(target error) bool { return target == ErrEmptyRemoteRepository } ++ ++func NewEmptyRemoteRepositoryError(refs []*plumbing.Reference) error { ++ return &EmptyRemoteRepositoryError{Unborn: NewRemoteRefs(refs).Unborn} ++} +--- a/plumbing/transport/http/handshake.go (smartPackSession.GetRemoteRefs, v2 branch) +- return nil, transport.ErrEmptyRemoteRepository ++ return nil, transport.NewEmptyRemoteRepositoryError(refs) +--- a/plumbing/transport/pack_stream.go (StreamSession.GetRemoteRefs, v2 branch) +- return nil, ErrEmptyRemoteRepository ++ return nil, NewEmptyRemoteRepositoryError(refs) +``` + +To reproduce the experiment: + +1. Copy the module from `/go/pkg/mod/github.com/go-git/go-git/v6@v6.0.0-alpha.5` into a scratch + directory and apply the patch. +2. Copy `go.mod` and `go.sum` to `exp.mod` and `exp.sum`, and append + `replace github.com/go-git/go-git/v6 => /go-git`. +3. Run `go test -modfile=exp.mod ./internal/git -run `. + +An upstream PR should probably also expose the unborn target on `Remote.List`'s caller-facing API +rather than only through the error. Discuss the shape with the maintainers. diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md new file mode 100644 index 00000000..c581b166 --- /dev/null +++ b/docs/design/gittarget-parent-hardening.md @@ -0,0 +1,659 @@ +# GitTarget parent branch: hardening pass before #407 merges + +> **Plan**, to be executed from a fresh context. Nothing in it is implemented yet. +> +> Revision 2, 2026-10-01. It folds in a second review of revision 1; §7 records what changed and +> why. The source was a three-way review of #407 at `dcbc8b82` plus a second opinion on that +> review. +> +> Siblings: [`gittarget-parent-branch.md`](gittarget-parent-branch.md) is the original plan, and +> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) follows this one. + +## 0. Handoff: read this first + +### Where the code is + +- Branch `fix/new-branch-checks-parent` = PR #407. #408 (the `spec.parentBranch` field) is already + merged into it. +- It is checked out in `/workspaces/gitops-reverser2`. Ignore the worktree + `/tmp/gitops-parent-branch`; it holds the stale `feat/gittarget-parent-branch`. +- The tip was `dcbc8b82` when this plan was written. Run `git fetch` and `git log origin/main..HEAD` + before starting. + +### Neighbouring work, out of scope here + +- `todo.md` holds follow-ups from #404, which are unrelated. +- [`gittarget-parent-observation.md`](gittarget-parent-observation.md) is a separate proposal: keep + the parent visible after the write branch exists. Do not implement it here. +- [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) runs after this + pass. + +### Process + +- Follow `AGENTS.md`. Its validation sequence is `task fmt`, `generate`, `manifests`, `vet`, + `lint`, `test`, `test-e2e`. Run the e2e commands sequentially, and check `docker info` first. +- **When to push: the user's standing instruction overrides AGENTS.md's "When to push".** AGENTS.md + says Git-write-path changes wait for a local e2e pass. The user has said, more than once: + "Don't wait for passing of e2e, as soon as linting is green you push", and "never ever keep + these commits unpushed: now we have to wait another 30 minutes for CI". + - So commit and push in one step as soon as `task lint` and `task test` pass. + - Keep local e2e running in the background, or read the CI legs, and fix forward. + - The PR is not ready until e2e is green somewhere. +- Use one commit per work item. Each item's red tests come first: either in their own commit, or + in the same commit with the red-then-green recorded in the PR body. +- At the end, update the #407 PR body. Add one entry per item to its "Review fixes" section. + After each push, read the CodeRabbit inline comments on `pulls/407/comments`; they are not the + same as `gh pr view --comments`. + +### Test harness facts + +- Any claim about server-side compare-and-swap, or about a refused upload, needs + `startRealGitServer` (canonical `git http-backend` over HTTP). The go-git v6 `file://` + transport never compares `cmd.Old`, so such a test passes vacuously over `file://`. Tests that + only assert our client-side rejection may use `file://`. +- `pushAtomicFn` is a package variable, so a test can wrap it. `fetchRemoteBranchHashFn` is one + too. Where §4.1 needs a hook at a specific point, add a nil-by-default package variable and set + it only in tests. +- `TestGitRoundTripLedger` runs against its golden file `internal/git/testdata/git-roundtrip-ledger.golden`. + The golden file counts connections to the Git host for each operation. It is the cost gate + (§1). +- Run controller envtest alone. It binds `:8080`, so a parallel run on the same host fails 21 + unrelated specs. + +**Do not re-raise.** These were fixed in earlier review rounds: + +- the concurrently created write branch, at the advertisement; +- the stale repository identity behind `ParentBranchNotFound`; +- retained writes not rebuilt on a parent change; +- path validity gating parent conflicts; +- the mutability of `parentBranch`, which is now immutable; +- the parent-change flag lost on a failed rebuild (`dcbc8b82`). §4.1 replaces that mechanism, but + its regression test `TestBranchWorker_AParentChangeSurvivesAFailedRebuild` must stay green. + +## 1. The contract this pass must not break + +**API first, not API only.** The design is optimized for a remote that rarely moves under us. The +normal path is: + +> use the trusted checkout → validate it through the push advertisement (a connection the push +> opens anyway) → fetch, reset and replay **only** when that advertisement says the base is +> stale. + +There is **no unconditional preflight fetch on a trusted checkout**. A pull before a push cannot +close the race anyway: the remote can move between any fetch and any push. What closes it is the +compare-and-swap plus the bounded replay, and both already exist. The legitimate fetches are: + +- bootstrap; +- a base that was invalidated, including by a parent-configuration change (§4.1); +- recovery after a detected change, a failed write, or a refused push. + +The guarantee we offer, and the only one: + +- The parent matches the push advertisement. +- The server's compare-and-swap protects the write-branch update. `Old` is the advertised tip, or + zero for creation. +- The parent can still move after the advertisement, and nothing pretends otherwise. + +### Acceptance criteria for every item + +1. **Cost.** Unchanged remotes keep today's request counts. Ledger rows **1–12** must not change. + §2 names the new rows. +2. **Bounded recovery.** A detected change triggers recovery that is bounded in attempts, in + connections, and in how long it waits. +3. **Progress.** Pending work eventually lands **without another cluster edit**, with + `--git-refresh-interval` on or `0`. That includes a recovery attempt that fails transiently + (§4.4). +4. **Loud failure.** A situation we cannot resolve correctly surfaces as a condition reason with + an actionable message. It never produces an orphan branch, or a branch on the wrong parent. + +## 2. Use cases, ranked by how often they happen + +The ranking decides what must stay fast: + +- **Hot paths (H)** have a connection budget that the ledger enforces. +- **Rare paths (R)** may spend a fetch, but each one needs a deterministic test and either a + correct outcome or a clear failure. + +| # | Situation | Expected | Budget (connections) | +|---|---|---|---| +| H1 | Write branch exists, remote unchanged, steady edits | commit and push | 2 per publication, 1 when there is nothing to commit (rows 3–5, unchanged) | +| H2 | Write branch absent (standby), idle | nothing written; the refresher only reads | 1 advertisement per refresh tick, 0 fetches (row 12, plus `TestRefresh_AnAbsentBranchWithAnUnmovedParentCostsOneConnection`) | +| H3 | Standby; a no-op live write (a deployment echo) | the branch stays absent; the request resolves `AlreadyPresent` | 1, the push advertisement; 0 fetches. **New row** | +| H4 | Standby; first real edit; parent unmoved since the checkout | branch created on the parent's tip | 2, 0 fetches (row 11, unchanged) | +| H5 | Standby; parent moved since the checkout | client-side refusal → contention fetch → replay → push | **New row**, pinned for the new-branch case (expected to equal row 6) | +| H6 | Write branch deleted after a merge; next edit | back to standby, recreated from the parent | one replay (existing lifecycle test) | +| H7 | Explicit `spec.parentBranch`, and the parent exists | as H2–H6, against that branch | same budgets | +| R1 | Parent configuration changes on a live worker (all targets replaced before the worker sweep) | the next push not yet admitted uses the new parent (§4.1) | at most one fetch per change | +| R2 | Empty repository gains its first ref while writes are pending | writes rebuilt onto the new default branch and published, or refused per R3 | one replay | +| R3 | Non-empty repository with no resolvable default branch, parent omitted | refuse with an actionable reason; no orphan | probe budget (§4.4) | +| R4 | Configured parent missing, later pushed | recover and publish without another cluster edit | probe budget (§4.4) | +| R5 | Remote default branch switched (parent omitted) | followed at the next discovery; documented | one fetch at that discovery | +| R6 | Write branch created by someone else after our advertisement, with the parent unmoved | the server refuses the upload → replay onto their commit | **its own new row**, measured (§4.7). It is not row 6: row 6 is refused at the advertisement, before any upload | + +**Probe budget.** While a parent is missing or unresolved, the remote is asked at most once per +backoff deadline per worker. That holds however many targets share the worker, and however many +reconciles and refresh ticks occur. §4.4 defines it and tests it. + +## 3. Decisions already taken (do not reopen) + +- **Keep `status.remote.parent`.** Revise the vocabulary rule instead (§4.5). +- **An omitted parent means the default branch resolved at the last discovery** (a fetch, a + refresh, or a probe). + - Publication validates that branch's tip in the push advertisement. The receive-pack + advertisement carries no `HEAD`, so publication cannot see a switch, and we do not add a + lookup before pushes to rediscover it. + - Stable intent is what `spec.parentBranch` is for. +- **The push admission boundary** (§4.1) decides when a configuration change takes effect. + - A change before admission makes that attempt replay. + - A change after admission does not recall the push. If that push created the write branch from + the old parent, the branch now exists, and an existing branch does not follow its parent. + That is consistent with the contract; document it. +- **The e2e contract.** + - e2e asserts ancestry and content: the first commit's parent is B, the files are present, and + the branch stays absent on standby. + - Deterministic worker tests with no refresher prove that publication alone finds a moved + parent. + - e2e does not assert which fetch reason found B (§4.6). +- **Default-branch resolution follows go-git, not a stricter rule of our own** (§4.3). +- **Recovery comes before cost.** §4.4 writes the recovery tests first. If the worker-side probe + grows larger than expected, ship the recovery obligation and its tests with a simpler + controller-side backoff (§4.4, fallback). Never weaken recovery to save fetches. + +## 4. Work items, in order + +### 4.1 Parent changes on a live worker (review items 1–2) → R1 + +#### Problem + +`SetParentBranch` runs on the controller goroutine. It drops base trust once and raises +`parentChangedState`. The event loop can then undo or bypass that: + +- **During a push or fetch.** + - `runPushCycle` calls `setBaseTrusted(true)` after the push returns (`branch_worker.go` + ~2021). `updateBranchMetadataFromPullReport` does the same after a fetch (~2953). Either can + run *after* the drop. + - `recoverRetainedWrites` with nothing retained then swaps the flag to false (~1416), relying on + a distrust that is gone. + - The `newBranchParent` shortcut in `ensureWriteBranch` (~2346) then roots the cycle on + `(main, A)`, and the push is accepted. + - Reproduced by the worker reviewer. +- **Between recovery and publication.** + - `pushPending` (~1656) runs `recoverRetainedWrites`, then `pushPendingCommits`. A change that + lands between the two is pushed on the old root. + - Reproduced. + +#### Design: one configuration snapshot, generation-scoped trust, and one admission boundary + +1. **Configuration snapshot.** + - Replace `parentBranchName` + `parentChangedState` with one atomic pointer to an immutable + `parentConfig{name string; gen uint64}`. + - `SetParentBranch` installs a new value, with `gen+1`, only when the name changes. It keeps + its `invalidateBase` as a belt. + - Every operation that consults the parent (fetch/sync, refresher advertisement, the §4.4 + probe, the rebuild) loads the snapshot **once, at its start**, and uses that `{name, gen}` + throughout. + - The snapshot also goes into the observation that operation records: `ParentRequested` (see + §4.3) and the generation. + - Nothing re-reads the current name after the network call returns. That way an old lookup can + never report a result about a newly configured parent. +2. **Generation-scoped trust.** + - The worker keeps `baseParentGen` (under `repoMu`): the generation of the snapshot whose fetch + produced the checkout. + - Trust means `baseTrustedState && baseParentGen == current gen`, so `setBaseTrusted(true)` + after an old-generation push or fetch can never vouch for the current configuration. + - `newBranchParent` is honored only when `baseParentGen` is current. + - The root's generation, `pushCycleRootGen`, is stamped wherever `pushCycleRootBranch` is set: + in `ensureWriteBranch` and in the rebuild. +3. **Retained writes.** + - In `recoverRetainedWrites`, "parent changed" becomes `pushCycleRootGen != current gen`. + - A failed rebuild leaves the generations mismatched, so the next attempt rebuilds again. This + replaces the `dcbc8b82` hand-back. +4. **The admission boundary.** One local point in `runPushCycle`, under `repoMu`, immediately + before each `pushAtomicFn` call, including the retries: + - load the snapshot and compare its generation with `pushCycleRootGen`; + - **on mismatch, do not push.** Take the existing replay branch: `markReplayRequired`, sync + with the new snapshot, rebuild, re-stamp, loop. This adds no extra connection; + - on match, the attempt is **admitted**. Any change from here on is in flight, per §3. + + Add a test-only hook, `beforePushAdmission`, called just *before* the comparison. A change + injected there is a "before" case. A change injected inside the wrapped `pushAtomicFn` is an + "after" case. +5. **An existing write branch is unaffected.** + - When the cycle root is the write branch itself, and it was proven present, a parent change + cannot alter the plan. + - Re-stamp the generation without fetching. The push's compare-and-swap still guards the + branch. + +#### Tests (red first; run with `-race` too) + +1. **Change during a fetch.** + - Inject `SetParentBranch("release")` inside the sync of an initial fetch and of a contention + fetch. + - Assert that `feature`'s first commit sits on `release`'s tip. + - Assert that the observation recorded for that fetch names the parent it actually used + (`main`), not `release`. +2. **Change during a no-op push** that settles as `PushNoBranch`. This is the reviewer's + reproduction, an "after" case. Assert that the next real write lands on `release`. +3. **Change before admission** (the `beforePushAdmission` hook) in a cycle with retained writes. + - Assert that no push leaves on the old root. + - Assert that the replay builds on `release`. +4. **Change after admission** (inside the wrapped `pushAtomicFn`), in a cycle that creates the + branch. + - Assert that the branch is created from `main`, which is the documented in-flight rule. + - Assert that later writes go to the existing branch, with no attempt to re-root it. +5. **The new parent is missing**, in cases 1 and 3. + - Assert that `feature` is never created. + - Assert that the work stays retained. + - Assert that §4.4's recovery picks it up. +6. **An existing write branch, then a parent change.** + - Assert zero extra connections. + - Assert that the next push goes to the write branch. +7. `TestBranchWorker_AParentChangeSurvivesAFailedRebuild` and + `TestBranchWorker_AParentChangeRebuildsRetainedWrites` stay green. + +**Cost.** Everything added here is local. The ledger is unchanged. + +### 4.2 An empty repository gains a ref (review item 3) → R2 + +#### Problem + +Cloning an empty repository leaves `HEAD` unborn on the write branch, with the cycle root +`(feature, zero)`. If somebody then pushes `main`, neither check fires: + +- `validatePushState`'s missing-root check needs a non-zero `rootHash`. +- `remoteMovedSinceBase` compares `refs[feature]`, which is absent and therefore zero, with the + zero root. + +So the push creates `feature` as an orphan (`git_atomic_push.go` ~192–259). Reproduced on the real +server. + +**Defining "empty"**, once, and using it in discovery, publication and the observation: + +- A repository is empty when its advertisement carries **no hash refs at all**: no branches, no + tags, nothing else. +- A symbolic `HEAD` pointing at an unborn branch does not count. go-git reports that as + `RemoteRefs.Unborn`. +- A repository with only tags is **not empty**. + +#### Design + +- **A new typed error, `RepositoryNotEmptyError{Refs int}`, in `git_atomic_push.go`.** Do not + reuse `RemoteMovedError`: its `Branch` and `Advertised` fields mean "the root, compared", and + filling them with another branch's hash would make them lie. + - `validatePushState` returns it when `rootHash.IsZero()`, the write branch is not advertised, + and the advertisement is not empty by the definition above. + - The check reads the advertisement the push already holds, so it costs zero extra connections. +- **`remoteMovedDuringPush` classifies it directly as contention**, without the fallback probe. The + existing replay then runs: + 1. fetch, which resolves the default branch through §4.3, or refuses; + 2. rebuild, which **re-plans** from the retained writes rather than reusing the orphan commits; + 3. push. +- **Tags only:** discovery finds a non-empty repository with no resolvable default branch. It + refuses per §4.3, keeps the work retained, and §4.4 takes over. + +#### Tests (red first; real git server) + +1. **Empty remote with retained work, then `main` is pushed externally.** Start from an empty + remote holding a retained live write and a `CommitRequest`. Push `main` from outside before + publication. Assert: + - `feature`'s first commit has `main`'s tip as its parent; + - the content is present; + - `git merge-base feature main` is non-empty; + - exactly one contention fetch happened; + - the request resolves `Committed`. +2. **The same with a no-op write.** The branch stays absent, and nothing is orphaned. +3. **Empty remote, then only a tag is pushed.** Assert: + - no orphan is created; + - the work stays retained; + - the observation is `Missing` per §4.3; + - recovery happens once a branch with a resolvable `HEAD` appears. +4. **Empty remote throughout.** Ledger row 1 is unchanged, and the first push still creates the + root commit. + +### 4.3 Empty versus unresolved default branch (review item 4) → R3 + +#### Problem + +`SmartFetchFrom` returns `""`, the "empty repository" result, in two situations: + +- when the advertisement is empty (`git_smart_fetch.go` ~62); +- when refs exist but no default branch resolves and no parent is configured (~77–83, via + `analyzeRemoteRefs`, which only logs "Remote HEAD is broken"). + +The caller then runs `makeHeadUnborn`, and the result is an orphan. The surrounding status is +wrong in the same way: + +- the observation says `Unborn`; +- `recordAdvertisement` in `refresh.go` (~260) says the same; +- `parentBranchReadiness` treats every omitted parent as fine (`gittarget_controller.go` ~823). + +#### What the evidence looks like after go-git (verified at v6.0.0-alpha.5) + +`Remote.List` does not hand over the raw advertisement. A hash-only `HEAD`, from a server without +the `symref` capability or from a detached `HEAD`, is rewritten **before** `List` returns. +`packp.ResolveHeadFromHashHeuristic` turns it into a symbolic `HEAD`: + +- pointing to `master` if `master` has that hash; +- otherwise to the alphabetically first branch at that hash. + +That is git's own spirit too: `guess_remote_head` in `remote.c` prefers fallbacks over requiring a +unique match. So a stricter "exactly one branch matches" rule cannot be built on `Remote.List` +without dropping beneath it. + +**Decision: adopt go-git's resolution.** This is a deliberate choice, not an oversight. + +- Bypassing `Remote.List` to keep the raw evidence would mean a bespoke advertisement path for a + rare server configuration. +- The resolved branch is visible in `status.remote.parent.branch`. +- Anyone who needs certainty sets `spec.parentBranch`. + +The rules are then: + +- **`Unborn`**: the repository is empty by the §4.2 definition, and only then. +- **Resolved**: after `List`, `HEAD` is symbolic and its target is an advertised branch. This + covers go-git's heuristic. +- **Unresolved**: the repository is not empty, but `HEAD` is absent, symbolic to a missing branch, + or still a bare hash with no matching branch. + - Return a new sentinel, `ErrDefaultBranchUnresolved`. + - Handle it exactly like `ErrParentBranchNotFound` on every path: fetch, publication, refresh + and the §4.4 probe. + - The observation is `ParentState=Missing`. `ParentBranch` is the dangling `HEAD` target (for + example `master`), or empty when there is none. + - No write is planned. The work is retained, or the dropped writes are remembered (§4.4). + +#### Make an omitted parent able to stall + +- Add `ParentRequested` to `RemoteObservation`: the spec value, `""` when omitted, taken from the + §4.1 snapshot. +- `parentBranchReadiness` compares `ParentRequested` with `spec.parentBranch`, instead of the + resolved name, so an omitted parent can stall too. +- Use the same reason, `ParentBranchNotFound`, so that recovery stays one mechanism. Only the + message differs: + - "the remote's default branch (`HEAD` → `master`) does not exist; fix the remote's HEAD or set + `spec.parentBranch`"; + - or, with no `HEAD` at all: "the remote has no default branch". + +Update the documentation in three places: + +- the doc comment of `ParentUnborn` in `remote_observation.go`; +- the API types; +- the parent-branch contract table in `configuration.md`. That table should also record the + go-git resolution decision. + +**Tests**, all through the transport, against the real git server, not on synthetic ref lists: + +1. **`HEAD` symbolic to a missing branch.** Seed a bare repository with + `git symbolic-ref HEAD refs/heads/master` and only a `main` branch. Assert: + - the init observation and the refresh observation are both `Missing` with branch `master`; + - a write produces no orphan and stays retained; + - readiness is `ParentBranchNotFound` with the `HEAD` message. +2. **Detached `HEAD`.** Use `git update-ref --no-deref HEAD ` with two branches at that + hash. Assert that the resolution is the one go-git chooses (`master` if present, otherwise + the first alphabetically), and that the status names it. This pins the decision above. +3. **Recovery.** Repair the remote's `HEAD`, then recover through §4.4 with no new cluster edit. +4. **A truly empty remote** is still `Unborn`, and ledger row 1 is unchanged. + +### 4.4 Recovery from a missing or unresolved parent (review item 6) → R3, R4 + +#### Problem today + +- While `ParentBranchNotFound` holds, every reconcile sets `forceRecheck` + (`gittarget_controller.go` ~189, ~275). A target that is not Ready requeues every 10s. +- Each forced recheck restarts all of the target's streams. That means: + - a re-LIST of every watched type, which is the real cost; + - a replay that spends one advertisement connection. +- A typo such as `parentBranch: relase` keeps this up indefinitely. + +#### Why "force only on the transition, let the refresher probe" strands work + +- The refresher skips a branch that is mid-cycle (`refresh.go` ~101): retained writes, an open + window, a dirty worktree, or a required replay. +- A failed `pushPending` stops its push timer (~1673), and nothing re-arms it until the next + event. +- With `--git-refresh-interval=0` there is no refresher at all. +- An observation flip is not an obligation: + - If the first recovery attempt after `Found` fails transiently, the observation stays `Found`, + the timer is stopped, a probe gated on `Missing` no longer runs, and the work is stranded. + - A controller that keys off "the stored Ready reason was `ParentBranchNotFound`" can miss a + whole missing→found episode that happens between two of its reconciles. + +#### Design + +1. **A recovery obligation on the worker.** Its lifetime is decoupled from observations. + - The worker latches `parentRecovery` the moment a cycle fails with `ErrParentBranchNotFound` + or `ErrDefaultBranchUnresolved`. + - The latch has two independent halves: + - **`retained`**: set when writes were kept. It clears only when `pushPending` actually + publishes them, or settles them as `PushNoBranch`. + - **`snapshotNeeded`**: set when the worker **dropped** writes, or refused a resync, because + of the missing parent. Before implementing, verify which paths drop and which retain: + `noteMissingParent` at ~2425 and its callers. It clears only when a resync snapshot cycle + completes successfully *after* the parent was found. + - Neither half clears on an observation, on a probe result, or on a failed attempt. A failed + attempt re-arms the probe deadline (step 3). +2. **The worker drives recovery.** It runs on the event loop, while the latch is set, whatever the + refresh configuration and whether or not writes are retained. + - When the probe deadline fires, read one advertisement through `advertiseRemoteBranch`, with + the §4.1 snapshot. This is a read, not a fetch. + - If the parent is still missing or unresolved, record that and back off. + - If the parent (or the write branch) is now present, record it. Then: + - with `retained` set, call `pushPending`. It recovers through the normal replay: one fetch, + a rebuild, a push. A failure leaves the latch set and re-arms the deadline; + - with `snapshotNeeded` set, raise a **snapshot request**. That is a monotonically increasing + `snapshotRequestSeq` that the worker exposes alongside its observation. A failed snapshot + cycle raises it again on the next deadline. +3. **One probe deadline per worker, shared by every background probe.** + - The worker keeps `nextRemoteProbeAt`, with backoff 10s, doubling, capped at 5m. It resets on + a generation change (§4.1) and on success. + - While the latch is set, a **refresher tick** that arrives before the deadline is a no-op: no + connection. A tick after it counts as the probe and advances the deadline. + - **Reconciles never probe.** That holds however many targets share the worker. + - Live events, while the latch is set and before the deadline, are retained or dropped as + today, **without** a fetch. The deadline governs them too. +4. **The controller acts on the request sequence, not on the Ready reason.** + - It remembers, in memory per target (like `reconcileRequests.take`), the last + `snapshotRequestSeq` it acted on. + - When the worker's sequence is higher, it forces **one** recheck and records the sequence. A + missing→found episode between two reconciles is therefore never lost. + - After a controller restart the memory is empty. That is harmless: every stream starts fresh + with an initial list, which is itself the snapshot. + - Drop the per-reconcile `parentWasMissing` force. + - Ready reporting: + - `ParentBranchNotFound` stays Ready=False/Stalled while the parent is missing. + - While the obligation is open but the parent has been found, report Ready=False with + `Reconciling=True`, reason `RecoveringParentBranch`. This keeps the not-Ready requeue + cadence, so the controller sees the sequence within one requeue, and the state stays + visible. Verify that the not-Ready requeue applies to that reason. + - The 10s requeues read local observations only, so they cost zero connections. + +#### Tests + +Write the recovery tests first, against current code. They are the safety net for the design. + +1. **Worker, retained writes, real server.** The parent is missing while writes are retained. + Then: + - the parent is pushed externally; + - the **first recovery attempt fails** (inject a GitProvider read failure, as the reproduction + for `dcbc8b82` did); + - the next deadline recovers and publishes, **without another cluster edit**. + + Run it with refresh enabled and with refresh at `0`. +2. **Worker, dropped writes.** Assert that `snapshotNeeded` latches, that the sequence rises when + the parent appears, that it rises again after a failed snapshot cycle, and that the latch + clears only after a successful one. +3. **Controller unit.** A sequence bump forces exactly one recheck. A bump that happens entirely + between two reconciles is still acted on. Ten reconciles at an unchanged sequence force + nothing. +4. **Probe budget.** Take three targets sharing one worker, a parent that stays missing, refresh + ticks every few seconds, and repeated reconciles, over a simulated window of N backoff + periods. Assert that the advertisement connections stay ≤ the backoff schedule's count, and + that there are zero fetches. + + This needs an injectable clock on the probe deadline. Add a small `now func() time.Time` field + rather than sleeping. +5. **The §4.3 cases.** An unresolved `HEAD`, once repaired, recovers through the same path. +6. **e2e: dropped-write integration.** There is no controller+watch+worker harness outside e2e, + and building one is out of scope; e2e is that integration test. Add a third context to + `new_branch_parent_e2e_test.go`: + - set `parentBranch: release` while `release` is absent; + - assert `ParentBranchNotFound` and that no write branch exists; + - push `release` from outside; + - assert Ready, and the content on a new branch whose first commit's parent is `release`'s + tip; + - make no cluster edit after the push. + + This also covers the controller reviewer's item 4. +7. **Ledger rows.** Add "parent probe while missing" = 1 connection, 0 fetches. Row-level counts + do not prove frequency; test 4 does. + +#### Fallback, if step 2 or 3 grows too large + +- Keep step 1 (the obligation) and step 4 (the request sequence). The obligation is the + correctness part and is non-negotiable. +- Replace the worker probe with a controller-side forced recheck under exponential backoff + (`lastForcedAt` per target, 10s → 5m), still gated on the latch rather than on the Ready + reason. +- Tests 1–3 and 6 still apply. Test 4 then asserts the controller-side schedule. +- Record the worker probe as deferred work in the PR body, and adjust §5's memory note. + +### 4.5 Vocabulary (review item 5) + +Keep `status.remote.parent`. Reword the new rule in `docs/definitions.md` (~132): + +> "parent branch"; a bare "parent" is fine where the subject is already a branch, such as the +> `status.remote.parent` stanza or a sentence that has just named the parent branch. Use the full +> term wherever a parent *resource* could be meant. + +Then sweep `configuration.md`, `architecture.md` and the Go doc comments, but only for genuinely +ambiguous uses. + +### 4.6 e2e: no dependence on refresh timing (review item 7; CodeRabbit comment on `:238`) + +The last step, "the publication check, not the refresher, is what found B", asserts both that +`contention` rose and that `refresh` did not change. + +- With `--git-refresh-interval=30s`, a refresh can find B first. `contention` then legitimately + never rises. +- Relaxing only the `refresh` equality does not fix this. + +#### The fix + +1. Delete that step, and keep the ancestry and content assertions. +2. Add a comment pointing at the deterministic worker tests that prove publication alone finds B, + with no refresher: + - `TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip`; + - `TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent`. +3. Reply on the CodeRabbit thread with that rationale. + +### 4.7 Default-branch switches, and the refused upload (review item 8) → R5, R6 + +#### R5: make the decision in §3 true, and stop the status flapping + +- Today `followParent` (`refresh.go` ~276) compares hashes only. Make it also compare + `advertisement.parentBranch` with the branch the checkout was rooted on: `newBranchParent`, or + the branch the last sync resolved. +- On a mismatch, call `syncWithRemote(fetchReasonRefresh)`. That is one fetch, on a rare event. +- This also ends the flapping of `status.remote.parent.branch`, where no-op pushes record `main` + while refreshes record `trunk`. +- **Tests.** + - Switch `HEAD` from `main` to `trunk`, both at the same commit. Assert that a refresh resyncs + once, and that the next no-op push records `trunk`. + - With refresh at `0`, assert that the worker keeps `main` until the next discovery. That is the + documented behavior. +- **Docs.** Document the omitted-parent semantics in `configuration.md`. + +#### R6 needs a production fix, not only a test + +How it fails today: + +- `feature` is created by somebody else after our advertisement, and the parent `main` is + unmoved. +- Our upload with `Old=zero` is refused by the server, as an untyped error. +- `remoteMovedDuringPush` (~2084) finds no typed rejection. It falls back to + `fetchRemoteBranchHashFn` for the **root** branch, `main` (~2100). +- That probe sees `main` unmoved and concludes "not moved". There is no replay, and the work is + stuck. + +The fix: + +- Only on this exceptional path: when the cycle creates the branch (`rootBranch` is not the write + branch) and the failure is untyped, make the fallback discovery read **both** refs from one + advertisement. +- If the write branch is now advertised, treat it as contention, as the typed write-branch case + already does. +- Do not add any discovery to the normal path. + +**Tests** (real server; add a test-only seam between `validatePushState` and `performPush`): + +- Create `feature` from outside inside that seam. +- Assert that the server refuses the upload. +- Assert that the worker replays onto their commit, and that their commit is an ancestor of ours. + +**Ledger.** Add R6 as its own measured row. It includes a refused upload plus the fallback +discovery, so it will not equal row 6. Record whatever it measures, check that the number is +explained by the steps above, and do not tune toward row 6. + +## 5. Finish + +1. Regenerate the ledger golden file (`-update`) and **diff it by hand**. + - Rows 1–12 must be byte-identical. + - Only the new rows may appear: H3, H5, the parent probe, and R6. +2. Run `task lint` and `task test`, then commit and push per item (§0). +3. Update the #407 PR body: + - one "Review fixes" entry per item; + - the cost table from §2; + - the e2e contract and the go-git resolution decision from §3 and §4.3. +4. Read the CodeRabbit inline comments again after each push. +5. Run `task test-e2e`, and the `full-manager` leg in CI. §4.4's new e2e context must pass at least + once before calling the PR ready. +6. Update the memory note `gittarget-parent-branch-design` with what actually shipped: + - `parentConfig{name, gen}`, generation-scoped trust, and the push admission boundary replace + `parentChangedState`; + - `Unborn` means an advertisement with no hash refs; + - the default branch follows go-git's resolution; + - the recovery obligation, with `retained` and `snapshotNeeded`, and `snapshotRequestSeq`; + - **either** "a worker-side probe with one shared deadline" **or**, if the §4.4 fallback + shipped, "a controller-side backoff; the worker probe is deferred". + +## 6. Out of scope + +- Keeping the parent visible after the write branch exists. See + `docs/design/gittarget-parent-observation.md`. +- Resetting a surviving write branch onto its parent, and PR management. +- Any fetch before publication on a trusted checkout. + +## 7. Changes from revision 1, and where they push back + +Each point of the second review was checked against the code before it was accepted: + +1. **Recovery must stay pending until it succeeds.** Accepted. §4.4 now has the latch with + `retained` and `snapshotNeeded`, and the `snapshotRequestSeq` handshake. Test 1 injects a + failed first recovery. + - **Partly pushed back:** the dropped-write integration is the new e2e context, not a new + controller+watch+worker harness. None exists outside e2e, and building one would be larger + than the bug. The worker and controller halves get their own unit tests (tests 2 and 3). +2. **The generation check and the race test disagreed.** Accepted. §4.1 now defines a + `{name, gen}` snapshot loaded once per operation, an explicit admission boundary, and the + `beforePushAdmission` hook. Tests 3 and 4 sit on either side of the boundary. +3. **R6 needs a production fix.** Accepted, and verified: the fallback probes the root only + (`branch_worker.go` ~2100). R6 has its own measured ledger row. +4. **One probe deadline for all probes.** Accepted. One `nextRemoteProbeAt` per worker; refresher + ticks before it are no-ops; reconciles never probe. Test 4 counts connections over time, not + per row. +5. **Resolver evidence.** Verified: go-git rewrites a hash-only `HEAD` before `List` returns. + - **Pushed back on the conservative option:** the plan adopts go-git's resolution, which + follows git's fallback spirit, instead of going below `Remote.List` for a rare server + setup. + - The tests run through the real transport, including a detached `HEAD`, to pin the + decision. The "the way `git clone` does" wording is gone. +6. **Truthful evidence for "no longer empty", and tags.** Accepted. There is a new + `RepositoryNotEmptyError`, and "empty" is defined once (no hash refs). A repository with only + tags is non-empty and goes through §4.3's refusal. + +Smaller corrections: + +- ledger rows 1–12 instead of 1–10; +- "no unconditional preflight fetch on a trusted checkout"; +- the push instruction is quoted inline, because a fresh session cannot read the memory note; +- the memory update depends on which §4.4 variant shipped; +- H4 maps to the existing row 11, so it needs no new row. diff --git a/docs/design/gittarget-parent-observation.md b/docs/design/gittarget-parent-observation.md new file mode 100644 index 00000000..58ef20dd --- /dev/null +++ b/docs/design/gittarget-parent-observation.md @@ -0,0 +1,251 @@ +# GitTarget parent observations with shallow history + +> **design**: proposed follow-up to PR #407; nothing in this document is implemented. +> Date: 2026-10-01. + +Keep the parent branch visible after the write branch exists. Reuse remote ref advertisements to +report its current tip, then use bounded local history checks to describe the relationship where +the evidence allows it. An incomplete history produces an explicit `Unknown` result. + +The cost boundary is part of the design: no extra remote connection, history fetch, or deepening +for this status feature. Parent observations are informational. Existing publication checks still +decide whether a write can proceed. + +## Why keep observing the parent + +An operator needs to see whether an existing write branch still includes the parent's current +tip. The parent can advance, rewind, disappear, or acquire unrelated history while the write +branch remains active. Dropping the parent from status removes that context precisely when the +branches can start to differ. + +PR #407's [API types](../../api/v1alpha3/gittarget_types.go) describe `status.remote.parent` only +while the write branch is absent. Its `Found`, `Missing`, and `Unborn` states explain where a new +branch would start. This proposal retains that observation after creation and adds a separate +ancestry result. + +The parent remains the branch selected by `spec.parentBranch`, or the advertised remote default +branch when that field is omitted. This describes the current comparison target. Git does not +record a branch's historical creation parent, and this status makes no such claim. A change to +the remote default branch changes the comparison target, even if its tip hash stays the same. + +## What the existing operations can tell us + +The [remote refresher](../../internal/git/refresh.go) already reads the write branch and parent +from one ref advertisement. Keeping both observations adds no object transfer. The advertisement +still has network and ref-listing costs, so reuse the existing refresh cadence and write-related +observations rather than adding another poll. + +Two advertised hashes prove equality or inequality. They provide no history explaining the +difference. A changed parent hash can mean an advance, a rewind, or a force push. Calling all three +`ParentAhead` would give operators the wrong information. + +[`SmartFetchFrom`](../../internal/git/git_smart_fetch.go) fetches with `Depth: 1`. That limits +downloaded history, but local commits can extend the fetched tip: + +```text +older history unavailable <- A <- W1 <- W2 + ^ ^ + parent tip write tip +``` + +If the objects from `W2` back to `A` remain available, their parent links prove that the write tip +includes `A`. There is no need to fetch the history before `A` for that answer. + +There is a second boundary: Git's normal revision walk treats commits listed in `.git/shallow` +as roots, even though their commit objects retain parent links. See +[Git's shallow repository documentation](https://git-scm.com/docs/shallow). An exploratory check +with system Git 2.39.5 found that fetching the pushed write tip again with `--depth=1` shortened +the visible log to that tip, while an earlier base object remained readable with `git cat-file`. + +This suggests inspecting locally stored commit objects directly for positive ancestry evidence. +The experiment does not establish the behavior of the repository's `go-git/v6` dependency; that +needs a focused test before implementation. Object retention is also optional: a new checkout, +repository replacement, or cleanup can remove evidence that a long-lived worker once had. + +## Proposed status contract + +Parent availability and ancestry answer different questions and use separate fields. + +### Parent availability + +Retain `status.remote.parent` after the write branch exists. Its existing fields continue to name +the selected parent and its last observed tip, with a parent-specific observation timestamp. + +| State | Meaning | +|---|---| +| `Found` | The selected parent branch was advertised with a commit. | +| `Missing` | The selected parent branch was absent from a successful advertisement. | +| `Unborn` | The repository was empty and no explicit parent was configured. | + +`Missing` blocks creation only while the write branch is absent, preserving PR #407's behavior +for an explicit parent. An existing write branch stays writable when its parent disappears. +`Unborn` must not become a catch-all for a nonempty repository with an unresolved default branch. +That case needs an explicit observation reason and no ancestry comparison. + +A failed advertisement does not prove that a branch is missing. Keep the last successful +observation and its timestamp; existing remote-error reporting describes the failed attempt. +Before the first successful observation, leave the parent observation absent. + +### Ancestry relationship + +Compare the advertised remote write tip with the advertised parent tip. Local `HEAD` can include +unpublished commits and must not substitute for the remote write tip. + +The recommended first implementation has four outcomes: + +| Relationship | Required evidence | +|---|---| +| `SameTip` | Both advertised hashes are equal. | +| `WriteAhead` | Distinct tips; a parent-link path leads from the write tip to the parent tip. | +| `ParentAhead` | Distinct tips; a parent-link path leads from the parent tip to the write tip. | +| `Unknown` | Available evidence and the work budget do not establish one of the above. | + +These values describe ancestry, not elapsed time or the number of commits. A parent rewind can +produce `WriteAhead`. Following a merge's second parent can also establish ancestry. Commit author +names and timestamps provide no substitute for parent links. + +`Unknown` carries a reason, such as `HistoryIncomplete`, `BudgetExceeded`, or +`RelationshipUnclassified`. The last reason covers a completed local check whose result falls +outside the initial enum. An object read failure also leaves the relationship unknown and must +retain a diagnostic; it cannot establish a negative ancestry result. + +When either branch has no observed tip, omit the comparison. The parent availability and write +branch observation already explain why comparing two commits is inapplicable. + +### Bind the result to the evidence + +Store the exact compared hashes and the time of their joint remote observation with the result. +An illustrative layout is below; field names remain subject to API review. Hashes are shortened +for readability, but status stores complete object IDs. + +```yaml +status: + remote: + commit: "b4e29d1" + lastVerifiedAt: "2026-10-01T12:00:00Z" + verifiedBy: Fetch + parent: + state: Found + branch: main + commit: "a73c012" + lastVerifiedAt: "2026-10-01T12:00:00Z" + comparison: + relationship: WriteAhead + writeCommit: "b4e29d1" + parentCommit: "a73c012" + observedAt: "2026-10-01T12:00:00Z" +``` + +A successful push verifies the new write tip. It does not by itself verify the parent's tip at +that same time. Preserve the parent's own observation time and clear the comparison when its +inputs no longer match. The next suitable advertisement can establish a new comparison. Never +refresh the parent timestamp solely because `status.remote.lastVerifiedAt` changed. + +## Bounded local comparison + +Run comparison as observation work, with finite limits on visited commits, queued parent edges, +and elapsed time. Support cancellation and serialize object reads with repository replacement or +cleanup. Choose internal limits using measurements before implementation; this proposal adds no +operator configuration for graph-walk budgets. + +1. Resolve both branch names and hashes from the same successful advertisement. If either tip + is absent, publish availability and omit the comparison. +2. Return `SameTip` immediately for equal hashes. No commit objects are needed. +3. Look for a cached result for this repository identity, resolved branch pair, and exact tip pair. +4. Walk locally available parent links from the write tip, looking for the parent hash. A found + path proves `WriteAhead`. Check the reverse direction within the same total budget for + `ParentAhead` when needed. +5. Stop on proof or budget exhaustion. A missing object leaves that path incomplete; other queued + paths may still prove ancestry. If neither direction supplies proof, publish `Unknown` and + the reason for the limit on the result. + +Bound the cache as well as the walk. Successful proofs for an unchanged pair can be reused after +a fresh advertisement. Cache unknown results only while the local object store has unchanged +evidence: a normal fetch may make the same pair answerable. Budget-limited results should not +start an immediate retry loop. Discard work whose repository or observed tips changed before the +result was published, and invalidate the cache when the repository identity changes. + +The comparison must not extend the write critical path without a bound or block readiness. +Continue existing status-write throttling, while publishing changed facts promptly. With +`--git-refresh-interval=0`, observations age until another existing operation reads the refs; +this feature does not introduce a replacement timer. + +## Why a missing overlap remains unknown + +A bounded search that finds no shared commit cannot distinguish missing history from histories +that never intersected. Even finding a shared commit does not by itself prove divergence: one tip +could still be an ancestor of the other along an unexplored merge parent. + +Two additional classifications are useful but deferred from the first implementation: + +| Relationship | Proof required before reporting it | +|---|---| +| `Diverged` | A common ancestor exists, and neither tip is an ancestor of the other. | +| `Unrelated` | Both complete ancestor graphs have been explored with no common commit. | + +Negative proofs require exploring every relevant parent edge or an equivalent sound proof. An +unresolved shallow boundary, missing object, or exhausted budget prevents that conclusion. Neither +outcome justifies downloading more history solely to fill status. + +There is also no requirement to publish the first overlapping commit. Git's graph can contain +multiple best common ancestors, so the first intersection found by a walk is not a reliable +merge-base contract. See [Git's merge-base documentation](https://git-scm.com/docs/git-merge-base). +The initial status needs only the relationship evidence above. + +## A cheaper alternative: compare with a remembered starting tip + +A worker could remember the verified parent hash used when it creates or rebuilds an absent write +branch, then compare later parent advertisements with that hash. This needs no graph walk: +`Unchanged`, `Changed`, or `Unknown` would describe movement relative to that recorded starting tip. + +This is a different question from ancestry. `Changed` cannot say whether the parent advanced, +rewound, or was replaced, nor whether the write branch later incorporated its new tip. The saved +hash also needs provenance tied to the repository and the particular branch creation. External +branch replacement, deletion and recreation, or lost worker state can invalidate that provenance. + +Keep this as an alternative if bounded local ancestry proves too costly or rarely informative. +Do not add both models in the first implementation. An in-memory starting tip is lost on restart; +durable origin tracking would need a separate design and must not be reconstructed by guessing. + +## Operational boundaries + +Parent comparison is status only. It does not merge, rebase, reset, delete, or recreate an existing +write branch. `Unknown` and a missing parent after branch creation do not change `Ready`. +The publication checks and replay behavior from PR #407 continue to protect creation of an absent +write branch independently of these observations. + +Exact ahead/behind counts, pull request management, and content-equivalence checks are outside +this proposal. A squash merge can integrate the same changes while preserving distinct ancestry; +this status must not claim that a branch is merged or that its changes remain unapplied. + +Extending the lifetime of `status.remote.parent` changes its documented meaning. Implementation +must update API comments, generated CRD descriptions, and user documentation together. Consumers +must use the remote write commit to determine branch presence; parent visibility no longer +implies write-branch absence. + +## Implementation and validation + +Start by testing the shallow-object behavior with the pinned `go-git/v6` version. Then retain +parent observations through the existing refresh path, add the bounded comparison, and project +the evidence into status. Keep changes on a follow-up branch to PR #407. + +The implementation needs these cases: + +- Equal advertised tips with no local objects; an absent write branch; an empty repository. +- Parent disappearance with and without an existing write branch; failed advertisements that + preserve the last successful observation; an unresolved default branch in a nonempty repository. +- A shallow base followed by local commits, including another depth-one fetch after publication. +- Restart with only the two tips available, yielding `Unknown` when intermediate objects are absent. +- Positive ancestry through a merge's second parent; no false divergence at a missing object. +- Both ancestry directions, a parent rewind, a force push, and a parent branch rename or default + branch change, including different names that resolve to the same hash. +- Count, memory, time, and cancellation limits; an unchanged tip pair becoming answerable after a + normal fetch; cache invalidation after repository replacement. +- A local unpublished commit and a later successful push, proving that comparisons never silently + switch from remote evidence to local `HEAD` or acquire an unearned freshness timestamp. +- Remote call and fetch counts showing no additional network work, unchanged readiness, and no + status-driven Git mutations with periodic refresh enabled or disabled. + +Before implementation, settle the exact field names, the representation of an unresolved default +branch, and measured internal work limits. Those choices must preserve the cost boundary and the +distinction between an unavailable answer and a proved relationship. diff --git a/todo.md b/todo.md new file mode 100644 index 00000000..56cecfd9 --- /dev/null +++ b/todo.md @@ -0,0 +1,57 @@ +# TODO: follow-ups from the #404 review (commit window surface) + +PR #404 (`feat/commit-window-surface`) is merged; these need a follow-up PR against main. + +## Worth fixing + +- [ ] **The controller's safety timeout shrank from 420s to about 124s.** + - Where: `internal/controller/commitrequest_controller.go` `resolveTimeout()`. + - Today it is `attachTimeout + maxDuration + 120s`, using the request's own values and counted from + `CreationTimestamp`. + - The design (`docs/design/commit-timing-surface.md`, "safety timeout") says it covers the *largest* `attachTimeout` + and `maxDuration`, plus the push cooldown and retries. + - Failure: if the remote is down for more than about 2 minutes, or the save waits in `WaitingForWorker`, the save is + marked `FinalizeFailed` / `Stalled`. The worker keeps the attach, so the commit (or a `CommitEmpty` record) still + lands, and status contradicts Git. + - The same happens to a request that is older than the bound when the controller first sees it, for example after a + restart. + - Fix: bound it by the schema maximums (5m + 5m + allowance), or start the clock at the worker's registration and + withdraw the attach when failing closed. +- [ ] **A suspended target makes a `CommitEmpty` save report Ready=True with no commit.** + - Where: `commit_request_attach_loop.go` `recordCommitRequest` returns `false, nil` on `errTargetSuspended`, and + `commit_executor.go` `recordEmpty` requires `!target.Suspend`. + - The design table says "target is suspended → the failure". An attached window that changed nothing resolves + `AlreadyPresent` the same way. + - Decide: make the save fail, or change the design table. +- [ ] **A `CommitEmpty` save is recorded while the render-fidelity gate is closed.** (Found by reading the code, not + reproduced.) + - Where: `recordCommitRequest` never calls `normalWritesAllowed`. + - While the gate is closed, the author's events are dropped. The save then times out and records "no writes were + seen", which is false. + - The attached-window path fails the request in the same situation. + +## Low + +- [ ] `expireWaitingCommitRequests` ranges over the `pendingCRs` map, so empty commits for saves that time out in the + same pass land in random order. Sort them by `seq`. +- [ ] `buildRequestRecordWrite` sets `RequestAuthor: UserInfo{Username: pcr.author}`, which loses the display name and + email that a window commit carries. Carry the full `UserInfo` on the attach. +- [ ] A record commit skips `requestTemplate`. The doc comment says it is phrased like every other commit. Fix the + code or the comment. +- [ ] A request whose GitTarget never starts a worker fails after the timeout with the generic "did not resolve within + the safety window". If the last phase was `WaitingForWorker`, say so in the message. +- [ ] `docs/UPGRADING.md` (the v1alpha3 window entry): + - Step 1: GitOps-managed GitTargets need their Flux or Argo source suspended (or the field removed from it) until + step 4. Otherwise the string `commit.window` is re-applied and breaks LIST. + - The Was/Is table needs a row for the chart value `quickstart.gitTarget.commit.window`, which is now an object. + - "A removed `closeDelay` is not a hazard" is too strong. It is pruned silently for programmatic clients (see + `duration_fields_admission_test.go`). +- [ ] Descriptions that still say a save "closes the window now": + - the CommitRequest godoc and its CRD description + - `docs/configuration.md` lines 14, 32 and 123 + - the GitTarget `Commit` godoc, which also leaves out the 1m `maxDuration` default. +- [ ] `docs/spec/commitrequest-design.md:126` still says `NoOpenWindow` and "close deadline". It should say `NoWindow` + and "attach deadline". +- [ ] Nits: + - `branch_worker.go` `commitWindowFor` has a stale doc comment stacked on top of the real one. + - The kstatus test names still say "close-delay". From 73a7e61e2ea62b4af0113c3b40fecc190fd8abc4 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 11:05:23 +0000 Subject: [PATCH 09/18] docs: keep the parent-branch pages to what is true, and set the merge boundary for #407 - gittarget-parent-branch.md now records the contract as built (immutable parentBranch, status.remote.parent, the conflict rule) and the reasoning behind it. The executed plan, the handoff, the architecture.md edits already applied, and the superseded "mutable" decision are gone. - gittarget-parent-hardening.md states the merge boundary, makes the CommitRequest timeout fix a prerequisite, tracks recovery per (target, collection) until published, and keeps the fallback's backoff per worker. - gittarget-parent-empty-repository.md is deferred: get the server's evidence through go-git first, then decide the bootstrap policy. Assuming main is recorded as considered and rejected. - gittarget-parent-observation.md ships availability before ancestry, and authorizes no mutation. - The #404 follow-ups move from a root todo.md into docs/TODO.md, by priority. INDEX lists the four pages. Co-Authored-By: Claude Opus 5.5 --- docs/INDEX.md | 6 +- docs/TODO.md | 53 ++ docs/design/gittarget-parent-branch.md | 699 ++++-------------- .../gittarget-parent-empty-repository.md | 196 ++--- docs/design/gittarget-parent-hardening.md | 169 +++-- docs/design/gittarget-parent-observation.md | 17 +- todo.md | 57 -- 7 files changed, 357 insertions(+), 840 deletions(-) delete mode 100644 todo.md diff --git a/docs/INDEX.md b/docs/INDEX.md index e39d606a..26d0e0cb 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -124,10 +124,14 @@ decided), **design, decided** (decision made, not built), **partly built**, **bu **deferred** (parked, kept as a decision record). The label is the first thing in the page, so you never have to read a proposal to find out it already shipped. -### Open — 23 pages +### Open — 27 pages | Doc | Open question | |---|---| +| [`gittarget-parent-branch.md`](design/gittarget-parent-branch.md) | **built in #407, not merged yet.** `spec.parentBranch`: a target stays on standby until a write needs a branch, then creates its write branch from the parent's tip as checked at publication. The contract as built, the defect it started from, prior art and naming. Open: parent per `GitProvider.status.branches` entry. | +| [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **plan, before #407 merges.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, all without adding a pull to the fast path. Depends on the CommitRequest timeout fix in [`TODO.md`](TODO.md). | +| [`gittarget-parent-empty-repository.md`](design/gittarget-parent-empty-repository.md) | **deferred.** An empty repository's default branch: the server reports it over protocol v2 but go-git discards it (upstream fix first); then a separate policy decision on which branch the first commit may create. Assuming `main` was considered and rejected. | +| [`gittarget-parent-observation.md`](design/gittarget-parent-observation.md) | **deferred.** Keep the parent visible after the write branch exists: availability first, ancestry only after measuring how often a shallow checkout can answer it. Informational; authorizes no branch mutation. | | [`gittarget-red-status-plan.md`](design/gittarget-red-status-plan.md) | **design, partly built.** The smaller alternative to Git write preflight: make a `GitTarget` turn red and explain why no commit was made, without moving Git checks into the admission webhook and without adding a second state source. Keeps the existing kstatus trio and domain conditions, ranks the most likely operator-facing failures, and has now shipped the first status-transition slice: refusal messages republish when details change, scoped refusals cannot be cleared by unrelated scope success, `BlockMessage()` carries stable issue kind/path/actor detail, and the unsupported-folder e2e covers Git-side break and recovery. | | [`open-asks-priority.md`](design/open-asks-priority.md) | **the work queue.** Merges three overlapping backlogs — the downstream consumer asks, the API-surface block left unbuilt by the status and configuration-model review, and the config-surface proposal (B1–B6) — into one ordered queue under four stated tests, and says where we deliberately do **not** do what was asked. The standing caveat narrowed once the layout model reversed: a Tier 2 entry belongs to postponed [#294](https://github.com/ConfigButler/gitops-reverser/issues/294) only if it breaks a `GitTarget` field, and everything else is independently schedulable. Makes one design call against what was asked: **delete Option C sibling inference** rather than ship an off-switch for it, because it let a human's edit to the repository change operator behavior with nothing in status recording the move. That deletion has shipped, and "what the deletion taught" records what building it found. **F9 is Tier 1**: the only item whose answer is unknown rather than whose work is unscheduled, and it gates planning the enum work | | [`proving-the-metric-surface.md`](design/proving-the-metric-surface.md) | **design, partly built.** What identity an operator needs to locate a failure, and what an e2e check has to do to be worth trusting. Written after a metric value that was declared, switched on, surfaced as a condition reason and counted turned out to have no producer at all, and the check added to catch that class then skipped whenever the counter read zero — so deleting every recording call left the suite green. Two rules follow: an assertion must not take its trigger from the metric under test, and no enum value may be reachable only through an injected fake. Carries a cardinality policy that separates bounded configuration identity from per-request identity, a table of which instrument families should own `source_cluster` directly rather than by join, and the PromQL reasoning for `increase()` versus `max_over_time` (neither applies mechanically: one cannot see an event before the first sample, the other cannot reconstruct totals across a reset). Steps 1–4 are built; step 5, diagnostics export, the `e2e_run` scrape label and a pinned-seed regression are not | diff --git a/docs/TODO.md b/docs/TODO.md index 2af5e7bf..e7af054b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -2,6 +2,59 @@ This file is meant to track the smaller current backlog, not historical notes. +## Follow-ups from #404 (commit window surface) + +PR #404 is merged; these need fix PRs against main, in this order. + +**Before #407 merges**: the parent-branch recovery holds work longer than this timeout (see +[gittarget-parent-hardening.md](design/gittarget-parent-hardening.md), "Prerequisite"). + +- [ ] **CommitRequest timeout and outcome agree with the worker.** + `resolveTimeout()` in `commitrequest_controller.go` is `attachTimeout + maxDuration + 120s` (about + 124s with defaults), counted from creation, while the worker keeps the attach. A remote down for + about two minutes, a long `WaitingForWorker`, or a controller restart reports `FinalizeFailed`, and + the commit (or a `CommitEmpty` record) still lands. Raising the bound only moves the contradiction: + the worker is authoritative while it holds a request, the bound applies to requests it does not + know, and failing closed withdraws the attach first. + +**Next, in one fix PR:** + +- [ ] **A suspended target makes a `CommitEmpty` save report Ready=True with no commit.** + `recordCommitRequest` returns `false, nil` on `errTargetSuspended`; the design table + (`docs/design/commit-timing-surface.md`) says a suspended target is a failure. An attached window + that changed nothing resolves `AlreadyPresent` the same way. Decide: fail the save, or change the + table. +- [ ] **A `CommitEmpty` save is recorded while the render-fidelity gate is closed.** Found by reading; + reproduce first. `recordCommitRequest` never calls `normalWritesAllowed`, so a save whose events the + gate dropped records "no writes were seen". The attached-window path fails the request instead. + +**Before the next release** (docs only): + +- [ ] `docs/UPGRADING.md`, the v1alpha3 window entry: + - step 1: GitOps-managed GitTargets need their Flux or Argo source suspended (or the field removed + there) until step 4, or the string `commit.window` is re-applied and breaks LIST; + - the Was/Is table needs a row for the chart value `quickstart.gitTarget.commit.window`, now an + object; + - "a removed `closeDelay` is not a hazard" is too strong: it is pruned silently for programmatic + clients (see `duration_fields_admission_test.go`). +- [ ] Descriptions that still say a save "closes the window now": the CommitRequest godoc and CRD + description, `docs/configuration.md` (lines 14, 32, 123), and the GitTarget `Commit` godoc, which + also omits the 1m `maxDuration` default. `docs/spec/commitrequest-design.md:126` still says + `NoOpenWindow` and "close deadline". + +**Later, non-blocking:** + +- [ ] `expireWaitingCommitRequests` ranges over a map, so empty commits for saves that time out in + the same pass land in random order; sort by `seq`. +- [ ] `buildRequestRecordWrite` keeps only the username; carry the full `UserInfo` so the record has + the same author identity as a window commit. +- [ ] A record commit skips `requestTemplate`; fix the code or the doc comment that says it is phrased + like every other commit. +- [ ] A request whose GitTarget never starts a worker fails with the generic safety-window message; + name `WaitingForWorker` when that was the last phase. +- [ ] Nits: a stale doc comment stacked on `commitWindowFor`; kstatus test names that still say + "close-delay". + ## Current backlog - [ ] Finish making `GitTarget` turn red quickly and explain why no commit was made. diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md index 14919b03..18fa4ed3 100644 --- a/docs/design/gittarget-parent-branch.md +++ b/docs/design/gittarget-parent-branch.md @@ -1,595 +1,158 @@ # GitTarget parent branch: keep a target on standby until a write needs a branch -> **design**: implemented in #407 (with #408's `spec.parentBranch` folded in). Open follow-ups: -> [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md) and -> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md). -> Date: 2026-10-01. +> **design**: built in #407, with #408's `spec.parentBranch` folded in. This page records the +> contract as built and the reasoning behind it. Open work: +> [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md) (before #407 merges), then +> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and +> [`gittarget-parent-observation.md`](gittarget-parent-observation.md) (deferred). > Related: [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), -> [`gittarget-configuration-freshness.md`](gittarget-configuration-freshness.md), -> [`../definitions.md`](../definitions.md) +> [`../definitions.md`](../definitions.md), and `architecture.md` (Ground rules; "A branch that does +> not exist yet"). -**The use case.** A `GitTarget` stays on standby while the cluster matches `main`. Its watches are -running, its write branch does not exist on the remote, and normal Git changes can keep flowing +**The use case.** A `GitTarget` stays on standby while the cluster matches its parent branch. Its +watches run, its write branch does not exist on the remote, and normal Git changes keep flowing through the cluster's GitOps reconciler. When a captured cluster edit needs a commit, the target -creates its write branch using a freshly checked parent. A target that waited a week must not +creates its write branch from the parent's current tip. A target that waited a week must not publish that edit on a week-old checkout. -**The proposal.** Let the target name its parent branch. Omitting the field keeps today's default -of the remote's default branch. Setting `release-1.x` or `env/prod` supports the same standby flow -against another branch. The write branch can later serve as a PR head, with the parent as its base; -creating PRs is separate work. - ```yaml spec: - branch: reverser/prod-edits # the write branch (unchanged, immutable) - parentBranch: main # optional; omitted = the remote's default branch + branch: reverser/prod-edits # the write branch (immutable) + parentBranch: env/prod # optional, immutable; omitted = the remote's default branch ``` **The constraint.** An idle, converged target creates no remote branch and no commit. Publication -validates the parent against the push session's advertisement and replays captured writes if it -moved. If replay leaves no difference, the target stays on standby. Existing commit windows and -push scheduling still apply: activation needs no manual switch or refresh tick, but publication -does not bypass those timers. Here, "no branch" means no branch on the remote; local working refs -may exist. - -**Independent of polling.** We already refresh remote observations with a **ten-minute default** -(`--git-refresh-interval`, Helm `git.refreshInterval`). Build this contract as though that refresher -were absent: it must hold with `--git-refresh-interval=0`, before the first tick, and between ticks. -Periodic refresh improves the idle view of Git; the publication check protects the first write. - -**Before the feature, a defect to fix (§2).** Code review identifies an untested defect: the push is -designed to refuse a parent branch that moved (the parent is recorded as the push root), but -`validatePushState` only makes that comparison when the write branch already exists. So when it does -not, a cycle that records the parent as its root can publish from a stale checkout. The exact worker -setup matters (§2.3); the push primitive's missing check is visible in the code. - -## Use cases and scope - -### Capture an operational edit for review - -The cluster normally follows `main`, and `reverser/prod-edits` is absent. An operator changes a -Deployment during an incident. Reverser captures that change, checks the parent when publishing, -and creates the write branch for review. Any intervening commits to `main` belong in its starting -history. Replay re-plans the captured edit against that new tree, including any changed file layout. - -This records the edit; keeping it active in the cluster still depends on the GitOps reconciler's -existing reconciliation policy. The parent-branch feature does not change that policy. - -### Keep normal deployments quiet - -A deployment advances `main`, and the reconciler applies the same manifests to the cluster. The -resulting watch events can initially look like edits against the cached parent. After checking and -replaying against the newer parent, they produce no difference. Reverser creates neither an empty -review branch nor a commit merely to record that it caught up. - -An idle remote refresh only reads and reports. Parent movement alone must not start a write or -turn the cluster's temporarily older state into a proposed rollback. Explicit resyncs retain their -existing cluster-to-Git semantics; this proposal does not resolve deployment-order races for them. - -### Capture changes against a release or environment branch - -A cluster following `release-1.x` or `env/prod` needs its proposed changes based on that branch. -`parentBranch` expresses this directly. Branching from `main` could otherwise include unrelated -release changes in the eventual PR. An explicit parent that is missing blocks branch creation. - -### Return to standby after review - -After a PR is merged and its write branch is deleted, the same target can use the parent again -when the next edit arrives. This depends on detecting deletion and replaying against the parent, -so it needs a lifecycle test. Automatic deletion, PR management, and deciding whether a surviving -write branch is safe to reset remain separate work (§7). While the write branch exists, new -commits build on it; this proposal does not continuously rebase it onto the parent. - -## 1. What happens today - -The default branch is already the parent branch, implicitly. - -- `SmartFetch` ([`git_smart_fetch.go`](../../internal/git/git_smart_fetch.go)) lists the remote refs. - If `spec.branch` exists it fetches it; otherwise it falls back to the ref the remote `HEAD` points at. - If neither exists (an empty repository) the worker makes `HEAD` unborn and the first commit is a root - commit. -- The worker compares the cluster against whatever it checked out. A resync fetches first - ([`resync_flush.go:132`](../../internal/git/resync_flush.go)), so a resync compares against the - default branch's current tip. -- `ensureWriteBranch` prepares a local `spec.branch` from the checked-out commit before executing - the writes. This can happen even when execution produces no commit. A later push creates the - branch on the remote. -- Once `spec.branch` exists on the remote, the default branch no longer matters. Nothing ever rebases - the write branch onto it. -- The [refresher](../../internal/git/refresh.go) checks aging remote observations on a reconcile - tick. The [refresh setting](../configuration.md#keeping-an-idle-target-fresh---git-refresh-interval) - defaults to ten minutes and sets the observation-age threshold for that check. - An unchanged branch costs one advertisement, and a moved branch also costs a fetch and reset. - Today it returns early when the write branch is absent, so the ten-minute setting does not keep - that target's parent checkout current. Parent following in the refresher is proposed in §2.4. - -## 2. Defect: a live event on an absent write branch commits onto a stale parent - -### 2.1 The path - -1. A worker for `branch: feature`, absent on the remote, finishes a fetch and reset with local - `HEAD` still on `main` at A. `SmartFetch` selected `main` as the fallback. This is the starting - state to establish explicitly; a completed snapshot resync can change it (§2.3). -2. `updateBranchMetadataFromPullReport` - ([`branch_worker.go:2767`](../../internal/git/branch_worker.go)) marks the base **trusted**, on - purpose, including for an absent branch. Its comment gives the reasoning: "for a branch the remote - does not have, a worktree based on the default branch IS that state. There is nothing on the remote - left to learn", and "a push onto a branch we believe is absent declares Old = zero, which the server - rejects if somebody has since created it". -3. Nothing changes in the cluster for a week. Meanwhile `main` moves to commit B. The refresher looks - only at `feature`, finds it absent, and returns without touching trust. -4. A live event arrives. `ensureBaseForCycle` ([`branch_worker.go:1824`](../../internal/git/branch_worker.go)) - sees a trusted, clean base and skips the fetch. The write is planned and committed on top of **A**. - `ensureWriteBranch` records the push-cycle root as `(main, A)` (`pushCycleRootBranch`/`Hash`, - [`branch_worker.go:218`](../../internal/git/branch_worker.go): "for a brand new branch it is the - default branch we based the new branch on. PushAtomic uses this pair to detect concurrent updates"). -5. `PushAtomic` receives that root. But `validatePushState` - ([`git_atomic_push.go:165`](../../internal/git/git_atomic_push.go)) compares the root's recorded - hash with its advertised tip **only inside `if found`**, that is, only when the write branch - already exists on the remote. For a new branch the only root check is "the root still exists". So - `main` at B versus the recorded A is never compared, the push goes out with `Old = zero`, and - `feature` is born on A, a week stale. - -So the design is right and already in place: the parent is recorded as the push root precisely so the -push can refuse a moved parent. The implementation skips that comparison in the one case it was -recorded for. `TestAtomicPush_PushToOther` -([`git_atomic_push_test.go:72`](../../internal/git/git_atomic_push_test.go)) is the only test of a -root that differs from the pushed branch, and its `main` never moves. - -### 2.1a A second hole in the same branch of code - -When the root differs from the pushed branch and the pushed branch **is** found (somebody created -`feature` between our fetch and our push), `oldHash` becomes their commit and the root check compares -`main` against `main`. If `main` did not move, the push sends `Old = `, -`New = `, and nothing checks that ours descends from theirs: their commit is overwritten by a -non-fast-forward update on any server that allows one. The comment in -`updateBranchMetadataFromPullReport` ("a push onto a branch we believe is absent declares Old = zero, -which the server rejects if somebody has since created it") describes behavior this code does not -enforce. Also from reading only. - -### 2.2 Consequences - -- The write was planned against A's folder. If B changed that folder (another writer, a merged PR), - the plan is wrong. At best the new branch lacks B's changes. At worst it re-writes a document B - restructured, and the eventual merge conflicts or quietly undoes B. -- The PR flow is where this surfaces: a freshly created PR branch that is already behind its base. -- A new worker starts untrusted and fetches before planning, so a restart can mask the stale state. - -### 2.3 Why the existing test does not catch it - -`TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain` -([`branch_worker_test.go:668`](../../internal/git/branch_worker_test.go)) checks out A, moves `main` -to B, then commits through a **newly constructed** worker. A new worker starts with the base untrusted, -so it fetches and lands on B. The test proves the cold path; the defect is on the warm path, where the -same worker already trusts its checkout of A. - -The primitive's missing comparison is clear from reading; no reproduction has been executed here. -Step 1 of the plan (§6) proves it with a failing unit test. - -There is a second setup detail: a completed no-op resync calls `ensureWriteBranch` before learning -that there is no difference. It can leave local `HEAD` on `feature`, although `feature` is absent -on the remote. The next cycle then records `(feature, A)`, and the existing missing-root check can -already force a fetch. The original assumption that any warm resync leaves `(main, A)` as the next -push root is therefore too broad. Test both states, and inspect the recorded root before claiming -that the worker or e2e reproduction must fail. - -### 2.3a A no-difference live event likely creates the write branch today - -Verified by reading on 2026-10-01, not executed. This breaks the core constraint more directly than -§2.1 does, so it gets its own red test. A live window that plans no change is **still retained and -pushed**: `finalizeOpenWindowWithReason` appends it "including for a no-diff window", because only -the remote can settle "already present". `commitPendingWrites` has already run `ensureWriteBranch`, -so local `feature` exists at the parent's commit. `runPushCycle` has no zero-commit short cut, and -`validatePushState` returns `PushNoBranch` only when the local hash is zero. With `feature` absent -and the local hash nonzero, the plan is `Old = zero, New = `. The push creates -`feature` on the remote with no commit of its own. If HEAD was already on a local `feature` (§2.3), -the missing-root rejection replays first, and the result is the same branch created at the new -parent tip. - -The "keep normal deployments quiet" use case hits exactly this: the reconciler applies `main`, the -watch sees the apply, and the window finds no difference. §2.4 item 2 is the fix; §6.2 item 4 and -§6.3 step 3b are the tests, and both should be red today. - -### 2.4 Fix - -**Keep publication responsible for freshness.** Use the existing advertisement and replay path: - -1. **`validatePushState` (the fix).** Compare `rootHash` with the root's advertised tip whether or not - the pushed branch exists. And when the root is a different branch from the one being pushed (a new - branch), require the pushed branch to still be absent: if it was found, return `RemoteMovedError` - for the pushed branch instead of pushing over it (§2.1a). Both rejections already flow into the - existing path: `remoteMovedDuringPush` invalidates the base, fetches, rebuilds the retained writes, - and pushes again. No extra connection, since the advertisement is read on the push session anyway. - Keep `Old = zero` for branch creation so the server also rejects a branch created after the - advertisement. The parent comparison is a client-side check: it is not atomic with the server's - update of the write branch. A parent can move during upload, with no fixed time bound on that - gap. The contract is the parent tip observed in the successful attempt's advertisement. -2. **Preserve standby after a no-op.** Live writes still reach the advertisement when they produce - no commit. After validation and any replay, if no pending write produced a commit and the write - branch remains absent, finish without creating it. Today `PushNoBranch` only covers an unborn - local `HEAD`; a local branch pointing at a nonzero parent needs explicit coverage too. Keep the - expected remote absence and resolved parent available across local branch switches and no-op - cycles. Local `HEAD` alone does not describe that relationship (§2.3). -3. **In the refresher (the folder view).** While the write branch is absent, check the parent branch - in the same advertisement. If it moved past the local checkout, fetch, reset, and re-read the - folder, so `status.placement` and refusals describe what the next commit will build on. - Still one connection when nothing moved; a moved parent also costs the fetch path. No commit. - This improves idle observations independently of the publication guarantee. Delaying or - disabling it must not change which parent a new write branch uses. - -Replay means re-planning the retained cluster writes against the fetched tree. It preserves -unrelated content and applies the existing conflict policy for the captured objects; it is not a -Git rebase of a previously published branch. If the remote cannot be checked, retain the pending -work for retry. If the parent keeps moving, use the existing bounded retry behavior and retain the -work rather than publishing an unchecked result. - -Rejected: removing trust for an absent write branch. The comment in step 2 records that an earlier -version did exactly that and was reverted for cost, and it would still leave the window between a -fetch and the push. - -This fix stands on its own and should ship first, without the new field: it is a correctness bug in -today's default-branch behavior. - -## 3. Prior art - -The original survey read both projects from `external-sources/` and the module cache. - -### Flux: `ImageUpdateAutomation` - -`image-automation-controller/api@v1.2.3`, `api/v1/git.go`: `spec.git.checkout.ref` (a full -`GitRepositoryRef`: branch, tag, semver, name or commit) and `spec.git.push.branch`, which "is created -using `.spec.checkout.branch` as the starting point, if it doesn't already exist". Omitted `checkout` -falls back to the `GitRepository`'s ref, and that to `master`. - -The starting point matters only when the push branch is created. `SwitchBranch` -(`external-sources/flux/pkg/git/gogit/client.go:462` in the local checkout): push -branch on the remote, build on it; absent, create it from the checked-out HEAD. There is no rebase. -Once the push branch exists, new commits on `main` are invisible to it; the usual remedy is to delete -the branch after the PR merges. - -### Argo CD: the source hydrator - -`pkg/apis/application/v1alpha1/types.go:428`: `drySource.targetRevision` (what it reads, any -revision), `syncSource.targetBranch` (the starting point), and an optional `hydrateTo.targetBranch` -(a staging branch "an external system would then have to move ... e.g. by pull request"). - -- Free-form revisions appear only where Argo reads. Everything it writes to, or creates a branch from, - is typed `targetBranch`. -- `CheckoutOrNew(target, base)` (`util/git/client.go:1637`): check out `hydrateTo` if it exists, else - create it from `syncSource`. Starting point at creation only, never a rebase. -- No difference, no commit: `WriteForPaths` returns `shouldCommit=false` and the commit server records - a git note instead (`commitserver/commit/commit.go:176`). -- `CheckoutOrOrphan` (`util/git/client.go:154`) creates an orphan branch when the sync branch is - absent: the explicit "empty branch" case of §5. - -### What we take from them - -1. A branch-typed field at a write position (Argo). -2. Both treat the starting point as a creation-time input only. We already compare against the parent - while the write branch is absent, and §7 is where we can go further than either. - -### Git tools that use "parent branch" - -git-town (`git town set-parent`, `git town sync`) and Graphite's stacks use **parent branch** for a -branch relationship that also includes synchronization. Our initial contract is narrower: follow -the parent while the write branch is absent, then build on the write branch. The name must not -promise automatic synchronization of an existing write branch; that remains deferred (§7). - -## 4. Proposal - -### 4.1 API - -```go -// GitTargetSpec - - // ParentBranch is the branch the write branch (spec.branch) is created from, and that the target - // compares against while the write branch does not exist on the remote. It is only read, so it - // does not need to be in the GitProvider's allowedBranches. Omitted: the branch the remote's HEAD - // points at (its default branch). When the write branch is absent, an explicitly set parent - // must exist or branch creation is refused. An existing write branch is used independently. - // +optional - // +kubebuilder:validation:MinLength=1 - ParentBranch string `json:"parentBranch,omitempty"` -``` - -- **Mutable.** It says where a branch starts; changing it abandons nothing already written (unlike - `branch`/`path`, which are the destination). A change takes effect the next time the write branch is - absent. -- **Not in `allowedBranches`.** We never write to it. -- **`parentBranch == branch` is legal** when that branch exists. If it is absent, the explicit - parent requirement blocks creation, including in an empty repository. Omission retains today's - bootstrap behavior. +checks the parent against the push session's advertisement and replays the captured writes if it +moved. If the replay leaves no difference, the target stays on standby. Commit windows and push +scheduling still apply. "No branch" means none on the remote; local working refs may exist. + +**Independent of polling.** The contract holds with `--git-refresh-interval=0`, before the first +refresh tick and between ticks. Periodic refresh improves the idle view of Git; the publication +check is what protects the first write. + +## Use cases + +- **Capture an operational edit for review.** The cluster follows `main` and `reverser/prod-edits` + is absent. An operator changes a Deployment during an incident. The edit is captured, the parent + is checked when publishing, and the write branch is created for review. Commits that landed on + `main` in the meantime are part of its starting history, and the replay re-plans the edit + against that tree, including a changed file layout. Keeping the edit active in the cluster still + depends on the GitOps reconciler's own policy. +- **Keep normal deployments quiet.** A deployment advances `main`, and the reconciler applies the + same manifests. The watch events can first look like edits against the cached parent; after the + check and replay they produce no difference, so no empty review branch and no catch-up commit + is created. An idle refresh only reads and reports. +- **A release or environment branch.** A cluster following `release-1.x` or `env/prod` needs its + proposed changes based on that branch. `parentBranch` says so. A set parent that is missing + blocks branch creation. +- **Return to standby after review.** When the PR is merged and the write branch deleted, the next + edit recreates it from the parent's current tip. While the write branch exists, new commits build + on it; it is never rebased onto the parent. + +## The contract as built -### 4.2 Naming - -[`definitions.md`](../definitions.md) binds here. The candidates it rules out: +| Write branch on remote | Parent branch on remote | Behavior | +|---|---|---| +| yes | any | work on the write branch; the parent is not followed | +| no | yes | compare against the parent; create the write branch from its tip, checked at publication | +| no | no, **set** | refuse: `Ready=False`, `Stalled=True`, reason `ParentBranchNotFound`; nothing written, no orphan branch | +| no | no, omitted, empty repository | unborn `HEAD`; the first commit is a root commit on the write branch | + +- **Immutable**, including adding or removing it, like `branch` and `path`. It decides what history + a new write branch gets. Delete and recreate the `GitTarget` to change it. +- **Not in `allowedBranches`.** It is only read. +- **`parentBranch == branch`** is legal when that branch exists. When it is absent, creation is + refused, including in an empty repository. Only an omitted parent bootstraps. +- **One parent per write branch.** One worker owns one checkout, so two targets on the same + provider and write branch that name different parents are a `TargetConflict`; the later one + loses. Values are compared as written: omitted and `main` conflict even when the remote's `HEAD` + is `main`. The answer needs no round trip and does not flip when the default branch changes. +- **`status.remote.parent`** shows, while the write branch is absent, what it would be created + from: `state` (`Found`, `Missing` or `Unborn`), `branch`, and `commit` when found. It disappears + once the write branch exists. + +## The defect this started from + +Before #407 the push already recorded the parent as the cycle's root, so that it could refuse a +moved parent. But `validatePushState` compared the root with its advertised tip only when the write +branch existed: the one case the root was recorded for was the one it skipped. That left three +holes, each now fixed and tested: + +1. **Stale parent.** A warm worker that trusted its checkout of `main` at A created the write + branch on A after `main` moved to B. +2. **A concurrently created branch.** If somebody created the write branch between our fetch and + our push, our non-descendant commit could overwrite theirs. +3. **A no-op created the branch.** A window with no difference is still pushed, because only the + remote can confirm "already present", and it created the write branch at the parent's commit. + +The fix compares the root whether or not the pushed branch exists, rejects a write branch that +appeared, and settles a new branch whose tip is still the parent's as "no branch". Both +rejections take the existing replay path, and no connection was added: the advertisement is read +on the push session anyway. + +**Rejected: distrust the base while the write branch is absent.** An earlier version did that and +was reverted for its cost, and it would still leave the gap between a fetch and the push. + +**What is and is not guaranteed.** The parent is checked against the advertisement of the +successful push attempt. Creation is protected by the server's compare-and-swap with `Old = zero`. +Neither locks the parent, which can still move after the advertisement. + +## Prior art + +- **Flux `ImageUpdateAutomation`.** `spec.git.push.branch` "is created using + `.spec.checkout.branch` as the starting point, if it doesn't already exist". The starting point + matters only at creation; there is no rebase, and the usual remedy is to delete the branch after + the PR merges. +- **Argo CD source hydrator.** Free-form revisions appear only where Argo reads; everything it + writes to or creates a branch from is a typed `targetBranch`. `CheckoutOrNew(target, base)` uses + the base at creation only. With no difference there is no commit (a git note instead). +- **git-town and Graphite** use "parent branch" for a relationship that includes synchronization. + Ours is narrower: follow the parent only while the write branch is absent. + +We took a branch-typed field at a write position (Argo), and the parent as a creation-time input +(both). + +## Naming + +[`definitions.md`](../definitions.md) binds here. | Candidate | Problem | |---|---| | `sourceRevision` | "source" is the source *cluster* (`clusterProviderRef`, `rules[].sourceNamespace`) | -| `baseRevision`, `baseRef`, `baseBranch` | "base" already means a kustomize **read-only base** and **base trust** (rule 1: one concept, one word); `revision` is reserved for a polymorphic Flux-style identifier; a `...Ref` field must reference an object (rule 2) | -| `checkout.ref.branch` | verbatim Flux, but a one-field block nested twice, and "checkout" names the action while this field names a relationship | -| `startPoint` | Git's own word (`git switch -c `), but it means creation only, and the field also governs what is followed | - -`parentBranch` is a single fact, so it is flat (rule 2), and it names the standby relationship. The -word "parent" appears in the subresource specs ("parent resource"), but "parent branch" always carries -its noun. In prose: **the parent branch** and **the write branch**. Both go into `definitions.md` -under "Write side" when this ships. - -The cost of the name: it can never hold a commit or tag. That is accepted (§5). +| `baseRevision`, `baseRef`, `baseBranch` | "base" already means a kustomize read-only base and base trust; `revision` is reserved for a polymorphic identifier; a `...Ref` field must reference an object | +| `checkout.ref.branch` | verbatim Flux, but a one-field block nested twice, and "checkout" names an action | +| `startPoint` | Git's own word, but it means creation only, and the field also governs what is followed | -### 4.3 One parent branch per write branch - -The branch worker is keyed by `(GitProvider namespace, GitProvider name, branch)` and owns one -checkout, so every target on that write branch must agree on its parent. `checkForConflicts` -([`gittarget_controller.go:1087`](../../internal/controller/gittarget_controller.go)) gains a second -clause: same provider and write branch, different `parentBranch` is a `TargetConflict`, and the -later-created target loses, as it does for overlapping paths. - -Values are compared as written, with omitted as its own value, so omitted and `main` conflict even -when the remote's `HEAD` is `main`. The controller decides without a remote round trip, and the answer -must not flip when somebody changes the remote's default branch. - -The worker receives the parent branch alongside `RepoIdentity`, and `EnsureWorker` treats a change -like a GitProvider repoint: invalidate the base, keep the worker. - -### 4.4 Worker behavior - -| Write branch on remote | Parent branch on remote | Behavior | -|---|---|---| -| yes | any | unchanged: fetch and work on the write branch | -| no | yes | compare against the parent branch; create the write branch from its current tip on the first commit (§2.4) | -| no | no, **set explicitly** | **refuse**: `Ready=False`, reason `ParentBranchNotFound`, nothing written | -| no | no, omitted, remote `HEAD` unset | unchanged: unborn `HEAD`, root commit (empty repository) | +The cost of the name: it can never hold a commit or a tag. -`SmartFetch` takes the parent branch instead of always using remote `HEAD`. When a configured parent -branch is missing it returns a sentinel error rather than `""`, so the worker never starts an orphan -branch because of a typo. +## Implicit when omitted, strict when set -`status.remote` keeps meaning the write branch (empty `commit` = not created yet). - -## 5. Implicit when omitted, strict when set; deferred explicit forms - -**The principle.** Omitted is lenient: the default branch, and an empty branch with no history when -the repository itself is empty, because that is the first-install case and requiring configuration for -it is friction. Set is strict: it must exist, or the target refuses. Making the empty-repository case -explicit would force everyone to configure today's default. +Omitted is lenient: the default branch, and a root commit when the repository is empty, because +that is the first-install case. Set is strict: it must exist, or the target refuses. Which branch +an empty repository's first commit may create is the open question in +[`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md). **Deferred, additive when someone asks:** -- **A deliberately empty write branch in a non-empty repository.** A dedicated `cluster-state` branch - with no history from `main`, the `gh-pages` pattern; Argo's `CheckoutOrOrphan`. Legitimate, unasked. - It would have its own field. -- **A pinned commit or tag** ("branch off exactly what is deployed"). Small value: the starting point - only decides where the write branch is born, after which it carries on independently, and a pin - leaves the refresher nothing to follow. Fetching one commit by hash also needs server support - (`uploadpack.allowReachableSHA1InWant` or similar); support varies between servers, which is why - Flux lets `commit` be paired with `branch`. It cannot go in `parentBranch`; it would be a sibling - field. - -## 5a. Handoff: executing this from a fresh context - -Read this section first. - -- **Historical.** This was the handoff for #407 and #408, both implemented. The argument lives in the - PR bodies, and the lasting text in `architecture.md` (§8). -- **Do not disturb the commit-window work** on `feat/commit-window-surface` in the main checkout. Work - in a separate git worktree on a new branch off `origin/main`. Before running `task test-e2e`, ask - the user: the e2e cluster is shared and a second run dies on its per-cluster lock. -- **Two PRs, in this order.** - - **PR 1, `fix(git)`:** §6.1, the §6.3 e2e spec, the §2.4 fix, the §6.2 tests, and the §8.1 to 8.3 - `architecture.md` edits. Also correct the comment in `updateBranchMetadataFromPullReport` that - claims an absent branch is pushed with `Old = zero` and that the server rejects it if somebody - created it. - - **PR 2, `feat(api)`:** the field, §4 and §6.4, plus §8.4. -- **This is the Git write path** (`internal/git/`), which AGENTS.md treats as high risk. Follow the - repo's validation sequence; the coverage ratchet bump (`.coverage-baseline`) is committed with the - change. -- **Show red before green for the defect.** The two §6.1 atomic-push tests should fail before the - fix. Worker tests must distinguish the local states in §2.3; an acceptance test that already - passes through missing-root recovery does not reproduce the primitive defect. Inspect the - recorded root and replay path, and retain that test as lifecycle coverage. - -### End-to-end test pitfalls found while planning §6.3 - -- **The Gitea repository starts empty.** Seed `main` with commit A from outside before creating the - target, or there is no parent to go stale. -- **Every helper hard-codes `main`.** `remoteBranchHead` (`repo_assertions_test.go:46`) runs - `ls-remote --heads origin main`, and `pushKustomizationFromOutside` (`refresh_remote_e2e_test.go:175`) - fetches and pushes `main`. The spec needs a branch parameter for the first, and a "commit file X to - `main` from outside" variant of the second. -- **The spec can pass without exercising the defect.** The stale-parent path needs a **warm** worker - that checked out A and trusts it *before* B is pushed. If the worker has not cloned yet when the - ConfigMap event arrives, it fetches on the cold path, lands on B, and the spec passes without - proving anything. So step 2 must observe that the worker fetched at A: candidates are the per-branch - fetch metric (the `fetchReason*` series in `branch_worker.go`, scraped by the e2e Prometheus) or the - controller log line `Preparing branch for operations` for that branch. Also account for a local - branch created by a no-op resync (§2.3). Do not use `status.remote` as proof of the checked-out - parent: it describes the write branch, and publication has its own cadence. -- **The target's own initial snapshot must not create the branch.** Use a `WatchRule` with a label - selector that matches nothing (the namespace always holds `kube-root-ca.crt`), so the snapshot is - an in-sync no-op that still clones. -- **The GitProvider's `allowedBranches`** must include the write branch. - -## 6. Plan and tests - -Order matters: prove today's behavior, fix the defect on today's default, then add the field. - -### 6.1 Reproduce the defect (unit, fails today) - -At the push primitive, next to `TestAtomicPush_PushToOther`: - -- `TestAtomicPush_PushToOther_DetectsMovedRoot`: local `feature` built on `main` at A, `main` moved to - B on the remote, `feature` absent. Expect `RemoteMovedError{Branch: main, Expected: A, Advertised: B}`. -- `TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch`: `main` unmoved, but `feature` created - on the remote by somebody else. Expect `RemoteMovedError` for `feature`, and their commit untouched. - -At the worker, through the whole cycle, cover both local states: - -`TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip`: one worker, seeded `main` -at A, write branch `feature` absent on the remote. First exercise a trusted fetch/reset that leaves -`HEAD` on `main`; separately exercise a completed no-op resync that leaves a local `feature`. -Assert the setup in each case. Push B to `main` from a second clone, then send a live write through -the **same** worker, with no intervening refresh or resync. Assert the first published commit on -`feature` has parent B and contains B's file. The first variant targets the missing parent check; -the second guards the normal standby lifecycle and can already pass through missing-root recovery. - -### 6.2 Fix (§2.4), and cover the "no commit while idle" side - -All with a seeded `main` and `feature` absent; every existing test of these rules runs on `main` or on -an empty remote: - -| Existing test | Why it does not cover the case | -|---|---| -| `TestResync_WorkerNoopDoesNotRetainOrPush` | write branch is `main` | -| `TestBranchWorker_DoesNotCreateBootstrapOnlyCommit` | empty remote | -| `TestRefresh_AnAbsentBranchCostsOneConnection` | empty remote, nothing to follow | -| `TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain` | cold worker only (§2.3) | -| e2e `commit_request_e2e_test.go:97` ("branch is not created") | empty repository, branch `main` | - -1. An in-sync snapshot creates no `feature`, and no push is attempted. -2. `main` moves (an unrelated file); a second in-sync snapshot still creates no `feature`. -3. `main` moves so the cluster now differs; the resync diffs against the new `main`. -4. `main` gains exactly the document the cluster has; replay leaves no commit and `feature` is - never created on the remote. Also cover a live write that already matches an unmoved parent. - Both still verify the remote before reporting completion. The live-write variant is expected red - today (§2.3a). -5. Push-time check: parent moved between plan and push; the push is treated as rejected, replays, and - `feature` parents on the new tip. Happy path: no extra connection (count with the existing fetch - and connection metrics, as `refresh_test.go` does). -6. Refresher: parent moved while `feature` is absent: advertisement followed by the fetch/reset - path and folder re-read, no commit. Parent unmoved: one connection, no fetch. -7. Publication with periodic refresh disabled, and with the last refresh completed immediately before - `main` advances: both start from the parent checked during publication. Exercise parent movement - while a commit window is open too. A refresh tick is never a precondition for activation. -8. Merge and delete the write branch, advance `main`, then send another edit through the same - worker. The recreated branch contains the new parent history and only the new pending edits. -9. Parent movement during replay retries: each attempt checks again. An unavailable remote or - exhausted retries retain the pending work and leave the remote write branch uncreated. - -### 6.3 e2e: the first commit lands on the latest default branch (required) - -This spec is the acceptance test for the standby lifecycle and is not optional. Write it after 6.1, -before the fix. If it already passes, inspect whether it used missing-root recovery (§2.3); the -primitive tests still have to reproduce the missing comparison. - -There is none today: apart from a negative `allowedBranches` check -(`gitprovider_validation_e2e_test.go:99`), every e2e `GitTarget` writes to `main`. New spec, -`Label("manager")`, no `parentBranch` set: - -1. Seed the Gitea repo's `main` with commit A. The `GitProvider` allows `main` and - `reverser-e2e-`. -2. Create a `WatchRule` whose label selector matches nothing yet, and a `GitTarget` writing to - `reverser-e2e-`. Wait until its initial snapshot has completed, which is what makes the - worker check out A and trust it. -3. Assert the write branch does not exist (`ls-remote`), and still does not after a steady tick: - idle means no branch and no commit. - 3b. **A live event with no difference.** Seed A with the folder already holding a ConfigMap - document, then create that ConfigMap in the cluster with matching content and the selector's - label. Assert the write branch is still absent once the commit window and push cooldown have - passed. This is the "keep normal deployments quiet" case, and §2.3a expects it to fail today. -4. Push commit B to `main` from outside, touching a file inside the target's folder, the way - `pushKustomizationFromOutside` does in `refresh_remote_e2e_test.go`. -5. Create a `ConfigMap` that matches the selector. Avoid a new resync between B and this live event. - Prove that recovery happened on the **publication** path by asserting which fetch fixed it: the - per-branch `GitFetchesTotal` series must gain a `contention` fetch (the rejected push) and no - `refresh` fetch between steps 4 and 6. That removes the refresher from the proof without a - dedicated deployment. The e2e controller runs with `--git-refresh-interval=30s` for every spec, - and redeploying it with `0` for one spec would disturb the rest of its leg. The - refresh-disabled case is cheap at unit level, where §6.2 item 7 covers it. -6. Assert the write branch now exists, its tip's parent is B, B's file is present, and the - `ConfigMap` document is in the folder. - -The bring-up costs one Gitea repository and one target; it fits an existing `manager` leg. - -### 6.4 The field - -- `parentBranch: release` with `release` ≠ default: the new write branch parents on `release`'s - tip checked during publication. -- Configured parent missing: `ParentBranchNotFound`, no orphan branch, no push; recovers once the - branch is pushed. -- `parentBranch == branch`: existing branch works; absent branch refuses, including on an empty - remote. An omitted parent still allows empty-repository bootstrap. -- Once the write branch exists, moving the parent changes nothing (the Flux/Argo rule we keep for now). -- `TargetConflict` for two targets on one write branch with different parents; created-later loses; - omitted versus `main` conflicts. -- Changing `parentBranch` on a live target while the write branch is absent re-reads the folder on the - next refresh. -- CRD: `parentBranch` non-empty when set; mutable. -- e2e: the §6.3 spec repeated with `parentBranch` naming a non-default branch. - -## 7. Next phase: following again after a merge - -Deleting the write branch after merge lets the target return to standby (§6.2). A write branch -left on the remote needs a separate policy: its next change builds on that surviving branch, which -may have fallen behind the parent. With a squash merge, a later PR can also show old commit history. - -Candidate rule: **when the write branch adds nothing to the parent branch, reset it to the parent's -tip** (force-push, or delete it and let the next commit recreate it). "Adds nothing" is easy for a -merge commit or fast-forward: the write tip is an ancestor of the parent tip. A squash merge is -different: the commits never enter `main`'s history, so the test has to be content-based, for example -"the folder's tree at the write tip equals the folder's tree at the parent tip". Open points: compare -the folder only, or every path the branch ever touched; and whether resetting a branch with an open PR -is acceptable on forges that treat it as closing the PR. Settle this with the PR-creation design. - -## 8. Changes to `docs/architecture.md`, applied with the fix - -Not applied yet, because `architecture.md` describes what the code does and today it does not do this. -Each edit lands in the PR that makes it true: 8.1 to 8.3 with the §2.4 fix, 8.4 with the field. - -`architecture.md` says nothing today about where a new write branch starts, and its ground rule -overstates the guarantee for a branch that does not exist yet: "a stale 'trusted' is caught by the -compare-and-swap on the next push" (line 73). The rule to state, briefly: - -> An idle target creates no branch and no commit. When the write branch does not exist yet, the -> target compares against its parent branch. Publication checks that parent against the push -> session's advertisement and replays retained writes when it moved. If replay leaves no commit, -> the write branch stays absent. This works with periodic remote refresh disabled. - -### 8.1 Ground rules (after the "stale trusted" sentence, around line 73) - -> For a new write branch, the worker checks the parent's advertised tip before sending the push. -> A moved parent triggers fetch, reset, and replay. The server's compare-and-swap protects creation -> of the write branch with `Old = zero`; it does not lock the parent. The parent can still move -> after the advertisement, so the guarantee is freshness at that observation, not at server acceptance. - -### 8.2 Local clones and conflict retry (next to the "deleted branch" paragraph, around line 1234) - -> **A branch that does not exist yet** is created from the remote's default branch (its parent) -> only when a write has something to commit, so an idle or in-sync target never creates it. -> Until then the worker compares the folder against the -> parent. Periodic refresh, ten minutes by default, updates the idle view independently of the -> publication check. The push records the parent and the commit it started from as its root, and -> treats a moved parent like a moved branch: fetch, reset, replay, check again. Proved end to end -> by the e2e spec `` with periodic refresh disabled. - -### 8.3 "Remote moved while we were writing" flowchart (around line 570) - -The decision node becomes `Write branch (or, while it is absent, its parent) still at the expected -commit?`, and the prose above it gains half a sentence: "...or, for a branch not created yet, if its -parent branch moved". - -### 8.4 Configuration model, GitTarget key fields (around line 358) +- **A deliberately empty write branch in a non-empty repository** (the `gh-pages` pattern; Argo's + `CheckoutOrOrphan`). It would get its own field. +- **A pinned commit or tag.** Small value: the starting point only decides where the write branch is + born, and a pin leaves nothing to follow. Fetching one commit by hash also needs server support. + It would be a sibling field, never `parentBranch`. -> - `spec.parentBranch`: optional, mutable branch the write branch is created from and compared -> against while it does not exist. Omitted means the remote's default branch. An explicit parent -> must exist when creating the write branch. It does not need to be in `allowedBranches` because -> it is only read. +## After a merge -Plus one line beside the existing overlap rule: "...and rejects two GitTargets on one branch that name -different parent branches." Update §8.2's default-branch wording to include the configured parent -in the same feature PR. +Deleting the write branch after its PR merges returns the target to standby. A write branch left +on the remote is a separate policy question: its next change builds on that branch, which may have +fallen behind the parent, and after a squash merge a later PR can show old history. A candidate +rule is to reset the write branch to the parent's tip when it adds nothing to it, but resetting, +force-pushing or deleting is a mutation that needs its own authorization. It belongs with the +PR-creation design, not with status. What the operator can see about the relationship in the +meantime is [`gittarget-parent-observation.md`](gittarget-parent-observation.md). -## 9. Open questions +## Open questions -1. Report the parent branch's observed commit in status (`status.remote.parent.commit`?) so an - operator can see what an absent write branch would be created from. Useful, but it grows a block the - refresher samples on the steady tick. -2. `ParentBranchNotFound` as a `Ready=False` reason versus its own condition. It gates writing, like - `BranchNotAllowed`, so a `Ready` reason looks right; confirm against - [`../spec/status-conditions-guide.md`](../spec/status-conditions-guide.md). -3. `GitProvider.status.branches` lists write branches; should each entry show its parent branch? -4. An omitted parent resolves through remote `HEAD` during fetch. Define when to re-resolve it if - the default branch changes while the old parent still exists: the current push session does - not advertise symbolic `HEAD`, so checking the recorded parent's hash alone cannot detect that - change. Explicit `parentBranch: main` avoids this ambiguity for the primary standby use case. +1. Should each `GitProvider.status.branches` entry show its write branch's parent? +2. With the parent omitted, when is the default branch re-resolved after the remote's `HEAD` + changes? The push session does not advertise `HEAD`. The hardening pass settles this as "the + default branch resolved at the last discovery" (its §4.7, R5). diff --git a/docs/design/gittarget-parent-empty-repository.md b/docs/design/gittarget-parent-empty-repository.md index 9b035be3..58aa02df 100644 --- a/docs/design/gittarget-parent-empty-repository.md +++ b/docs/design/gittarget-parent-empty-repository.md @@ -1,14 +1,11 @@ # GitTarget parent branch: an empty repository's default branch -> **Plan**. The experiment in §2 is done; nothing else is implemented. Dated 2026-10-01. +> **design**: deferred; not part of #407. The experiment in §2 is done, nothing else is built. +> Dated 2026-10-01. > -> Siblings: -> -> - [`gittarget-parent-branch.md`](gittarget-parent-branch.md), the original plan; -> - [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md), the hardening pass on #407. -> -> Run it **after** the hardening pass. Its §4.2 and §4.3 define "empty" and `Unborn`, and this -> plan refines that `Unborn` case. +> Related: [`gittarget-parent-branch.md`](gittarget-parent-branch.md), the feature as built; +> [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md), which keeps today's +> empty-repository behavior and defines "empty" (§4.2) and `Unborn` (§4.3). ## 1. The problem @@ -20,11 +17,14 @@ Today the first write creates **`reverser/edits` as the repository's first branc - The commit has no parent, and no `main` exists afterwards. - What we pushed does not match the branch the repository says is its default. - On GitHub, the first branch pushed to an empty repository probably becomes the default branch - (verify this, §5 step 0). If so, the operator's review branch silently becomes the repository's - default branch. Even if not, the eventual PR `reverser/edits` → `main` cannot be made: the two - branches have no common history. + (unverified, §5). If so, the operator's review branch silently becomes the repository's default + branch. Even if not, the eventual PR `reverser/edits` → `main` cannot be made: the two branches + have no common history. -This is not a status detail. It is the wrong branch name on the first commit we make. +This is more than a status detail: for a review-branch workflow it is the wrong branch on the first +commit we make. It is also what the bootstrap contract says today ("only an omitted parent +bootstraps", and the first commit is a root on the write branch), so changing it is a policy and +compatibility decision, not a bug fix. `branch: main`, which is what the quickstart uses, is unaffected: the first commit creates `main`, which is correct. @@ -61,7 +61,7 @@ disabled. It skips only without network. It asserts that `DefaultBranch` is **em the local HEAD is unborn on the *write* branch (`cool-test`). It pins today's behavior rather than the server's knowledge. -**A small, backward-compatible go-git patch recovers the answer.** I tested this patch through a +**A small, backward-compatible go-git patch recovers the answer.** It was tested through a temporary `-modfile` `replace`; nothing in the repository changed. The patch: - adds `transport.EmptyRemoteRepositoryError{Unborn plumbing.ReferenceName}`, whose `Is(ErrEmptyRemoteRepository)` @@ -84,134 +84,73 @@ no `Unborn`. unknown; the harness assumes v0. SSH to GitHub needs `GIT_PROTOCOL` sent over the session; that is not verified. -## 3. Proposed behavior - -Everything below turns on one name, the **default branch D** of an empty repository. There are two -ways to know D, and they ship in two stages. - -### 3.1 Stage 1, which this plan leads with: assume `main` - -**D is `main`**, hardcoded. This needs no go-git change, so it can ship now, alongside or right -after the hardening pass. - -- `main` is what GitHub, GitLab and Azure DevOps create for a new repository, and what `git init` - suggests. `master` is not an acceptable default in 2026, and nothing here falls back to it. -- Nothing is guessed silently. The assumption is visible in `status.remote.parent.branch`, it is - named in the refusal message, and it is documented in `configuration.md`. +## 3. Direction -### 3.2 Stage 2: ask the server +Two things, in this order: -When the server reports its unborn HEAD (§2: protocol v2 `ls-refs` with `unborn`, once go-git -passes it through, §4), **D is what the server said**. This only refines stage 1: the rules below -are the same, and only where D comes from changes. +1. **Get the evidence.** Recover the server's unborn `HEAD` through go-git (§4). With it, an empty + repository can report what it actually is: `status.remote.parent = {state: Unborn, branch: X}` + when the server said X, and an empty `branch` when it did not. Behavior does not change. +2. **Then decide the policy**, separately, with an explicit compatibility note. The question: when + the server says the default branch is X and the write branch is not X, may the first commit + still create the write branch as the repository's first branch? -### 3.3 The rules, given D + | Option | Effect | + |---|---| + | keep today's bootstrap | the write branch becomes the first branch (as now) | + | refuse until X exists | `Ready=False`, `Stalled=True`, a reason naming X: "push a first commit to 'X', or set `spec.branch: X`"; recovery is the normal standby flow | + | opt-in bootstrap of X | one atomic push: an empty-tree commit on X and the content on the write branch; X must be in `allowedBranches`. This writes to a branch the parent contract says is only read, so it needs its own field | -| Empty repository, and… | Behavior | -|---|---| -| `branch == D`, parent omitted | as today: the first commit creates D as the root. `status.remote.parent = {state: Unborn, branch: D}` | -| `branch != D`, parent omitted | **refuse.** Ready=False/Stalled, reason `ParentBranchUnborn`: "the repository is empty; its default branch 'D' has no commits yet. Push a first commit to 'D', or set `spec.branch: D`". Nothing is written; the work is retained, or remembered per hardening §4.4. When D appears, the normal standby flow creates the write branch from it | -| `parentBranch: P` | the same refusal and reason, naming P. Today this is `Missing`, a less precise message. See the escape hatch below for `P == branch` | + Whichever is chosen applies only when the server reported X. Without that evidence, today's + behavior stays. -**The cost of stage 1.** A repository whose real default branch is not `main` (say `trunk`), on a -server that does not report its unborn HEAD (v0/v1; Azure DevOps is unverified), with -`branch: trunk`: +### Considered and rejected: assume `main` -- Today that bootstraps `trunk` correctly. -- Under stage 1 it is refused, because we believe D is `main`. The message says exactly what to - do: push a first commit to `trunk`, after which everything works. -- Stage 2 removes the refusal for every server that reports its unborn HEAD, which includes - GitHub. +Assuming the default branch is `main` when the server does not say (never `master`) was considered +as a stage that needs no go-git change. It is rejected: -**Escape hatch, decision for the user:** allow `parentBranch == branch` on an empty repository to -mean "start this branch with no history". +- It turns working configurations into refusals. A repository whose default is `trunk`, with + `branch: trunk`, bootstraps correctly today and would be refused. +- The default is configurable in practice: `git init` takes `init.defaultBranch`, and GitHub + organizations set the default name for new repositories. A guess is wrong exactly for those + users. +- It would publish a guessed name inside a remote observation. Making the guess visible does not + make it evidence; `status.remote` reports what the remote said. -- This is explicit intent, needs no knowledge of D, and fixes the cost above without a manual - push. -- It reverses one rule from #408. Today, `parentBranch == branch` on an absent branch is refused - even in an empty repository. +Also not introduced here: reading `parentBranch == branch` on an empty repository as "start this +branch with no history". That would reverse a rule of the built contract as a side effect; if an +explicit bootstrap form is wanted, it gets its own design. -**Why refuse rather than bootstrap D ourselves.** The parent-branch contract is that the parent is -*only read*, and it is not in `allowedBranches`. Creating D ourselves would break that, and would -mean inventing an "initialize repository" commit on a branch we don't own. Refusing is the "clear -failure" this design asks for, and recovery is the existing standby flow. - -**Second decision for the user:** an opt-in that bootstraps instead. - -- The mechanism would be one atomic push of two refs: an empty-tree commit on D, and the content - commit on the write branch on top of it, with D required to be in `allowedBranches`. -- This plan does not include it. If you want it, it is a follow-up. - -**`RemoteObservation` and the API.** For `Unborn`, `ParentBranch` and `status.remote.parent.branch` -now always carry D. Update the doc comments that today say "empty when Unborn": - -- `internal/git/remote_observation.go`; -- `api/v1alpha3/gittarget_types.go` ~498–512; -- `configuration.md`. - -## 4. Getting the go-git change (stage 2 only) +## 4. Getting the go-git change 1. **Upstream first.** Open a go-git issue and a PR with §7: the error type, both paths, and tests - against a v2 `git http-backend` and over `file://`. - - The change is additive, and backward-compatible for every `errors.Is` caller. - - For prior art on our upstream write-ups, see the `file://` CAS write-up in - `external-sources/go-git/` (gitignored). -2. **Our side waits for a go-git release that carries it.** Until then stage 1 applies, so nothing - waits on upstream. -3. **Fallback, if upstream stalls:** a temporary `replace` to a fork tag. - - Only with the user's agreement. We already pin an alpha. - - Do not hand-roll a v2 `ls-refs` client in this repository. - -## 5. Steps - -### 5.1 Stage 1 (now) - -0. **Verify the GitHub claim** in §1 on a *throwaway* repository the user creates, never on - `ConfigButler/empty.git`. The question: does pushing a non-default branch first make it the - default? Record the answer in §1. It sharpens the motivation but does not change the design. -1. **Discovery.** The empty-repository result carries D. For now that is the constant - `DefaultBranchForEmptyRepository = "main"`, with one comment explaining why `main` and why - never `master`. `CheckRepo` returns `DefaultBranch{ShortName: "main", Unborn: true}`. -2. **Worker.** - - The empty-repository path applies the §3.3 table before `makeHeadUnborn`. - - The refusal is a new sentinel, `ErrParentBranchUnborn`. Handle it like - `ErrParentBranchNotFound` on fetch, refresh, publication and the hardening probe, so the same - recovery obligation applies. - - The observation becomes `ParentState=Unborn` with `ParentBranch=D`. -3. **Controller.** `parentBranchReadiness` maps that observation (with a `ParentRequested` match) - to `ParentBranchUnborn`, Stalled, using the §3.3 message. -4. **Tests**, red first: - - empty repository with `branch: main`: a root commit on `main`, as today. This covers the - quickstart path; - - `branch: feature`: refused, nothing pushed, the work retained. Then push `main` externally, - and assert that `feature` is created from it without another cluster edit; - - `parentBranch: release` on an empty repository: the `ParentBranchUnborn` message; - - the escape hatch, if chosen; - - ledger: rows unchanged. A constant costs no connection. -5. **Live test.** `TestCheckRepo_PublicConnectivityEmpty` expects `main` / `Unborn`, and its - `PrepareBranch("cool-test")` half now expects the refusal. -6. **Docs.** - - In `configuration.md`'s parent-branch section: add the empty-repository rows, the `main` - assumption, and the escape hatch, if chosen. - - In UPGRADING: a target writing anything other than `main` into an empty repository now waits - for `main`, or for the escape hatch. - -### 5.2 Stage 2 (once go-git carries the change) - -1. **Harness.** Add a `startRealGitServerV2`, or an option, that keeps `Git-Protocol`, so v2 - reaches `git http-backend`. Leave the ADO/v0 server and the ledger rows untouched. -2. **Discovery.** `listRemoteRefs`/`SmartFetchFrom` and `CheckRepo` use `errors.As` to read the - unborn target. When present it replaces the constant; when absent, the constant stays. -3. **Tests**, on the v2 harness, with a bare repository made by `init -b trunk`: - - `branch: trunk` bootstraps `trunk`; - - `branch: feature` is refused, naming `trunk`. - - On the v0 harness, the stage 1 behavior holds and is pinned. The ledger is unchanged: the - unborn target rides on the `ls-refs` call discovery already makes. + against a v2 `git http-backend` and over `file://`. It is additive and backward-compatible for + every `errors.Is` caller. An upstream PR may prefer to expose the unborn target on the + `Remote.List` result rather than only through the error; settle the shape with the + maintainers. +2. **Our side waits for a go-git release that carries it.** Until then nothing changes. +3. **Fallback, if upstream stalls:** a temporary `replace` to a fork tag, only with the user's + agreement. Do not hand-roll a v2 `ls-refs` client in this repository. + +## 5. Steps once go-git carries it + +1. **Verify the GitHub claim** in §1 on a *throwaway* repository, never on `ConfigButler/empty.git`: + does pushing a non-default branch first make it the default? Record the answer in §1. +2. **Harness.** Add a v2 variant of `startRealGitServer` that keeps `Git-Protocol`. Leave the + ADO/v0 server and the ledger rows untouched. +3. **Discovery.** `CheckRepo`, `listRemoteRefs` and `SmartFetchFrom` read the unborn target with + `errors.As`. `CheckRepo` returns `DefaultBranch{ShortName: X, Unborn: true}`; the observation + carries `ParentBranch=X` for `Unborn`. Update the doc comments that say "empty when Unborn" + (`remote_observation.go`, `api/v1alpha3/gittarget_types.go`, `configuration.md`). +4. **Tests.** On the v2 harness with a bare repository made by `init -b trunk`, status reports + `trunk`; on the v0 harness it reports an empty branch. Behavior is unchanged in both. The + ledger is unchanged: the target rides on the `ls-refs` call discovery already makes. +5. **Live test.** `TestCheckRepo_PublicConnectivityEmpty` expects `main` / `Unborn`. +6. **Then the policy decision in §3**, as its own change with an UPGRADING note. ## 6. Out of scope -- The opt-in bootstrap (§3.3, a decision for the user). +- The policy decision in §3, until the evidence exists. - Repositories that are non-empty but whose default branch is unresolved. That is hardening §4.3. ## 7. The go-git patch used in the experiment (against v6.0.0-alpha.5) @@ -257,6 +196,3 @@ To reproduce the experiment: 2. Copy `go.mod` and `go.sum` to `exp.mod` and `exp.sum`, and append `replace github.com/go-git/go-git/v6 => /go-git`. 3. Run `go test -modfile=exp.mod ./internal/git -run `. - -An upstream PR should probably also expose the unborn target on `Remote.List`'s caller-facing API -rather than only through the error. Discuss the shape with the maintainers. diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md index c581b166..b4c45bde 100644 --- a/docs/design/gittarget-parent-hardening.md +++ b/docs/design/gittarget-parent-hardening.md @@ -1,13 +1,15 @@ # GitTarget parent branch: hardening pass before #407 merges -> **Plan**, to be executed from a fresh context. Nothing in it is implemented yet. +> **Plan**: open, nothing built. Dated 2026-10-01. It comes from a three-way review of #407 and two +> rounds of second opinions; §7 records the design choices those rounds settled. > -> Revision 2, 2026-10-01. It folds in a second review of revision 1; §7 records what changed and -> why. The source was a three-way review of #407 at `dcbc8b82` plus a second opinion on that -> review. +> **The merge boundary.** #407 merges when this pass is done: the existing parent-branch feature +> behaves correctly under races and recovery, and stays as cheap as it is today. New policy +> (empty-repository bootstrap rules, ancestry status) is not part of it. > -> Siblings: [`gittarget-parent-branch.md`](gittarget-parent-branch.md) is the original plan, and -> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) follows this one. +> Related: [`gittarget-parent-branch.md`](gittarget-parent-branch.md), the feature as built; +> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and +> [`gittarget-parent-observation.md`](gittarget-parent-observation.md), deferred follow-ups. ## 0. Handoff: read this first @@ -15,18 +17,33 @@ - Branch `fix/new-branch-checks-parent` = PR #407. #408 (the `spec.parentBranch` field) is already merged into it. -- It is checked out in `/workspaces/gitops-reverser2`. Ignore the worktree - `/tmp/gitops-parent-branch`; it holds the stale `feat/gittarget-parent-branch`. -- The tip was `dcbc8b82` when this plan was written. Run `git fetch` and `git log origin/main..HEAD` - before starting. +- Run `git fetch` and `git log origin/main..HEAD` before starting. -### Neighbouring work, out of scope here +### Prerequisite: CommitRequest liveness -- `todo.md` holds follow-ups from #404, which are unrelated. -- [`gittarget-parent-observation.md`](gittarget-parent-observation.md) is a separate proposal: keep - the parent visible after the write branch exists. Do not implement it here. -- [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) runs after this - pass. +The recovery in §4.4 can hold work for up to its 5m backoff cap. Today the controller fails a +CommitRequest closed after `attachTimeout + maxDuration + 120s` (about 124s with defaults), counted +from creation, and the worker keeps the attach. So a request retained through a recovery can be +reported `FinalizeFailed` and then committed anyway. + +Fix this first, in a small PR against main (it is the first item in [`../TODO.md`](../TODO.md)). +Raising the timeout only moves the contradiction. Controller and worker must agree: + +- **While the worker holds the request** (waiting, collecting, retained for a push, or blocked on a + missing parent), the worker is authoritative and the controller keeps polling. A blocked request + reports a phase that names the cause. +- **The controller's bound applies only to a request the worker does not know** (a vanished worker). +- **Failing closed withdraws the attach first**, so a request reported failed can never be + committed later. + +Test: a request retained through a recovery longer than the old bound resolves `Committed`. + +### Out of scope for this pass + +- [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md): which branch an + empty repository's first commit may create. Keep today's bootstrap behavior here. +- [`gittarget-parent-observation.md`](gittarget-parent-observation.md): the parent's state after + the write branch exists. ### Process @@ -176,7 +193,7 @@ reconciles and refresh ticks occur. §4.4 defines it and tests it. a distrust that is gone. - The `newBranchParent` shortcut in `ensureWriteBranch` (~2346) then roots the cycle on `(main, A)`, and the push is accepted. - - Reproduced by the worker reviewer. + - Reproduced in review. - **Between recovery and publication.** - `pushPending` (~1656) runs `recoverRetainedWrites`, then `pushPendingCommits`. A change that lands between the two is pushed on the old root. @@ -232,7 +249,7 @@ reconciles and refresh ticks occur. §4.4 defines it and tests it. - Assert that `feature`'s first commit sits on `release`'s tip. - Assert that the observation recorded for that fetch names the parent it actually used (`main`), not `release`. -2. **Change during a no-op push** that settles as `PushNoBranch`. This is the reviewer's +2. **Change during a no-op push** that settles as `PushNoBranch`. This is the review's reproduction, an "after" case. Assert that the next real write lands on `release`. 3. **Change before admission** (the `beforePushAdmission` hook) in a cycle with retained writes. - Assert that no push leaves on the old root. @@ -425,10 +442,16 @@ Update the documentation in three places: - The latch has two independent halves: - **`retained`**: set when writes were kept. It clears only when `pushPending` actually publishes them, or settles them as `PushNoBranch`. - - **`snapshotNeeded`**: set when the worker **dropped** writes, or refused a resync, because - of the missing parent. Before implementing, verify which paths drop and which retain: - `noteMissingParent` at ~2425 and its callers. It clears only when a resync snapshot cycle - completes successfully *after* the parent was found. + - **`snapshotNeeded`, per scope**: the set of `(target, collection)` scopes whose writes the + worker **dropped**, or whose resync it refused, because of the missing parent. One worker + serves several targets, and resyncs run per collection, so a successful ConfigMap + snapshot for target A proves nothing about a Deployment of A, or about target B. (The + refusal recovery in `resync_flush.go` is scoped the same way, for the same reason.) + Before implementing, verify which paths drop and which retain: `noteMissingParent` at + ~2425 and its callers. + - A scope clears only when a resync of that scope, started *after* the parent was found, is + **published**: its commit pushed, or settled as a no-op. A resync reply is not enough, + because a resync can succeed while its write still waits for the push. - Neither half clears on an observation, on a probe result, or on a failed attempt. A failed attempt re-arms the probe deadline (step 3). 2. **The worker drives recovery.** It runs on the event loop, while the latch is set, whatever the @@ -439,9 +462,9 @@ Update the documentation in three places: - If the parent (or the write branch) is now present, record it. Then: - with `retained` set, call `pushPending`. It recovers through the normal replay: one fetch, a rebuild, a push. A failure leaves the latch set and re-arms the deadline; - - with `snapshotNeeded` set, raise a **snapshot request**. That is a monotonically increasing - `snapshotRequestSeq` that the worker exposes alongside its observation. A failed snapshot - cycle raises it again on the next deadline. + - for every target with an open scope, raise a **snapshot request**: a monotonically + increasing `snapshotRequestSeq` per target, exposed alongside the observation. A scope + that fails raises its target's sequence again on the next deadline. 3. **One probe deadline per worker, shared by every background probe.** - The worker keeps `nextRemoteProbeAt`, with backoff 10s, doubling, capped at 5m. It resets on a generation change (§4.1) and on success. @@ -478,9 +501,11 @@ Write the recovery tests first, against current code. They are the safety net fo - the next deadline recovers and publishes, **without another cluster edit**. Run it with refresh enabled and with refresh at `0`. -2. **Worker, dropped writes.** Assert that `snapshotNeeded` latches, that the sequence rises when - the parent appears, that it rises again after a failed snapshot cycle, and that the latch - clears only after a successful one. +2. **Worker, dropped writes, two scopes.** Two targets share the worker; writes for both are + dropped while the parent is missing. When it appears, one scope's resync succeeds and the + other's fails once. Assert that only the published scope clears, that the failing target's + sequence rises again, and that it clears after a later success. A resync that succeeded but + has not been pushed yet must not clear its scope. 3. **Controller unit.** A sequence bump forces exactly one recheck. A bump that happens entirely between two reconciles is still acted on. Ten reconciles at an unchanged sequence force nothing. @@ -502,7 +527,7 @@ Write the recovery tests first, against current code. They are the safety net fo tip; - make no cluster edit after the push. - This also covers the controller reviewer's item 4. + It also covers recovery from `ParentBranchNotFound`, which no test exercises today. 7. **Ledger rows.** Add "parent probe while missing" = 1 connection, 0 fetches. Row-level counts do not prove frequency; test 4 does. @@ -510,11 +535,15 @@ Write the recovery tests first, against current code. They are the safety net fo - Keep step 1 (the obligation) and step 4 (the request sequence). The obligation is the correctness part and is non-negotiable. -- Replace the worker probe with a controller-side forced recheck under exponential backoff - (`lastForcedAt` per target, 10s → 5m), still gated on the latch rather than on the Ready - reason. -- Tests 1–3 and 6 still apply. Test 4 then asserts the controller-side schedule. -- Record the worker probe as deferred work in the PR body, and adjust §5's memory note. +- Replace the worker probe with a controller-side forced recheck under exponential backoff, still + gated on the latch rather than on the Ready reason. +- **The schedule is per worker, not per target**, so the per-worker budget in §2 still holds: key + `lastForcedAt` by the worker's `(provider, branch)` and force all of that worker's affected + targets in one round. Per-target backoff would give three targets on one worker three + independent retry streams. +- Tests 1–3 and 6 still apply. Test 4 then asserts the controller-side schedule across three + targets on one worker. +- Record the worker probe as deferred work in the PR body. ### 4.5 Vocabulary (review item 5) @@ -606,54 +635,34 @@ explained by the steps above, and do not tune toward row 6. 4. Read the CodeRabbit inline comments again after each push. 5. Run `task test-e2e`, and the `full-manager` leg in CI. §4.4's new e2e context must pass at least once before calling the PR ready. -6. Update the memory note `gittarget-parent-branch-design` with what actually shipped: - - `parentConfig{name, gen}`, generation-scoped trust, and the push admission boundary replace - `parentChangedState`; - - `Unborn` means an advertisement with no hash refs; - - the default branch follows go-git's resolution; - - the recovery obligation, with `retained` and `snapshotNeeded`, and `snapshotRequestSeq`; - - **either** "a worker-side probe with one shared deadline" **or**, if the §4.4 fallback - shipped, "a controller-side backoff; the worker probe is deferred". +6. Fold what shipped into [`gittarget-parent-branch.md`](gittarget-parent-branch.md) (the + built contract) and `architecture.md`, and mark this plan done. ## 6. Out of scope -- Keeping the parent visible after the write branch exists. See - `docs/design/gittarget-parent-observation.md`. -- Resetting a surviving write branch onto its parent, and PR management. +- The two deferred follow-ups named in §0. +- Resetting, force-pushing or deleting a surviving write branch, and PR management. Ancestry + status or folder equality is not enough authorization for those mutations. - Any fetch before publication on a trusted checkout. -## 7. Changes from revision 1, and where they push back - -Each point of the second review was checked against the code before it was accepted: - -1. **Recovery must stay pending until it succeeds.** Accepted. §4.4 now has the latch with - `retained` and `snapshotNeeded`, and the `snapshotRequestSeq` handshake. Test 1 injects a - failed first recovery. - - **Partly pushed back:** the dropped-write integration is the new e2e context, not a new - controller+watch+worker harness. None exists outside e2e, and building one would be larger - than the bug. The worker and controller halves get their own unit tests (tests 2 and 3). -2. **The generation check and the race test disagreed.** Accepted. §4.1 now defines a - `{name, gen}` snapshot loaded once per operation, an explicit admission boundary, and the - `beforePushAdmission` hook. Tests 3 and 4 sit on either side of the boundary. -3. **R6 needs a production fix.** Accepted, and verified: the fallback probes the root only - (`branch_worker.go` ~2100). R6 has its own measured ledger row. -4. **One probe deadline for all probes.** Accepted. One `nextRemoteProbeAt` per worker; refresher - ticks before it are no-ops; reconciles never probe. Test 4 counts connections over time, not - per row. -5. **Resolver evidence.** Verified: go-git rewrites a hash-only `HEAD` before `List` returns. - - **Pushed back on the conservative option:** the plan adopts go-git's resolution, which - follows git's fallback spirit, instead of going below `Remote.List` for a rare server - setup. - - The tests run through the real transport, including a detached `HEAD`, to pin the - decision. The "the way `git clone` does" wording is gone. -6. **Truthful evidence for "no longer empty", and tags.** Accepted. There is a new - `RepositoryNotEmptyError`, and "empty" is defined once (no hash refs). A repository with only - tags is non-empty and goes through §4.3's refusal. - -Smaller corrections: - -- ledger rows 1–12 instead of 1–10; -- "no unconditional preflight fetch on a trusted checkout"; -- the push instruction is quoted inline, because a fresh session cannot read the memory note; -- the memory update depends on which §4.4 variant shipped; -- H4 maps to the existing row 11, so it needs no new row. +## 7. Design choices the review rounds settled + +Kept here so they are not re-litigated. Each was checked against the code. + +- **No pull before push.** A pull cannot close the race; the compare-and-swap plus bounded replay + does. Every fix in this pass reads evidence the push or discovery already holds. +- **Recovery is an obligation, not an observation.** It stays open until the work is published, + so a transient failure after the parent reappears cannot strand it (§4.4). It is tracked per + `(target, collection)` scope. +- **A configuration change is applied at a push admission boundary**, with one `{name, gen}` + snapshot per operation (§4.1). A push already admitted is not recalled. +- **One probe budget per worker**, whichever variant of §4.4 ships. +- **An empty repository has no hash refs**, tags included, and gaining one is reported with its own + typed error, not a `RemoteMovedError` that would misstate its fields (§4.2). +- **Default-branch resolution follows go-git**, which rewrites a hash-only `HEAD` before `List` + returns; a stricter rule would need a bespoke advertisement path for a rare server setup (§4.3). +- **The refused upload (R6) needed a production fix**: the fallback probe looked at the root only + (§4.7). It gets its own ledger row, because a refused upload costs more than a refusal at the + advertisement. +- **The dropped-write integration test is e2e.** No controller, watch and worker harness exists + outside it, and building one would be larger than the bug. diff --git a/docs/design/gittarget-parent-observation.md b/docs/design/gittarget-parent-observation.md index 58ef20dd..5bb369db 100644 --- a/docs/design/gittarget-parent-observation.md +++ b/docs/design/gittarget-parent-observation.md @@ -1,6 +1,6 @@ # GitTarget parent observations with shallow history -> **design**: proposed follow-up to PR #407; nothing in this document is implemented. +> **design**: deferred follow-up to PR #407; nothing in this document is implemented. > Date: 2026-10-01. Keep the parent branch visible after the write branch exists. Reuse remote ref advertisements to @@ -225,9 +225,18 @@ implies write-branch absence. ## Implementation and validation -Start by testing the shallow-object behavior with the pinned `go-git/v6` version. Then retain -parent observations through the existing refresh path, add the bounded comparison, and project -the evidence into status. Keep changes on a follow-up branch to PR #407. +Ship it in two steps, on a follow-up branch after PR #407: + +1. **Parent availability and timestamps**: retain the parent observation through the existing + refresh path after the write branch exists, and project it into status. This needs no history. +2. **Ancestry classification**, only after measuring how often the shallow checkout can answer + it. If most results come out `Unknown`, the graph walk is worth less than it looks, and the + remembered starting tip above may be enough. + +Before step 2, test the shallow-object behavior with the pinned `go-git/v6` version. + +Status is informational only. Nothing here authorizes resetting, force-pushing or deleting a write +branch; that stays with the PR-creation design. The implementation needs these cases: diff --git a/todo.md b/todo.md deleted file mode 100644 index 56cecfd9..00000000 --- a/todo.md +++ /dev/null @@ -1,57 +0,0 @@ -# TODO: follow-ups from the #404 review (commit window surface) - -PR #404 (`feat/commit-window-surface`) is merged; these need a follow-up PR against main. - -## Worth fixing - -- [ ] **The controller's safety timeout shrank from 420s to about 124s.** - - Where: `internal/controller/commitrequest_controller.go` `resolveTimeout()`. - - Today it is `attachTimeout + maxDuration + 120s`, using the request's own values and counted from - `CreationTimestamp`. - - The design (`docs/design/commit-timing-surface.md`, "safety timeout") says it covers the *largest* `attachTimeout` - and `maxDuration`, plus the push cooldown and retries. - - Failure: if the remote is down for more than about 2 minutes, or the save waits in `WaitingForWorker`, the save is - marked `FinalizeFailed` / `Stalled`. The worker keeps the attach, so the commit (or a `CommitEmpty` record) still - lands, and status contradicts Git. - - The same happens to a request that is older than the bound when the controller first sees it, for example after a - restart. - - Fix: bound it by the schema maximums (5m + 5m + allowance), or start the clock at the worker's registration and - withdraw the attach when failing closed. -- [ ] **A suspended target makes a `CommitEmpty` save report Ready=True with no commit.** - - Where: `commit_request_attach_loop.go` `recordCommitRequest` returns `false, nil` on `errTargetSuspended`, and - `commit_executor.go` `recordEmpty` requires `!target.Suspend`. - - The design table says "target is suspended → the failure". An attached window that changed nothing resolves - `AlreadyPresent` the same way. - - Decide: make the save fail, or change the design table. -- [ ] **A `CommitEmpty` save is recorded while the render-fidelity gate is closed.** (Found by reading the code, not - reproduced.) - - Where: `recordCommitRequest` never calls `normalWritesAllowed`. - - While the gate is closed, the author's events are dropped. The save then times out and records "no writes were - seen", which is false. - - The attached-window path fails the request in the same situation. - -## Low - -- [ ] `expireWaitingCommitRequests` ranges over the `pendingCRs` map, so empty commits for saves that time out in the - same pass land in random order. Sort them by `seq`. -- [ ] `buildRequestRecordWrite` sets `RequestAuthor: UserInfo{Username: pcr.author}`, which loses the display name and - email that a window commit carries. Carry the full `UserInfo` on the attach. -- [ ] A record commit skips `requestTemplate`. The doc comment says it is phrased like every other commit. Fix the - code or the comment. -- [ ] A request whose GitTarget never starts a worker fails after the timeout with the generic "did not resolve within - the safety window". If the last phase was `WaitingForWorker`, say so in the message. -- [ ] `docs/UPGRADING.md` (the v1alpha3 window entry): - - Step 1: GitOps-managed GitTargets need their Flux or Argo source suspended (or the field removed from it) until - step 4. Otherwise the string `commit.window` is re-applied and breaks LIST. - - The Was/Is table needs a row for the chart value `quickstart.gitTarget.commit.window`, which is now an object. - - "A removed `closeDelay` is not a hazard" is too strong. It is pruned silently for programmatic clients (see - `duration_fields_admission_test.go`). -- [ ] Descriptions that still say a save "closes the window now": - - the CommitRequest godoc and its CRD description - - `docs/configuration.md` lines 14, 32 and 123 - - the GitTarget `Commit` godoc, which also leaves out the 1m `maxDuration` default. -- [ ] `docs/spec/commitrequest-design.md:126` still says `NoOpenWindow` and "close deadline". It should say `NoWindow` - and "attach deadline". -- [ ] Nits: - - `branch_worker.go` `commitWindowFor` has a stale doc comment stacked on top of the real one. - - The kstatus test names still say "close-delay". From 0474b34a73afa83238772bd3e5c82690a3cc4615 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 11:39:01 +0000 Subject: [PATCH 10/18] fix(commit): never fail a CommitRequest the worker still holds, and withdraw one before failing it The controller failed a request closed attachTimeout + maxDuration + 120s after creation (about 124s with defaults) while the worker kept the attach. A remote down for two minutes, or a parent branch recovery, therefore reported FinalizeFailed for a request the worker then committed anyway. Controller and worker now agree: - A request the worker holds (CollectingWindow or WaitingForPush) has no controller-side timeout; only the worker ends it. - An unheld request past the bound is withdrawn first. The withdraw rides the worker's FIFO behind every attach, cancels a request that is only waiting for a window (or was never registered), and leaves a tombstone so a late attach cannot register it again. For a request the worker holds it is a no-op, and the controller keeps waiting. - The controller fails the request only on the worker's answer, and reports the real outcome when the worker resolved it first. A GitTarget that never started a worker is named in the message. Co-Authored-By: Claude Opus 5.5 --- docs/INDEX.md | 2 +- docs/TODO.md | 16 +-- docs/api-first-publication.md | 7 +- docs/design/commit-timing-surface.md | 6 +- docs/design/gittarget-parent-hardening.md | 26 ++-- docs/spec/commitrequest-design.md | 15 +- .../controller/commitrequest_controller.go | 86 ++++++++---- .../commitrequest_controller_unit_test.go | 111 ++++++++++++++- .../commitrequest_outcome_metric_test.go | 11 +- internal/git/branch_worker.go | 27 ++++ internal/git/commit_request_attach.go | 12 ++ internal/git/commit_request_attach_loop.go | 21 ++- internal/git/commit_request_withdraw_test.go | 129 ++++++++++++++++++ internal/git/types.go | 3 + internal/watch/event_router.go | 50 +++++++ internal/watch/event_router_test.go | 81 +++++++++++ 16 files changed, 531 insertions(+), 72 deletions(-) create mode 100644 internal/git/commit_request_withdraw_test.go diff --git a/docs/INDEX.md b/docs/INDEX.md index 26d0e0cb..ad70e018 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -129,7 +129,7 @@ so you never have to read a proposal to find out it already shipped. | Doc | Open question | |---|---| | [`gittarget-parent-branch.md`](design/gittarget-parent-branch.md) | **built in #407, not merged yet.** `spec.parentBranch`: a target stays on standby until a write needs a branch, then creates its write branch from the parent's tip as checked at publication. The contract as built, the defect it started from, prior art and naming. Open: parent per `GitProvider.status.branches` entry. | -| [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **plan, before #407 merges.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, all without adding a pull to the fast path. Depends on the CommitRequest timeout fix in [`TODO.md`](TODO.md). | +| [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **plan, before #407 merges.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, all without adding a pull to the fast path. | | [`gittarget-parent-empty-repository.md`](design/gittarget-parent-empty-repository.md) | **deferred.** An empty repository's default branch: the server reports it over protocol v2 but go-git discards it (upstream fix first); then a separate policy decision on which branch the first commit may create. Assuming `main` was considered and rejected. | | [`gittarget-parent-observation.md`](design/gittarget-parent-observation.md) | **deferred.** Keep the parent visible after the write branch exists: availability first, ancestry only after measuring how often a shallow checkout can answer it. Informational; authorizes no branch mutation. | | [`gittarget-red-status-plan.md`](design/gittarget-red-status-plan.md) | **design, partly built.** The smaller alternative to Git write preflight: make a `GitTarget` turn red and explain why no commit was made, without moving Git checks into the admission webhook and without adding a second state source. Keeps the existing kstatus trio and domain conditions, ranks the most likely operator-facing failures, and has now shipped the first status-transition slice: refusal messages republish when details change, scoped refusals cannot be cleared by unrelated scope success, `BlockMessage()` carries stable issue kind/path/actor detail, and the unsupported-folder e2e covers Git-side break and recovery. | diff --git a/docs/TODO.md b/docs/TODO.md index e7af054b..0413dc2b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -4,18 +4,8 @@ This file is meant to track the smaller current backlog, not historical notes. ## Follow-ups from #404 (commit window surface) -PR #404 is merged; these need fix PRs against main, in this order. - -**Before #407 merges**: the parent-branch recovery holds work longer than this timeout (see -[gittarget-parent-hardening.md](design/gittarget-parent-hardening.md), "Prerequisite"). - -- [ ] **CommitRequest timeout and outcome agree with the worker.** - `resolveTimeout()` in `commitrequest_controller.go` is `attachTimeout + maxDuration + 120s` (about - 124s with defaults), counted from creation, while the worker keeps the attach. A remote down for - about two minutes, a long `WaitingForWorker`, or a controller restart reports `FinalizeFailed`, and - the commit (or a `CommitEmpty` record) still lands. Raising the bound only moves the contradiction: - the worker is authoritative while it holds a request, the bound applies to requests it does not - know, and failing closed withdraws the attach first. +PR #404 is merged; these need fix PRs against main, in this order. (Its CommitRequest timeout +finding is fixed in #407: the controller no longer fails a request the worker holds.) **Next, in one fix PR:** @@ -50,8 +40,6 @@ PR #404 is merged; these need fix PRs against main, in this order. the same author identity as a window commit. - [ ] A record commit skips `requestTemplate`; fix the code or the doc comment that says it is phrased like every other commit. -- [ ] A request whose GitTarget never starts a worker fails with the generic safety-window message; - name `WaitingForWorker` when that was the last phase. - [ ] Nits: a stale doc comment stacked on `commitWindowFor`; kstatus test names that still say "close-delay". diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index 877ab549..2d41bfa2 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -309,8 +309,11 @@ leaving it to time out, so a shutdown mid-publication is reported as a failure a Use `Pushed=True` plus `status.commit` when a particular commit must exist in Git. `Ready=True` also covers successful outcomes with no commit. Status visibility follows the controller's polling -cadence and Kubernetes status writes. The controller's safety timeout is `420s` from object -creation; resolved worker outcomes are retained for `15m` with cleanup on subsequent resolutions. +cadence and Kubernetes status writes. A request the worker holds (attached to a window, or committed +and waiting for the push) has no controller-side timeout: it ends when the worker says so, however +long a down remote takes. A request the worker has not acted on within +`attachTimeout + maxDuration + 120s` of creation is withdrawn from the worker and then fails, so a +failed request is never committed later. Resolved worker outcomes are retained for `15m`. See the [request contract](spec/commitrequest-design.md). The two-second `attachTimeout` is not a guarantee that an earlier API edit has reached the diff --git a/docs/design/commit-timing-surface.md b/docs/design/commit-timing-surface.md index 66ad035f..2f572cff 100644 --- a/docs/design/commit-timing-surface.md +++ b/docs/design/commit-timing-surface.md @@ -215,8 +215,10 @@ after the remote confirmed the push. Every one of these comes from the worker. Sending an attach does not prove an attach, so the controller reports a phase only when the worker says the save reached it. -The controller's safety timeout covers the largest `attachTimeout`, the largest `maxDuration`, and -the push cooldown and retries. +The controller's safety timeout applies only to a save the worker has not acted on: +`attachTimeout + maxDuration + 120s` from creation, after which the save is withdrawn from the worker +and fails. Once the worker holds a save (attached, or committed and waiting for the push), only the +worker ends it, so the controller never reports a failure the worker later contradicts. ## Naming diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md index b4c45bde..3214c85b 100644 --- a/docs/design/gittarget-parent-hardening.md +++ b/docs/design/gittarget-parent-hardening.md @@ -19,24 +19,14 @@ merged into it. - Run `git fetch` and `git log origin/main..HEAD` before starting. -### Prerequisite: CommitRequest liveness - -The recovery in §4.4 can hold work for up to its 5m backoff cap. Today the controller fails a -CommitRequest closed after `attachTimeout + maxDuration + 120s` (about 124s with defaults), counted -from creation, and the worker keeps the attach. So a request retained through a recovery can be -reported `FinalizeFailed` and then committed anyway. - -Fix this first, in a small PR against main (it is the first item in [`../TODO.md`](../TODO.md)). -Raising the timeout only moves the contradiction. Controller and worker must agree: - -- **While the worker holds the request** (waiting, collecting, retained for a push, or blocked on a - missing parent), the worker is authoritative and the controller keeps polling. A blocked request - reports a phase that names the cause. -- **The controller's bound applies only to a request the worker does not know** (a vanished worker). -- **Failing closed withdraws the attach first**, so a request reported failed can never be - committed later. - -Test: a request retained through a recovery longer than the old bound resolves `Committed`. +### Done first: CommitRequest liveness + +Recovery can hold work for minutes, so a request must not be failed on a controller clock while the +worker still holds it. That is already fixed on this branch: the controller waits for any request +the worker holds (`CollectingWindow`, `WaitingForPush`), and fails an unheld one only after +withdrawing it from the worker (`docs/spec/commitrequest-design.md`). The §4.4 tests must still +include a request retained through a recovery longer than `attachTimeout + maxDuration + 120s`, +resolving `Committed`. ### Out of scope for this pass diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index c3f0b914..37a66cf4 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -103,9 +103,12 @@ object is visible. | Finalize or push error | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | `Reconciling=True` is the normal in-progress state, with the phase the worker reports as its reason: -`Progressing`, `WaitingForWorker`, `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. The controller fails -with `FinalizeFailed` only if the worker does not resolve the request within its bounded safety window; it -never polls indefinitely. +`Progressing`, `WaitingForWorker`, `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. Once the +request is `CollectingWindow` or `WaitingForPush` the worker holds it, and only the worker ends it. Before +that, a request still unresolved `attachTimeout + maxDuration + 120s` after creation is withdrawn from the +worker, which cancels it for good, and fails with `FinalizeFailed`; a GitTarget that never started a worker +is named in the message. The withdraw rides the worker's queue behind every attach, and is a no-op for a +request the worker holds, so the controller's failure and the worker's answer cannot disagree. **`Committed` and `AlreadyPresent` are decided by the push, including the no-commit one.** (`WindowMismatch` is decided locally, at the deadline: no window was attached, so there is nothing @@ -127,8 +130,8 @@ the re-send is recognized as the same request: it cannot register afresh and exp same-author window and stamp its message on somebody else's commit. Its close deadline is spent and no longer arms anything. -**A worker that stops while still holding the write fails the request**, rather than leaving the -controller to poll until its own safety window expires: a timeout says nothing about what happened, -and "the worker stopped before the commit reached the remote" does. +**A worker that stops while still holding the write fails the request**, because the controller +does not time out a request the worker holds, and "the worker stopped before the commit reached the +remote" says what happened. The complete status vocabulary is in the [status conditions guide](status-conditions-guide.md). diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index 154cefba..e0445803 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -33,6 +33,10 @@ import ( // registration and runs the window's timers, so the controller never holds the finalize itself. type CommitRequestFinalizer interface { ServiceCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) + // WithdrawCommitRequest cancels a request the worker has not acted on and reports the worker's + // answer: resolved with git.ErrCommitRequestWithdrawn when it is cancelled, the real outcome + // when the worker got there first, or resolved=false with the worker's phase while it decides. + WithdrawCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) } // CommandAuthorLookup resolves the author of a CommitRequest from the submitter @@ -50,11 +54,14 @@ type CommandAuthorLookup interface { const windowMismatchMessage = "the open commit window belongs to a different author or GitTarget; " + "nothing was committed for this request" -// resolveTimeoutMessage explains the fail-closed resolve bound: the worker never -// reported an outcome for the attached request within the safety window (e.g. the -// branch worker vanished), so the request fails closed rather than poll forever. +// resolveTimeoutMessage explains the fail-closed resolve bound: the worker never acted on the +// request within the safety window, so it was withdrawn rather than polled forever. const resolveTimeoutMessage = "the CommitRequest finalize did not resolve within the safety window" +// noWorkerTimeoutMessage is resolveTimeoutMessage for a GitTarget that never started a branch +// worker, the one cause the controller can name without asking anybody. +const noWorkerTimeoutMessage = "the CommitRequest's GitTarget did not start a branch worker within the safety window" + const ( // commitRequestPollInterval is the requeue cadence while polling the worker // for the attached request's outcome (attach-then-poll). @@ -108,9 +115,11 @@ func resolveRequestWindow(spec configbutleraiv1alpha3.CommitRequestSpec) request return resolved } -// resolveTimeout bounds the attach-then-poll wait, measured from object creation: the request's -// own attach wait, then its collection, then the push. Past it, a request the worker never resolved -// (e.g. a vanished worker) fails closed instead of polling forever. +// resolveTimeout bounds how long a request may go without the worker acting on it, measured from +// object creation: the request's own attach wait, then its collection, then the push. It does NOT +// bound a request the worker holds (attached to a window, or committed and waiting for the push): +// only the worker can say how that ends, however long a down remote or a recovery takes. Past the +// bound, an unheld request is withdrawn, and fails only once the worker agrees. func (w requestWindow) resolveTimeout() time.Duration { return w.attachTimeout + w.maxDuration + commitRequestPushAllowance } @@ -232,7 +241,7 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques // (no controller-side delay — the worker anchors the wait at its registration) // and poll the outcome. window := resolveRequestWindow(commitRequest.Spec) - result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, git.AttachCommitRequest{ + attach := git.AttachCommitRequest{ Namespace: commitRequest.Namespace, Name: commitRequest.Name, UID: string(commitRequest.UID), @@ -246,9 +255,10 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques IdleTimeout: window.idleTimeout, MaxDuration: window.maxDuration, CommitEmpty: commitRequest.Spec.WhenNothingToCommit == configbutleraiv1alpha3.NothingToCommitCommitEmpty, - }) + } + result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, attach) if serviceErr != nil || !resolved { - return r.awaitAttachOutcome(ctx, log, req, commitRequest, attribution, window, result.Phase, serviceErr) + return r.awaitAttachOutcome(ctx, log, req, commitRequest, attribution, window, attach, result.Phase, serviceErr) } if result.Err != nil { @@ -260,9 +270,11 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } -// awaitAttachOutcome handles an attach that has not produced an outcome yet: it keeps polling -// while the request is inside its safety window, and fails closed once that window is spent. A -// service error is a not-yet-serviceable worker, so it polls on the same terms. +// awaitAttachOutcome handles an attach that has not produced an outcome yet. It keeps polling +// while the worker holds the request, or while the request is inside its safety window. Past the +// window it withdraws the request and fails it only on the worker's answer, so a request reported +// failed can never be committed afterwards. A service error is a not-yet-serviceable worker, so it +// polls on the same terms. func (r *CommitRequestReconciler) awaitAttachOutcome( ctx context.Context, log logr.Logger, @@ -270,6 +282,7 @@ func (r *CommitRequestReconciler) awaitAttachOutcome( commitRequest *configbutleraiv1alpha3.CommitRequest, attribution commitRequestAttribution, window requestWindow, + attach git.AttachCommitRequest, phase git.CommitRequestPhase, serviceErr error, ) (ctrl.Result, error) { @@ -278,27 +291,54 @@ func (r *CommitRequestReconciler) awaitAttachOutcome( "name", req.NamespacedName, "err", serviceErr.Error()) } - if time.Since(commitRequest.CreationTimestamp.Time) < window.resolveTimeout() { - if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase); err != nil { - return ctrl.Result{}, err + if phase.Held() || time.Since(commitRequest.CreationTimestamp.Time) < window.resolveTimeout() { + return r.pollCommitRequest(ctx, commitRequest, attribution, phase) + } + + result, resolved, withdrawErr := r.Finalizer.WithdrawCommitRequest(ctx, attach) + if withdrawErr != nil || !resolved { + if withdrawErr != nil { + log.V(1).Info("CommitRequest withdraw not yet serviceable; will retry", + "name", req.NamespacedName, "err", withdrawErr.Error()) } - return ctrl.Result{RequeueAfter: commitRequestPollInterval}, nil + // Not answered yet, or the worker holds the request after all: keep polling. + return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) } - log.Info("CommitRequest did not resolve within the safety window; failing closed", - "name", req.NamespacedName) - // A nil serviceErr means the last attempt DID get the GitTarget and simply had no outcome - // yet, so the identity is known; a non-nil one never resolved it and takes the fallback. + // The worker resolved it, by the withdraw or on its own just before; a GitTarget that was read + // carries its branch, and only then is its identity known. target := commitRequestTarget{} - if serviceErr == nil { + if result.Branch != "" { target = resolvedCommitRequestTarget(commitRequest) } - r.writeTerminalStatus(ctx, log, commitRequest, - git.FinalizeResult{}, errors.New(resolveTimeoutMessage), attribution, target) + if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { + r.writeTerminalStatus(ctx, log, commitRequest, result, result.Err, attribution, target) + return ctrl.Result{}, nil + } + message := resolveTimeoutMessage + if result.Phase == git.PhaseWaitingForWorker || phase == git.PhaseWaitingForWorker { + message = noWorkerTimeoutMessage + } + log.Info("CommitRequest was not acted on within the safety window; withdrawn and failed closed", + "name", req.NamespacedName, "reason", message) + r.writeTerminalStatus(ctx, log, commitRequest, git.FinalizeResult{}, errors.New(message), attribution, target) return ctrl.Result{}, nil } +// pollCommitRequest reports where the worker says the request stands and polls again. +func (r *CommitRequestReconciler) pollCommitRequest( + ctx context.Context, + commitRequest *configbutleraiv1alpha3.CommitRequest, + attribution commitRequestAttribution, + phase git.CommitRequestPhase, +) (ctrl.Result, error) { + if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: commitRequestPollInterval}, nil +} + // stampFirstSightConditions stamps the still-running conditions the first time a request is seen, // so the object reports its progress (kstatus InProgress) and AuthorAttributed is settled // immediately. A disabled controller returns before this, so it never stamps. diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index 0cb87b3f..a59ae789 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -36,6 +36,20 @@ type fakeFinalizer struct { err error calls []git.AttachCommitRequest + + // The withdraw reply. The zero value is "not answered yet", so a test that never reaches the + // fail-closed path needs none of it. + withdrawResult git.FinalizeResult + withdrawResolved bool + withdrawErr error + withdrawCalls []git.AttachCommitRequest +} + +func (f *fakeFinalizer) WithdrawCommitRequest( + _ context.Context, attach git.AttachCommitRequest, +) (git.FinalizeResult, bool, error) { + f.withdrawCalls = append(f.withdrawCalls, attach) + return f.withdrawResult, f.withdrawResolved, f.withdrawErr } func (f *fakeFinalizer) ServiceCommitRequest( @@ -424,22 +438,115 @@ func TestCommitRequestReconcile_ServiceErrorPolls(t *testing.T) { assert.False(t, commitRequestIsTerminal(&got)) } -// Past the resolve safety window an attach the worker never resolved fails closed. +// Past the safety window a request the worker never acted on is withdrawn, and fails closed once +// the worker confirms the withdraw. func TestCommitRequestReconcile_ResolveTimeoutFailsClosed(t *testing.T) { // Zero CreationTimestamp: far past the resolve bound. cr := newCommitRequest("save-stuck") c := newCommitRequestClient(t, nil, cr) - f := &fakeFinalizer{resolved: false} + f := &fakeFinalizer{ + result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, + withdrawResult: git.FinalizeResult{Branch: "main", Err: git.ErrCommitRequestWithdrawn}, + withdrawResolved: true, + } r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} reconcileCommitRequest(t, r, "save-stuck") + require.Len(t, f.withdrawCalls, 1, "the request is withdrawn before it is failed") got := fetchCommitRequest(t, c, "save-stuck") requireCondition(t, got, ConditionTypeReady, metav1.ConditionFalse, crReasonFinalizeFailed) stalled := requireCondition(t, got, ConditionTypeStalled, metav1.ConditionTrue, crReasonFinalizeFailed) assert.Equal(t, resolveTimeoutMessage, stalled.Message) } +// A request the worker holds is never failed on the controller's clock: a down remote or a parent +// branch recovery can keep it in WaitingForPush far past the bound, and the worker still commits it. +func TestCommitRequestReconcile_AHeldRequestOutlivesTheBound(t *testing.T) { + for _, phase := range []git.CommitRequestPhase{git.PhaseCollectingWindow, git.PhaseWaitingForPush} { + t.Run(string(phase), func(t *testing.T) { + cr := newCommitRequest("save-held") // zero CreationTimestamp: far past the bound + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{result: git.FinalizeResult{Phase: phase}} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + res := reconcileCommitRequest(t, r, "save-held") + + assert.Equal(t, commitRequestPollInterval, res.RequeueAfter) + assert.Empty(t, f.withdrawCalls, "a held request is not withdrawn") + got := fetchCommitRequest(t, c, "save-held") + assert.False(t, commitRequestIsTerminal(&got)) + + // The push lands much later: the request resolves Committed, not FinalizeFailed. + f.result = git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"} + f.resolved = true + reconcileCommitRequest(t, r, "save-held") + got = fetchCommitRequest(t, c, "save-held") + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + }) + } +} + +// Until the worker answers the withdraw the request keeps polling; and when the worker turns out to +// hold it after all (it attached between the poll and the withdraw), nothing fails. +func TestCommitRequestReconcile_AnUnansweredWithdrawKeepsPolling(t *testing.T) { + for name, reply := range map[string]git.FinalizeResult{ + "not handled yet": {}, + "the worker now holds it": {Phase: git.PhaseCollectingWindow}, + } { + t.Run(name, func(t *testing.T) { + cr := newCommitRequest("save-race") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, withdrawResult: reply} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + res := reconcileCommitRequest(t, r, "save-race") + + assert.Equal(t, commitRequestPollInterval, res.RequeueAfter) + got := fetchCommitRequest(t, c, "save-race") + assert.False(t, commitRequestIsTerminal(&got)) + }) + } +} + +// A worker that resolved the request on its own just before the withdraw wins: its real outcome is +// what the request reports. +func TestCommitRequestReconcile_AWithdrawThatLostTheRaceReportsTheRealOutcome(t *testing.T) { + cr := newCommitRequest("save-late") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + withdrawResult: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, + withdrawResolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + reconcileCommitRequest(t, r, "save-late") + + got := fetchCommitRequest(t, c, "save-late") + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + assert.Equal(t, "abc", got.Status.Commit) +} + +// A GitTarget that never started a branch worker is named as the cause. +func TestCommitRequestReconcile_NoWorkerIsNamedWhenItFailsClosed(t *testing.T) { + cr := newCommitRequest("save-no-worker") + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Phase: git.PhaseWaitingForWorker}, + withdrawResult: git.FinalizeResult{ + Branch: "main", Phase: git.PhaseWaitingForWorker, Err: git.ErrCommitRequestWithdrawn, + }, + withdrawResolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + reconcileCommitRequest(t, r, "save-no-worker") + + got := fetchCommitRequest(t, c, "save-no-worker") + stalled := requireCondition(t, got, ConditionTypeStalled, metav1.ConditionTrue, crReasonFinalizeFailed) + assert.Equal(t, noWorkerTimeoutMessage, stalled.Message) +} + func TestCommitRequestReconcile_TerminalShortCircuits(t *testing.T) { cr := withReadyCommitted(newCommitRequest("save-done")) c := newCommitRequestClient(t, nil, cr) diff --git a/internal/controller/commitrequest_outcome_metric_test.go b/internal/controller/commitrequest_outcome_metric_test.go index ac3fc9b7..8f2dec59 100644 --- a/internal/controller/commitrequest_outcome_metric_test.go +++ b/internal/controller/commitrequest_outcome_metric_test.go @@ -112,8 +112,13 @@ func TestCommitRequestMetric_UnresolvableTargetTakesTheFallback(t *testing.T) { // Older than the resolve timeout, so the poll gives up on this pass rather than requeueing. cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * resolveRequestWindow(cr.Spec).resolveTimeout())) c := newCommitRequestClient(t, nil, cr) - // A service error is what EventRouter.ServiceCommitRequest returns when the GitTarget Get fails. - f := &fakeFinalizer{err: errors.New("get GitTarget default/no-such-target: not found")} + // A service error is what EventRouter.ServiceCommitRequest returns when the GitTarget Get fails, + // and a missing GitTarget has no worker to hold the request, so the withdraw resolves at once. + f := &fakeFinalizer{ + err: errors.New("get GitTarget default/no-such-target: not found"), + withdrawResult: git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, + withdrawResolved: true, + } r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} reconcileCommitRequest(t, r, "save-bogus") @@ -187,7 +192,7 @@ func TestCommitRequestMetric_EveryOutcomeIsReachableThroughReconcile(t *testing. nil, crOutcomeAlreadyPresent, }, - {"failed", git.FinalizeResult{}, errors.New("attach failed"), crOutcomeFailed}, + {"failed", git.FinalizeResult{Branch: "main", Err: errors.New("commit failed")}, nil, crOutcomeFailed}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 5bf54630..9c3de80f 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -529,6 +529,28 @@ func (w *BranchWorker) EnqueueRequest(request *WriteRequest) { w.enqueueRequest(request) } +// EnqueueWithdraw asks the loop to cancel a CommitRequest it has not acted on. It never blocks: a +// full queue or a stopping worker drops it, and the controller sends it again on its next poll. +func (w *BranchWorker) EnqueueWithdraw(req *AttachCommitRequest) { + if req == nil { + return + } + w.inflightItems.Add(1) + w.pendingResyncsMu.Lock() + defer w.pendingResyncsMu.Unlock() + if w.stoppingLocked() { + w.inflightItems.Add(-1) + return + } + select { + case w.eventQueue <- WorkItem{Withdraw: req}: + default: + w.inflightItems.Add(-1) + w.Log.V(1).Info("Queue full, CommitRequest withdraw dropped (the controller re-sends)", + "request", req.Namespace+"/"+req.Name) + } +} + // EnqueueAttach adds a CommitRequest attach to this worker's queue. Riding the // same queue as resource events is what makes it process in audit order, after // every earlier write. The attach is fire-and-forget: the controller polls the @@ -1146,6 +1168,11 @@ func (l *branchWorkerEventLoop) handleQueueItem(item WorkItem) { return } + if item.Withdraw != nil { + l.handleWithdrawCommitRequest(item.Withdraw) + return + } + if item.Resync != nil { // Take the CURRENT request for this scope: a newer one may have replaced // it while this marker waited in the FIFO. Running it here keeps the diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index dd2d0c32..97b65be0 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -62,6 +62,18 @@ const ( PhaseWaitingForPush CommitRequestPhase = "WaitingForPush" ) +// Held reports whether the worker has acted on a request in this phase: it is attached to a window +// or committed. From there only the worker can say how it ends, so the controller waits for it +// however long that takes, instead of failing the request on a clock of its own. +func (p CommitRequestPhase) Held() bool { + return p == PhaseCollectingWindow || p == PhaseWaitingForPush +} + +// ErrCommitRequestWithdrawn is the outcome of a request the controller withdrew before the worker +// acted on it. The worker records it, so a late re-send of the attach can never register the +// request again, and the controller's failure and the worker's answer are the same answer. +var ErrCommitRequestWithdrawn = errors.New("the CommitRequest was withdrawn before the worker acted on it") + // FinalizeResult carries the resolved outcome of a CommitRequest back to the // controller, polled via LookupCommitRequestOutcome. type FinalizeResult struct { diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index 9b665ed1..d486c5f8 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -29,7 +29,8 @@ import ( // commitRequestOutcomeTTL bounds how long a resolved CommitRequest outcome is // retained for the controller to poll before it is GC'd. It comfortably exceeds -// the controller's poll cadence and safety bound. +// the controller's poll cadence, and a withdrawn request's tombstone must outlive +// any attach the controller could still have in flight. const commitRequestOutcomeTTL = 15 * time.Minute // recordCommitRequestOutcome stores a resolved outcome and GCs stale entries. The @@ -365,6 +366,24 @@ func (w *BranchWorker) buildRequestRecordWrite( }, nil } +// handleWithdrawCommitRequest cancels a request the worker has not acted on: one that is only +// waiting for a window, or that was never registered at all. A request attached to a window or +// already committed is held, and withdrawing it is a no-op; the controller reads the held phase and +// keeps waiting. Either way the worker's answer and the controller's agree. +func (l *branchWorkerEventLoop) handleWithdrawCommitRequest(req *AttachCommitRequest) { + id := req.id() + if l.w.hasCommitRequestOutcome(id) { + return // already resolved, or already withdrawn. + } + if pcr, ok := l.pendingCRs[id]; ok && (pcr.attached || pcr.committed) { + l.w.Log.Info("CommitRequest withdraw ignored: the worker already holds it", + "request", id.Namespace+"/"+id.Name) + return + } + l.resolveCommitRequest(id, FinalizeResult{Err: ErrCommitRequestWithdrawn}) + l.rearmAttachTimer() +} + // resolveCommitRequest records a request's terminal outcome for the controller to // poll and forgets it from the pending set. func (l *branchWorkerEventLoop) resolveCommitRequest(id commitRequestID, result FinalizeResult) { diff --git a/internal/git/commit_request_withdraw_test.go b/internal/git/commit_request_withdraw_test.go new file mode 100644 index 00000000..71acf2a8 --- /dev/null +++ b/internal/git/commit_request_withdraw_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// The controller withdraws a CommitRequest before failing it closed. These pin the worker's half of +// that agreement: a request the worker has not acted on is cancelled for good, and a request it +// holds (attached, or committed) is not, so a request reported failed is never committed later. + +import ( + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWithdraw_AWaitingRequestIsCancelledForGood(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", time.Hour) + serviceAttach(loop, req) + require.Equal(t, PhaseWaitingForWindow, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + + loop.handleQueueItem(WorkItem{Withdraw: req}) + + res, ok := outcome(t, worker) + require.True(t, ok, "a withdrawn request resolves") + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + assert.Empty(t, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + + // A late re-send of the attach, and a window it would have attached to, change nothing. + serviceAttach(loop, req) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, + CommitMode: CommitModePerEvent, + }}) + require.NotNil(t, loop.openWindow) + assert.Nil(t, loop.openWindow.pendingCR, "a withdrawn request never attaches") + assert.Empty(t, loop.pendingCRs) + res, _ = outcome(t, worker) + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) +} + +// A withdraw that arrives before the worker ever saw the attach leaves a tombstone, so the attach +// that was still in flight cannot register afterwards. +func TestWithdraw_AnUnregisteredRequestCannotRegisterLater(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", time.Hour) + loop.handleQueueItem(WorkItem{Withdraw: req}) + serviceAttach(loop, req) + + res, ok := outcome(t, worker) + require.True(t, ok) + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + assert.Empty(t, loop.pendingCRs) +} + +// A request attached to a window is held: the withdraw is a no-op, and the request still resolves +// with its commit. This is the case the controller reads as "keep waiting". +func TestWithdraw_AnAttachedRequestIsHeldAndStillCommits(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, + CommitMode: CommitModePerEvent, + }}) + req := attachReq("alice", time.Hour) + serviceAttach(loop, req) + require.Equal(t, PhaseCollectingWindow, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + + loop.handleQueueItem(WorkItem{Withdraw: req}) + + _, resolved := outcome(t, worker) + require.False(t, resolved, "a held request is not withdrawn") + assert.True(t, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName).Held()) + + forceDue(loop) + loop.pushPending() + + res, ok := outcome(t, worker) + require.True(t, ok) + require.NoError(t, res.Err) + assert.Equal(t, FinalizeCommitted, res.Outcome) +} + +// A request that already resolved keeps its outcome. +func TestWithdraw_AResolvedRequestKeepsItsOutcome(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", 0) + serviceAttach(loop, req) // no window: resolves NoOpenWindow at once + before, ok := outcome(t, worker) + require.True(t, ok) + + loop.handleQueueItem(WorkItem{Withdraw: req}) + + after, _ := outcome(t, worker) + assert.Equal(t, before, after) +} + +func TestEnqueueWithdraw_QueueFullIsDroppedAndNilIsANoOp(t *testing.T) { + w := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1)} + w.EnqueueWithdraw(nil) + assert.Empty(t, w.eventQueue) + + w.eventQueue <- WorkItem{} // saturate + w.EnqueueWithdraw(attachReq("alice", 0)) + assert.Zero(t, w.inflightItems.Load(), "a dropped withdraw must not leak an inflight count") +} + +func TestCommitRequestPhase_Held(t *testing.T) { + assert.True(t, PhaseCollectingWindow.Held()) + assert.True(t, PhaseWaitingForPush.Held()) + assert.False(t, PhaseWaitingForWindow.Held()) + assert.False(t, PhaseWaitingForWorker.Held()) + assert.False(t, CommitRequestPhase("").Held()) +} diff --git a/internal/git/types.go b/internal/git/types.go index fb5e9146..6b404a7b 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -408,6 +408,9 @@ type WorkItem struct { // Refresh asks the worker to re-prove where its branch is on the remote. It is the only // work item that never writes anything: see RefreshRequest. Refresh *RefreshRequest + // Withdraw cancels a CommitRequest the worker has not acted on yet. It rides the same FIFO as + // the attach, so it is always handled after every attach sent before it. + Withdraw *AttachCommitRequest } // ResyncScope restricts a resync's mark-and-sweep to the slice of the mirror the desired diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index a2cd42c6..c7e100a3 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -109,6 +109,56 @@ func (r *EventRouter) ServiceCommitRequest( return result, resolved, nil } +// WithdrawCommitRequest is the controller's fail-closed seam: it cancels a request the worker has +// not acted on, and reports the worker's answer, so the controller never fails a request the worker +// could still commit. +// +// - No worker for the GitTarget: nothing holds the request and nothing can commit it, so it +// resolves at once as withdrawn. The controller stops sending attaches, so a worker that starts +// later never sees it. +// - A worker: the withdraw rides the FIFO behind every attach already sent, and is a no-op for a +// request the worker holds. Until it is handled, resolved=false and Phase says where the +// request stands; a held phase means the controller keeps waiting. +func (r *EventRouter) WithdrawCommitRequest( + ctx context.Context, + attach git.AttachCommitRequest, +) (git.FinalizeResult, bool, error) { + var gitTarget configv1alpha3.GitTarget + if err := r.Client.Get(ctx, client.ObjectKey{ + Name: attach.GitTargetName, + Namespace: attach.GitTargetNamespace, + }, &gitTarget); err != nil { + if apierrors.IsNotFound(err) { + // A deleted GitTarget has no worker left to hold the request. + return git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, true, nil + } + return git.FinalizeResult{}, false, fmt.Errorf("get GitTarget %s/%s: %w", + attach.GitTargetNamespace, attach.GitTargetName, err) + } + + worker, exists := r.WorkerManager.GetWorkerForTarget( + gitTarget.Spec.GitProviderRef.Name, + gitTarget.Namespace, + gitTarget.Spec.Branch, + ) + if !exists { + return git.FinalizeResult{ + Branch: gitTarget.Spec.Branch, + Phase: git.PhaseWaitingForWorker, + Err: git.ErrCommitRequestWithdrawn, + }, true, nil + } + + if result, resolved := worker.LookupCommitRequestOutcome(attach.Namespace, attach.Name, attach.UID); resolved { + return result, true, nil + } + worker.EnqueueWithdraw(&attach) + return git.FinalizeResult{ + Branch: gitTarget.Spec.Branch, + Phase: worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID), + }, false, nil +} + // recordBackgroundResyncFailure counts a fire-and-forget resync whose apply failed or // timed out at the worker, so the failure is observable even though delivery was already // marked on enqueue. No-op until the counter is registered. diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index 7a5c6222..4593bb5a 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -375,6 +375,87 @@ func TestServiceCommitRequest_RegisteredWorkerResolvesNoOpenWindow(t *testing.T) assert.Equal(t, "main", result.Branch) } +// TestWithdrawCommitRequest_NothingHoldsItWithoutAWorker: a deleted GitTarget, or one whose worker +// never started, has nothing that could commit the request, so the withdraw resolves at once and +// says why when it can. +func TestWithdrawCommitRequest_NothingHoldsItWithoutAWorker(t *testing.T) { + scheme := eventRouterScheme(t) + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "team-a-provider"}, + Branch: "main", + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gitTarget).Build() + workerManager := git.NewWorkerManager( + client, logr.Discard(), git.BranchWorkerLimits{}, types.SensitiveResourcePolicy{}) + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + + result, resolved, err := router.WithdrawCommitRequest(context.Background(), saveAttach("team-a-config", "team-a")) + require.NoError(t, err) + require.True(t, resolved) + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + assert.Equal(t, git.PhaseWaitingForWorker, result.Phase) + assert.Equal(t, "main", result.Branch) + + result, resolved, err = router.WithdrawCommitRequest(context.Background(), saveAttach("deleted", "team-b")) + require.NoError(t, err) + require.True(t, resolved) + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + assert.Empty(t, result.Branch, "a deleted GitTarget has no identity to report") +} + +// TestWithdrawCommitRequest_ALiveWorkerCancelsAWaitingRequest: the withdraw rides the FIFO behind the +// attach, cancels the request while it only waits for a window, and an attach sent afterwards +// cannot bring it back. +func TestWithdrawCommitRequest_ALiveWorkerCancelsAWaitingRequest(t *testing.T) { + scheme := eventRouterScheme(t) + provider := &configv1alpha3.GitProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-provider", Namespace: "team-a"}, + Spec: configv1alpha3.GitProviderSpec{URL: "file:///tmp/does-not-need-to-exist"}, + } + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "team-a-provider"}, + Branch: "main", + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(provider, gitTarget).Build() + workerManager := git.NewWorkerManager( + client, logr.Discard(), git.BranchWorkerLimits{}, types.SensitiveResourcePolicy{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = workerManager.Start(ctx) }() + time.Sleep(100 * time.Millisecond) // allow the manager to record its context + require.NoError(t, workerManager.EnsureWorker( + ctx, "team-a-provider", "team-a", "main", git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, "")) + + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + attach := saveAttach("team-a-config", "team-a") + attach.AttachTimeout = time.Hour // it would wait an hour for a window + + _, resolved, err := router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err) + require.False(t, resolved) + + var result git.FinalizeResult + require.Eventually(t, func() bool { + var err error + result, resolved, err = router.WithdrawCommitRequest(context.Background(), attach) + require.NoError(t, err) + return resolved + }, 5*time.Second, 50*time.Millisecond, "the worker must answer the withdraw") + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + + result, resolved, err = router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err) + require.True(t, resolved, "a re-sent attach finds the withdrawn outcome") + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) +} + // TestDrainScopedResync_QueueFullIsDrainedNotOrphaned guards the ordering invariant behind // Failure A/B: EnqueueResync answers a dropped request on its reply channel, so the drain MUST be // started even when the request never entered the FIFO. From a7eccb2b8c16b9e6888b226213b729682043f582 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:05:02 +0000 Subject: [PATCH 11/18] fix(commit): withdraw a CommitRequest from the worker that holds it, and settle requests when a worker exits Review of 0474b34a found three ways a withdraw could report a request failed while a worker could still publish it, or never answer: - A missing GitTarget, or no manager entry, was read as "nothing holds the request". A worker is shared by every GitTarget on its branch, so it outlives a deleted one, and a retired worker is detached before it stops, so its last push runs unlisted. The manager now records which worker accepted each request's attach (commitRequestOwners), and the router asks that worker first. Only a request no worker holds falls back to the GitTarget. - The withdrawn outcome could be collected by its 15m TTL while an attach for it was still queued, and that attach then registered the request again. An outcome is now kept while any attach for it is queued, and past the safety window the controller only withdraws, never attaches. - A worker whose loop exited, including one that never started because its GitProvider could not be read, stayed listed while attaches and withdraws queued unanswered. On exit a worker now closes admission, drains its queue, gives back every request it never acted on, fails any it held, and answers a withdraw at once. The router regression test fails on the previous router with "the withdraw reported success while the worker still held the registration". Co-Authored-By: Claude Opus 5.5 --- .coverage-baseline | 2 +- docs/spec/commitrequest-design.md | 9 + .../controller/commitrequest_controller.go | 71 ++++--- .../commitrequest_controller_unit_test.go | 47 +++-- internal/git/branch_worker.go | 60 +++++- internal/git/commit_request_attach_loop.go | 98 +++++++++- internal/git/commit_request_owners.go | 68 +++++++ internal/git/commit_request_withdraw_test.go | 181 ++++++++++++++++++ internal/git/worker_manager.go | 16 +- internal/watch/event_router.go | 103 ++++++---- internal/watch/event_router_test.go | 57 +++++- 11 files changed, 615 insertions(+), 97 deletions(-) create mode 100644 internal/git/commit_request_owners.go diff --git a/.coverage-baseline b/.coverage-baseline index 9336dfa7..fa863583 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -79.1 +79.2 diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index 37a66cf4..3df4ebb8 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -110,6 +110,15 @@ worker, which cancels it for good, and fails with `FinalizeFailed`; a GitTarget is named in the message. The withdraw rides the worker's queue behind every attach, and is a no-op for a request the worker holds, so the controller's failure and the worker's answer cannot disagree. +The withdraw goes to the worker that accepted the request's attach, not to whichever worker the GitTarget +names now: a worker shared by several GitTargets outlives a deleted one, and a retired worker finishes its +last push after the manager stopped listing it. Only a request no worker holds falls back to the GitTarget, +and only then does "no GitTarget" or "no worker" mean that nothing can commit it. Once the controller starts +withdrawing it sends no further attach, and the worker keeps a request's outcome while any attach for it is +still queued, so a late attach can never register a withdrawn request again. A worker whose loop has exited, +including one that never started because its GitProvider could not be read, gives back every request it +never acted on and answers a withdraw at once. + **`Committed` and `AlreadyPresent` are decided by the push, including the no-commit one.** (`WindowMismatch` is decided locally, at the deadline: no window was attached, so there is nothing for a push to say. So is `NoWindow` under `Resolve`. Under `CommitEmpty` a `NoWindow` request records diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index e0445803..434743e0 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -256,9 +256,32 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques MaxDuration: window.maxDuration, CommitEmpty: commitRequest.Spec.WhenNothingToCommit == configbutleraiv1alpha3.NothingToCommitCommitEmpty, } + if time.Since(commitRequest.CreationTimestamp.Time) >= window.resolveTimeout() { + // Past the bound the controller only withdraws: an attach sent now would sit behind the + // withdraw on the worker's FIFO and could register the request after it was cancelled. + return r.withdrawCommitRequest(ctx, log, req, commitRequest, attribution, attach) + } + return r.attachCommitRequest(ctx, log, req, commitRequest, attribution, attach) +} + +// attachCommitRequest sends the attach and reports the outcome once the worker has one. Until then +// it records the worker's phase and polls; a service error is a not-yet-serviceable worker, so it +// polls on the same terms. +func (r *CommitRequestReconciler) attachCommitRequest( + ctx context.Context, + log logr.Logger, + req ctrl.Request, + commitRequest *configbutleraiv1alpha3.CommitRequest, + attribution commitRequestAttribution, + attach git.AttachCommitRequest, +) (ctrl.Result, error) { result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, attach) if serviceErr != nil || !resolved { - return r.awaitAttachOutcome(ctx, log, req, commitRequest, attribution, window, attach, result.Phase, serviceErr) + if serviceErr != nil { + log.V(1).Info("CommitRequest attach not yet serviceable; will retry", + "name", req.NamespacedName, "err", serviceErr.Error()) + } + return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) } if result.Err != nil { @@ -270,54 +293,48 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } -// awaitAttachOutcome handles an attach that has not produced an outcome yet. It keeps polling -// while the worker holds the request, or while the request is inside its safety window. Past the -// window it withdraws the request and fails it only on the worker's answer, so a request reported -// failed can never be committed afterwards. A service error is a not-yet-serviceable worker, so it -// polls on the same terms. -func (r *CommitRequestReconciler) awaitAttachOutcome( +// withdrawCommitRequest handles a request past its safety window. A request the worker holds keeps +// polling with no bound of its own: only the worker can say how it ends. Any other request is +// withdrawn, and fails only on the worker's answer, so a request reported failed can never be +// committed afterwards. A withdraw that cannot be serviced yet polls on the same terms. +func (r *CommitRequestReconciler) withdrawCommitRequest( ctx context.Context, log logr.Logger, req ctrl.Request, commitRequest *configbutleraiv1alpha3.CommitRequest, attribution commitRequestAttribution, - window requestWindow, attach git.AttachCommitRequest, - phase git.CommitRequestPhase, - serviceErr error, ) (ctrl.Result, error) { - if serviceErr != nil { - log.V(1).Info("CommitRequest attach not yet serviceable; will retry", - "name", req.NamespacedName, "err", serviceErr.Error()) - } - - if phase.Held() || time.Since(commitRequest.CreationTimestamp.Time) < window.resolveTimeout() { - return r.pollCommitRequest(ctx, commitRequest, attribution, phase) - } - result, resolved, withdrawErr := r.Finalizer.WithdrawCommitRequest(ctx, attach) if withdrawErr != nil || !resolved { if withdrawErr != nil { log.V(1).Info("CommitRequest withdraw not yet serviceable; will retry", "name", req.NamespacedName, "err", withdrawErr.Error()) } - // Not answered yet, or the worker holds the request after all: keep polling. + // Not answered yet, or the worker holds the request: keep polling. return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) } - // The worker resolved it, by the withdraw or on its own just before; a GitTarget that was read - // carries its branch, and only then is its identity known. + if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { + // The worker resolved it on its own before the withdraw: report that outcome, as the + // attach path would have. + r.writeTerminalStatus(ctx, log, commitRequest, result, result.Err, attribution, + resolvedCommitRequestTarget(commitRequest)) + return ctrl.Result{}, nil + } + + // Withdrawn. A worker's answer, or a GitTarget that was read, carries the branch, and only + // then is the request's identity known. target := commitRequestTarget{} if result.Branch != "" { target = resolvedCommitRequestTarget(commitRequest) } - if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { - r.writeTerminalStatus(ctx, log, commitRequest, result, result.Err, attribution, target) - return ctrl.Result{}, nil - } message := resolveTimeoutMessage - if result.Phase == git.PhaseWaitingForWorker || phase == git.PhaseWaitingForWorker { + switch { + case errors.Is(result.Err, git.ErrBranchWorkerStopped): + message = git.ErrBranchWorkerStopped.Error() + case result.Phase == git.PhaseWaitingForWorker: message = noWorkerTimeoutMessage } log.Info("CommitRequest was not acted on within the safety window; withdrawn and failed closed", diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index a59ae789..5d432bc9 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -37,11 +37,13 @@ type fakeFinalizer struct { calls []git.AttachCommitRequest - // The withdraw reply. The zero value is "not answered yet", so a test that never reaches the - // fail-closed path needs none of it. + // The withdraw reply. Left unset, the withdraw answers like the router does before it cancels + // anything: with the worker's outcome, or its phase while unresolved. withdrawPending makes an + // unset reply mean "not answered yet" instead. withdrawResult git.FinalizeResult withdrawResolved bool withdrawErr error + withdrawPending bool withdrawCalls []git.AttachCommitRequest } @@ -49,7 +51,10 @@ func (f *fakeFinalizer) WithdrawCommitRequest( _ context.Context, attach git.AttachCommitRequest, ) (git.FinalizeResult, bool, error) { f.withdrawCalls = append(f.withdrawCalls, attach) - return f.withdrawResult, f.withdrawResolved, f.withdrawErr + if f.withdrawPending || f.withdrawResolved || f.withdrawErr != nil || f.withdrawResult.Phase != "" { + return f.withdrawResult, f.withdrawResolved, f.withdrawErr + } + return f.result, f.resolved, f.err } func (f *fakeFinalizer) ServiceCommitRequest( @@ -79,12 +84,15 @@ func attributedAlice() *fakeAuthorLookup { return &fakeAuthorLookup{author: queue.CommandAuthor{Author: "alice"}, found: true} } +// newCommitRequest builds a request created just now, inside its safety window, as the API server +// would stamp it. pastTheBound ages one beyond it. func newCommitRequest(name string) *configv1alpha3.CommitRequest { return &configv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - UID: types.UID("uid-" + name), + Name: name, + Namespace: "default", + UID: types.UID("uid-" + name), + CreationTimestamp: metav1.Now(), }, Spec: configv1alpha3.CommitRequestSpec{ GitTargetRef: meta.LocalObjectReference{Name: "team-a-config"}, @@ -93,6 +101,12 @@ func newCommitRequest(name string) *configv1alpha3.CommitRequest { } } +// pastTheBound ages a request beyond its safety window, where the controller only withdraws. +func pastTheBound(cr *configv1alpha3.CommitRequest) *configv1alpha3.CommitRequest { + cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * resolveRequestWindow(cr.Spec).resolveTimeout())) + return cr +} + // withReadyCommitted stamps a terminal Committed CommitRequest (Ready=True). func withReadyCommitted(cr *configv1alpha3.CommitRequest) *configv1alpha3.CommitRequest { apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ @@ -441,8 +455,7 @@ func TestCommitRequestReconcile_ServiceErrorPolls(t *testing.T) { // Past the safety window a request the worker never acted on is withdrawn, and fails closed once // the worker confirms the withdraw. func TestCommitRequestReconcile_ResolveTimeoutFailsClosed(t *testing.T) { - // Zero CreationTimestamp: far past the resolve bound. - cr := newCommitRequest("save-stuck") + cr := pastTheBound(newCommitRequest("save-stuck")) c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{ result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, @@ -454,6 +467,7 @@ func TestCommitRequestReconcile_ResolveTimeoutFailsClosed(t *testing.T) { reconcileCommitRequest(t, r, "save-stuck") require.Len(t, f.withdrawCalls, 1, "the request is withdrawn before it is failed") + assert.Empty(t, f.calls, "past the bound no attach is sent: it would queue behind the withdraw") got := fetchCommitRequest(t, c, "save-stuck") requireCondition(t, got, ConditionTypeReady, metav1.ConditionFalse, crReasonFinalizeFailed) stalled := requireCondition(t, got, ConditionTypeStalled, metav1.ConditionTrue, crReasonFinalizeFailed) @@ -465,7 +479,7 @@ func TestCommitRequestReconcile_ResolveTimeoutFailsClosed(t *testing.T) { func TestCommitRequestReconcile_AHeldRequestOutlivesTheBound(t *testing.T) { for _, phase := range []git.CommitRequestPhase{git.PhaseCollectingWindow, git.PhaseWaitingForPush} { t.Run(string(phase), func(t *testing.T) { - cr := newCommitRequest("save-held") // zero CreationTimestamp: far past the bound + cr := pastTheBound(newCommitRequest("save-held")) c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{result: git.FinalizeResult{Phase: phase}} r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} @@ -473,7 +487,8 @@ func TestCommitRequestReconcile_AHeldRequestOutlivesTheBound(t *testing.T) { res := reconcileCommitRequest(t, r, "save-held") assert.Equal(t, commitRequestPollInterval, res.RequeueAfter) - assert.Empty(t, f.withdrawCalls, "a held request is not withdrawn") + assert.Empty(t, f.calls, "past the bound no attach is sent") + assert.Len(t, f.withdrawCalls, 1, "the withdraw finds the request held and changes nothing") got := fetchCommitRequest(t, c, "save-held") assert.False(t, commitRequestIsTerminal(&got)) @@ -495,9 +510,13 @@ func TestCommitRequestReconcile_AnUnansweredWithdrawKeepsPolling(t *testing.T) { "the worker now holds it": {Phase: git.PhaseCollectingWindow}, } { t.Run(name, func(t *testing.T) { - cr := newCommitRequest("save-race") + cr := pastTheBound(newCommitRequest("save-race")) c := newCommitRequestClient(t, nil, cr) - f := &fakeFinalizer{result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, withdrawResult: reply} + f := &fakeFinalizer{ + result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, + withdrawResult: reply, + withdrawPending: true, + } r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} res := reconcileCommitRequest(t, r, "save-race") @@ -512,7 +531,7 @@ func TestCommitRequestReconcile_AnUnansweredWithdrawKeepsPolling(t *testing.T) { // A worker that resolved the request on its own just before the withdraw wins: its real outcome is // what the request reports. func TestCommitRequestReconcile_AWithdrawThatLostTheRaceReportsTheRealOutcome(t *testing.T) { - cr := newCommitRequest("save-late") + cr := pastTheBound(newCommitRequest("save-late")) c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{ withdrawResult: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, @@ -529,7 +548,7 @@ func TestCommitRequestReconcile_AWithdrawThatLostTheRaceReportsTheRealOutcome(t // A GitTarget that never started a branch worker is named as the cause. func TestCommitRequestReconcile_NoWorkerIsNamedWhenItFailsClosed(t *testing.T) { - cr := newCommitRequest("save-no-worker") + cr := pastTheBound(newCommitRequest("save-no-worker")) c := newCommitRequestClient(t, nil, cr) f := &fakeFinalizer{ result: git.FinalizeResult{Phase: git.PhaseWaitingForWorker}, diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 9c3de80f..9bb3b8ac 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -155,6 +155,11 @@ type BranchWorker struct { // block on the lock, and send after Stop had finished draining, into a queue with no loop left // to read it — and still answer "accepted". stoppingState bool + // exitedState is set once the event loop has returned, for whatever reason, and the worker + // has settled every CommitRequest it knew. From then on nothing can act on a request, so a + // withdraw is answered at once instead of being queued for a loop that is gone. Guarded by + // pendingResyncsMu, like stoppingState. + exitedState bool // lastObservation is what this worker last PROVED about the target branch on the remote, and // when. See RemoteObservation: it replaces a fetch-only trio (branchExists, lastCommitSHA, @@ -299,6 +304,14 @@ type BranchWorker struct { // crPhases holds where each unresolved CommitRequest stands, for the controller to report // while it waits. Guarded by crOutcomesMu; the event loop is the only writer. crPhases map[commitRequestID]CommitRequestPhase + // crQueuedAttaches counts, per request, the attaches on the FIFO that the loop has not handled + // yet. An outcome is not collected while any are queued: the outcome is what turns a late attach + // into a no-op, so collecting it first would let that attach register a withdrawn request + // again, however long the outcome had been kept. Guarded by crOutcomesMu. + crQueuedAttaches map[commitRequestID]int + // crOwners is the manager's table of which worker holds each request. See commitRequestOwners. + // Set by the WorkerManager before Start; nil for a worker built on its own. + crOwners *commitRequestOwners } // branchWorkerLogFirsts logs the first successful commit and push of a worker's @@ -486,6 +499,7 @@ func (w *BranchWorker) Start(parentCtx context.Context) error { go func() { defer w.wg.Done() w.processEvents() + w.settleCommitRequestsAfterExit() }() return nil @@ -530,18 +544,25 @@ func (w *BranchWorker) EnqueueRequest(request *WriteRequest) { } // EnqueueWithdraw asks the loop to cancel a CommitRequest it has not acted on. It never blocks: a -// full queue or a stopping worker drops it, and the controller sends it again on its next poll. +// full queue or a stopping worker drops it, and the controller sends it again on its next poll. A +// worker whose loop has exited answers at once, because nothing is left that could act on it. func (w *BranchWorker) EnqueueWithdraw(req *AttachCommitRequest) { if req == nil { return } - w.inflightItems.Add(1) w.pendingResyncsMu.Lock() + if w.exitedState { + w.pendingResyncsMu.Unlock() + w.answerWithdrawAfterExit(req.id()) + return + } defer w.pendingResyncsMu.Unlock() if w.stoppingLocked() { - w.inflightItems.Add(-1) + // Still draining: its last push may yet carry the request, so only the exited worker can + // answer. The controller asks again on its next poll. return } + w.inflightItems.Add(1) select { case w.eventQueue <- WorkItem{Withdraw: req}: default: @@ -576,7 +597,18 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { w.Log.V(1).Info("Worker is stopping, CommitRequest attach refused (the controller re-sends)") return } + id := req.id() + if !w.crOwners.claim(id, w) { + // The router routes to the owner, so only a request that moved workers between two polls + // gets here. The owner answers for it; registering it twice could commit it twice. + w.pendingResyncsMu.Unlock() + w.inflightItems.Add(-1) + w.Log.Info("CommitRequest attach refused: another branch worker holds the request", + "request", req.Namespace+"/"+req.Name) + return + } w.markResyncTailForTargetLocked(req.GitTargetNamespace, req.GitTargetName) + w.countQueuedAttach(id) select { case w.eventQueue <- WorkItem{Attach: req}: w.pendingResyncsMu.Unlock() @@ -590,6 +622,7 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { // time, so an enqueue is visible to the next scrape whether or not the // loop has woken to notice it. default: + w.uncountQueuedAttach(id) w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) w.recordQueueDrop(queueDropAttach) @@ -1389,15 +1422,22 @@ func (l *branchWorkerEventLoop) failUnpushedCommitRequests() { } } -// drainUnhandledQueueItems clears items the exiting loop will never handle. Each was counted into -// inflightItems at enqueue and decremented only after handling, so without this the final depth -// would stay non-zero forever: the loop has stopped, so nothing republishes a corrected value. A -// buffered CommitRequest attach is dropped; the controller re-sends on its next poll. -func (l *branchWorkerEventLoop) drainUnhandledQueueItems() { +// drainUnhandledQueueItems clears items the exiting loop will never handle. See drainQueue. +func (l *branchWorkerEventLoop) drainUnhandledQueueItems() { l.w.drainQueue() } + +// drainQueue clears items no loop will handle. Each was counted into inflightItems at enqueue and +// decremented only after handling, so without this the final depth would stay non-zero forever: +// the loop has stopped, so nothing republishes a corrected value. A buffered CommitRequest attach +// is dropped, and the request given back unless the worker knows it otherwise; the controller +// re-sends on its next poll, to whichever worker holds it then. +func (w *BranchWorker) drainQueue() { for { select { - case <-l.w.eventQueue: - l.w.inflightItems.Add(-1) + case item := <-w.eventQueue: + if item.Attach != nil { + w.uncountQueuedAttach(item.Attach.id()) + } + w.inflightItems.Add(-1) default: return } diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index d486c5f8..433ab7e1 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -29,26 +29,115 @@ import ( // commitRequestOutcomeTTL bounds how long a resolved CommitRequest outcome is // retained for the controller to poll before it is GC'd. It comfortably exceeds -// the controller's poll cadence, and a withdrawn request's tombstone must outlive -// any attach the controller could still have in flight. +// the controller's poll cadence. It is not what keeps a withdrawn request withdrawn: +// an outcome is never collected while an attach for it is still queued (crQueuedAttaches), +// and the controller sends no attach once it has started withdrawing. const commitRequestOutcomeTTL = 15 * time.Minute +// ErrBranchWorkerStopped is the outcome of a request whose branch worker stopped before it could +// finish it. Once the loop has exited nothing can publish the request, so this answer is final. +var ErrBranchWorkerStopped = errors.New("the GitTarget's branch worker stopped before it finished the CommitRequest") + // recordCommitRequestOutcome stores a resolved outcome and GCs stale entries. The // event loop is the only caller, but it takes the mutex because the controller // reads concurrently via LookupCommitRequestOutcome. func (w *BranchWorker) recordCommitRequestOutcome(id commitRequestID, result FinalizeResult) { w.crOutcomesMu.Lock() defer w.crOutcomesMu.Unlock() + w.recordCommitRequestOutcomeLocked(id, result) +} + +// recordCommitRequestOutcomeLocked is recordCommitRequestOutcome for a caller holding crOutcomesMu. +func (w *BranchWorker) recordCommitRequestOutcomeLocked(id commitRequestID, result FinalizeResult) { if w.crOutcomes == nil { w.crOutcomes = map[commitRequestID]commitRequestOutcomeEntry{} } now := time.Now() w.crOutcomes[id] = commitRequestOutcomeEntry{result: result, resolvedAt: now} for k, entry := range w.crOutcomes { - if now.Sub(entry.resolvedAt) > commitRequestOutcomeTTL { + if now.Sub(entry.resolvedAt) > commitRequestOutcomeTTL && w.crQueuedAttaches[k] == 0 { delete(w.crOutcomes, k) + w.crOwners.release(k, w) + } + } +} + +// countQueuedAttach records an attach entering the FIFO. The caller holds pendingResyncsMu, so the +// count is raised before the loop can receive the item. +func (w *BranchWorker) countQueuedAttach(id commitRequestID) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if w.crQueuedAttaches == nil { + w.crQueuedAttaches = map[commitRequestID]int{} + } + w.crQueuedAttaches[id]++ +} + +// uncountQueuedAttach records an attach leaving the FIFO: handled by the loop, dropped on a full +// queue, or drained by an exiting worker. A request the worker then knows nothing else about is +// given back, so the controller's next poll finds the worker its GitTarget names. +func (w *BranchWorker) uncountQueuedAttach(id commitRequestID) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if n := w.crQueuedAttaches[id]; n > 1 { + w.crQueuedAttaches[id] = n - 1 + return + } + delete(w.crQueuedAttaches, id) + _, resolved := w.crOutcomes[id] + _, registered := w.crPhases[id] + if !resolved && !registered { + w.crOwners.release(id, w) + } +} + +// settleCommitRequestsAfterExit runs once the event loop has returned: after a shutdown, after a +// retirement, or when the loop never started because its GitProvider could not be read. Either way +// nothing will act on a request again, and every request the worker knew must get an answer +// rather than wait for one that cannot come. +// +// - A request the worker held (attached, or committed) fails with ErrBranchWorkerStopped. A clean +// shutdown has already settled those through its last push; this catches what it could not. +// - A request it never acted on is given back. The old loop can no longer commit it, so the +// controller is free to send it to the worker the GitTarget names now, or to withdraw it. +// +// From then on a withdraw is answered at once (answerWithdrawAfterExit). +func (w *BranchWorker) settleCommitRequestsAfterExit() { + w.pendingResyncsMu.Lock() + w.stoppingState = true + w.pendingResyncsMu.Unlock() + + // A loop that never ran left its queue as the producers filled it. + w.drainQueue() + + w.crOutcomesMu.Lock() + for id, phase := range w.crPhases { + if phase.Held() { + w.recordCommitRequestOutcomeLocked(id, FinalizeResult{Branch: w.Branch, Err: ErrBranchWorkerStopped}) + } else { + w.crOwners.release(id, w) } + delete(w.crPhases, id) + } + w.crOutcomesMu.Unlock() + + w.pendingResyncsMu.Lock() + w.exitedState = true + w.pendingResyncsMu.Unlock() +} + +// answerWithdrawAfterExit resolves a withdraw sent to a worker whose loop has exited. Nothing can +// act on the request any more, so it is withdrawn for good, unless it already has an outcome. +func (w *BranchWorker) answerWithdrawAfterExit(id commitRequestID) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if _, resolved := w.crOutcomes[id]; resolved { + return } + w.recordCommitRequestOutcomeLocked(id, FinalizeResult{ + Branch: w.Branch, + Err: errors.Join(ErrCommitRequestWithdrawn, ErrBranchWorkerStopped), + }) } // LookupCommitRequestOutcome returns a resolved CommitRequest outcome, or ok=false @@ -96,6 +185,9 @@ func (w *BranchWorker) hasCommitRequestOutcome(id commitRequestID) bool { // an idempotent re-send restarts nothing and closes nothing. func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitRequest) { id := req.id() + // Uncounted only once the request is registered or resolved, so the worker never looks as if it + // had forgotten a request it is in the middle of taking. + defer l.w.uncountQueuedAttach(id) if l.w.hasCommitRequestOutcome(id) { return // already resolved: a late idempotent re-send. } diff --git a/internal/git/commit_request_owners.go b/internal/git/commit_request_owners.go new file mode 100644 index 00000000..132b2131 --- /dev/null +++ b/internal/git/commit_request_owners.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import "sync" + +// commitRequestOwners records which branch worker accepted each CommitRequest's attach, so the +// controller's questions about a request reach the worker that can still act on it. +// +// The GitTarget cannot answer that. A worker serves every GitTarget on its (provider, branch), so it +// outlives a deleted target, and a retired worker is detached from the manager before it stops, +// so its last push runs while the manager no longer lists it. Asking "which worker does the target +// name now?" in either case finds none, and reading that as "nothing holds the request" is how a +// request was reported withdrawn while its worker could still publish it. +// +// An entry is claimed when an attach enters a worker's queue, and released when that worker +// forgets the request: when its outcome is collected, or when the worker exits without having +// acted on it. A request with no entry is therefore held by no worker, and only then may the +// controller fall back to the GitTarget. +// +// It is shared by the manager and every worker it creates. The lock is a leaf: callers may hold +// a worker's pendingResyncsMu or crOutcomesMu when they take it, never the other way round. +type commitRequestOwners struct { + mu sync.Mutex + owners map[commitRequestID]*BranchWorker +} + +// owner returns the worker holding the request, if any. A nil table (a worker built outside a +// manager) knows no owners. +func (o *commitRequestOwners) owner(id commitRequestID) (*BranchWorker, bool) { + if o == nil { + return nil, false + } + o.mu.Lock() + defer o.mu.Unlock() + w, ok := o.owners[id] + return w, ok +} + +// claim makes w the request's owner, unless another worker already is. It reports whether w owns +// the request afterwards. A nil table lets every worker claim. +func (o *commitRequestOwners) claim(id commitRequestID, w *BranchWorker) bool { + if o == nil { + return true + } + o.mu.Lock() + defer o.mu.Unlock() + if current, ok := o.owners[id]; ok { + return current == w + } + if o.owners == nil { + o.owners = map[commitRequestID]*BranchWorker{} + } + o.owners[id] = w + return true +} + +// release forgets the request, if w is still its owner. +func (o *commitRequestOwners) release(id commitRequestID, w *BranchWorker) { + if o == nil { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if o.owners[id] == w { + delete(o.owners, id) + } +} diff --git a/internal/git/commit_request_withdraw_test.go b/internal/git/commit_request_withdraw_test.go index 71acf2a8..22609e8e 100644 --- a/internal/git/commit_request_withdraw_test.go +++ b/internal/git/commit_request_withdraw_test.go @@ -7,12 +7,17 @@ package git // holds (attached, or committed) is not, so a request reported failed is never committed later. import ( + "context" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" ) func TestWithdraw_AWaitingRequestIsCancelledForGood(t *testing.T) { @@ -120,6 +125,182 @@ func TestEnqueueWithdraw_QueueFullIsDroppedAndNilIsANoOp(t *testing.T) { assert.Zero(t, w.inflightItems.Load(), "a dropped withdraw must not leak an inflight count") } +// The withdrawn outcome is what turns a late attach into a no-op, so it must outlive every attach +// still on the FIFO, however long that attach waits: a TTL alone only moves the problem. +func TestWithdraw_AnOutcomeIsKeptWhileAnAttachForItIsQueued(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", time.Hour) + loop.handleQueueItem(WorkItem{Withdraw: req}) + worker.EnqueueAttach(req) // a late attach, still on the FIFO + + // The outcome runs past its TTL while the attach waits, and a GC pass runs. + ageCommitRequestOutcome(worker, req.id(), 2*commitRequestOutcomeTTL) + worker.recordCommitRequestOutcome(commitRequestID{Namespace: "default", Name: "other"}, FinalizeResult{}) + _, kept := outcome(t, worker) + require.True(t, kept, "an outcome with an attach still queued is not collected") + + loop.handleQueueItem(<-worker.eventQueue) + loop.serviceCommitRequests() + assert.Empty(t, loop.pendingCRs, "the late attach does not register the withdrawn request") + res, _ := outcome(t, worker) + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + + // With nothing queued for it any more, the outcome is collected as usual. + ageCommitRequestOutcome(worker, req.id(), 2*commitRequestOutcomeTTL) + worker.recordCommitRequestOutcome(commitRequestID{Namespace: "default", Name: "other"}, FinalizeResult{}) + _, kept = outcome(t, worker) + assert.False(t, kept) +} + +// A worker whose loop never ran, because its GitProvider could not be read, is still listed by the +// manager, so attaches and withdraws keep arriving. It must answer them rather than queue them for +// a loop that is gone. +func TestWithdraw_AWorkerThatExitedAtStartupAnswers(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, configv1alpha3.AddToScheme(scheme)) + worker := NewBranchWorker(fake.NewClientBuilder().WithScheme(scheme).Build(), logr.Discard(), + "no-such-provider", "default", "main", RepoIdentity{URL: "file:///nowhere"}, nil, BranchWorkerLimits{}) + owners := &commitRequestOwners{} + worker.crOwners = owners + + req := attachReq("alice", time.Hour) + worker.EnqueueAttach(req) // queued before the loop gives up + require.NoError(t, worker.Start(context.Background())) + worker.wg.Wait() // the loop has exited + + _, owned := owners.owner(req.id()) + assert.False(t, owned, "a request the worker never acted on is given back") + assert.Zero(t, worker.inflightItems.Load(), "the queue it never read is drained") + + worker.EnqueueAttach(req) + _, resolved := outcome(t, worker) + assert.False(t, resolved, "an attach is refused, not queued") + + worker.EnqueueWithdraw(req) + res, resolved := outcome(t, worker) + require.True(t, resolved, "a withdraw is answered at once") + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + require.ErrorIs(t, res.Err, ErrBranchWorkerStopped) + assert.Zero(t, worker.inflightItems.Load()) +} + +// A worker that is retired keeps answering for the request it held: its shutdown settles it, and +// that outcome stays reachable through the owner table after the manager stopped listing it. +// Reading "no worker" as "nothing holds it" is what reported such a request withdrawn. +func TestWithdraw_ARetiredWorkerStillAnswersForWhatItHeld(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + owners := &commitRequestOwners{} + worker.crOwners = owners + require.NoError(t, worker.Start(context.Background())) + + require.True(t, worker.Enqueue(configMapTargetEvent("held", "alice", "team-a"))) + req := attachReq("alice", time.Hour) + worker.EnqueueAttach(req) + require.Eventually(t, func() bool { + return worker.LookupCommitRequestPhase("default", crName, "uid-"+crName).Held() + }, 5*time.Second, 10*time.Millisecond) + + worker.Stop() + + owner, owned := owners.owner(req.id()) + require.True(t, owned, "the retired worker still holds the request") + assert.Same(t, worker, owner) + res, resolved := outcome(t, worker) + require.True(t, resolved, "its shutdown settled the request, by its last push or by failing it") + require.NotErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + + worker.EnqueueWithdraw(req) + after, _ := outcome(t, worker) + assert.Equal(t, res, after, "a withdraw after the fact keeps the real outcome") +} + +// A request still waiting for a window when its worker stops is given back: the old loop can no +// longer commit it, so the controller may send it to the GitTarget's next worker. +func TestWithdraw_AStoppedWorkerGivesBackWhatItNeverActedOn(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + owners := &commitRequestOwners{} + worker.crOwners = owners + require.NoError(t, worker.Start(context.Background())) + + req := attachReq("alice", time.Hour) + worker.EnqueueAttach(req) + require.Eventually(t, func() bool { + return worker.LookupCommitRequestPhase("default", crName, "uid-"+crName) == PhaseWaitingForWindow + }, 5*time.Second, 10*time.Millisecond) + + worker.Stop() + + _, owned := owners.owner(req.id()) + assert.False(t, owned) + assert.Empty(t, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + _, resolved := outcome(t, worker) + assert.False(t, resolved, "giving it back is not an outcome") +} + +func TestCommitRequestOwners_OneOwnerAtATime(t *testing.T) { + a, b := &BranchWorker{}, &BranchWorker{} + id := commitRequestID{Namespace: "default", Name: crName} + owners := &commitRequestOwners{} + + require.True(t, owners.claim(id, a)) + assert.True(t, owners.claim(id, a), "claiming again is idempotent") + assert.False(t, owners.claim(id, b), "a second worker cannot take a held request") + owners.release(id, b) + got, ok := owners.owner(id) + require.True(t, ok, "only the owner can release") + assert.Same(t, a, got) + owners.release(id, a) + assert.True(t, owners.claim(id, b)) + + var none *commitRequestOwners + assert.True(t, none.claim(id, a), "a worker outside a manager owns what it is sent") + _, ok = none.owner(id) + assert.False(t, ok) + none.release(id, a) +} + +func TestEnqueueAttach_AnotherWorkersRequestIsRefused(t *testing.T) { + owners := &commitRequestOwners{} + holder := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1), crOwners: owners} + other := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1), crOwners: owners} + req := attachReq("alice", 0) + + holder.EnqueueAttach(req) + other.EnqueueAttach(req) + + assert.Len(t, holder.eventQueue, 1) + assert.Empty(t, other.eventQueue, "the request belongs to the worker that accepted it first") + assert.Zero(t, other.inflightItems.Load()) +} + +// A dropped attach gives the request back unless the worker knows it otherwise. +func TestEnqueueAttach_AQueueFullDropGivesTheRequestBack(t *testing.T) { + owners := &commitRequestOwners{} + w := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1), crOwners: owners} + w.eventQueue <- WorkItem{} // saturate + + req := attachReq("alice", 0) + w.EnqueueAttach(req) + + _, owned := owners.owner(req.id()) + assert.False(t, owned) + assert.Empty(t, w.crQueuedAttaches) +} + +func ageCommitRequestOutcome(w *BranchWorker, id commitRequestID, by time.Duration) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + entry := w.crOutcomes[id] + entry.resolvedAt = entry.resolvedAt.Add(-by) + w.crOutcomes[id] = entry +} + func TestCommitRequestPhase_Held(t *testing.T) { assert.True(t, PhaseCollectingWindow.Held()) assert.True(t, PhaseWaitingForPush.Held()) diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index bbbed976..f3e015eb 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -95,7 +95,11 @@ type WorkerManager struct { // the scrape goroutine. Take it BEFORE mu, never the other way round. lifecycleMu sync.Mutex workers map[BranchKey]*BranchWorker - ctx context.Context + // commitRequests records which worker holds each CommitRequest. It outlives the map above on + // purpose: a retired worker leaves the map before it stops, and keeps answering for the + // requests it held. See commitRequestOwners. + commitRequests *commitRequestOwners + ctx context.Context // mapper is the GVK->GVR resolver injected into every worker so store scans build a // resource-identity inventory. It is set once at startup (SetMapper) before any // worker is created; a nil mapper keeps workers structure-only. It is the LOCAL cluster's @@ -168,6 +172,7 @@ func NewWorkerManager( limits: limits.withDefaults(), sensitiveResources: sensitiveResources, workers: make(map[BranchKey]*BranchWorker), + commitRequests: &commitRequestOwners{}, remotes: make(map[BranchKey]RemoteObservation), replacements: make(map[BranchKey]struct{}), renderFidelityGate: NewRenderFidelityGate(), @@ -358,6 +363,7 @@ func (m *WorkerManager) EnsureWorker( worker.remoteReporter = func(observed RemoteObservation) { m.recordRemoteObservation(key, observed) } worker.scanAcceptance = m.scanAcceptance worker.renderFidelityGate = m.renderFidelityGate + worker.crOwners = m.commitRequests if err := worker.Start(m.ctx); err != nil { return fmt.Errorf("failed to start worker for %s: %w", key.String(), err) @@ -467,6 +473,14 @@ func (m *WorkerManager) GetWorkerForTarget( return worker, exists } +// CommitRequestOwner returns the worker that holds a CommitRequest: the one that accepted its +// attach and has not given it back. It is found whatever has happened to the request's GitTarget +// since, including a worker retired and still finishing its last push, so a caller that finds no +// owner knows that no worker can still commit the request. +func (m *WorkerManager) CommitRequestOwner(namespace, name, uid string) (*BranchWorker, bool) { + return m.commitRequests.owner(commitRequestID{Namespace: namespace, Name: name, UID: uid}) +} + // afterOrphanSelection runs between choosing the workers to retire and retiring them. It is nil in // production and exists because that gap is the ONLY place the lock discipline in ReconcileWorkers // can be observed: with lifecycleMu held across both steps a concurrent EnsureWorker blocks here, diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index c7e100a3..542baccd 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -59,7 +59,7 @@ func NewEventRouter( } // ServiceCommitRequest is the controller's attach-then-poll seam (§6.4.3): it -// resolves the GitTarget's branch worker, registers the CommitRequest attach +// resolves the worker that holds the request, or else the GitTarget's branch worker, registers the CommitRequest attach // idempotently on that worker's FIFO event queue (attach it to the author's window // with its message and timers), and returns the request's current // outcome. resolved=false means the worker has not finished — the controller @@ -77,24 +77,14 @@ func (r *EventRouter) ServiceCommitRequest( ctx context.Context, attach git.AttachCommitRequest, ) (git.FinalizeResult, bool, error) { - var gitTarget configv1alpha3.GitTarget - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: attach.GitTargetName, - Namespace: attach.GitTargetNamespace, - }, &gitTarget); err != nil { - return git.FinalizeResult{}, false, fmt.Errorf("get GitTarget %s/%s: %w", - attach.GitTargetNamespace, attach.GitTargetName, err) + worker, branch, err := r.commitRequestWorker(ctx, attach) + if err != nil { + return git.FinalizeResult{}, false, err } - - worker, exists := r.WorkerManager.GetWorkerForTarget( - gitTarget.Spec.GitProviderRef.Name, - gitTarget.Namespace, // provider is in the same namespace as the target - gitTarget.Spec.Branch, - ) - if !exists { + if worker == nil { r.Log.V(1).Info("ServiceCommitRequest: no branch worker for the GitTarget yet; will retry", "gitTarget", attach.GitTargetNamespace+"/"+attach.GitTargetName) - return git.FinalizeResult{Branch: gitTarget.Spec.Branch, Phase: git.PhaseWaitingForWorker}, false, nil + return git.FinalizeResult{Branch: branch, Phase: git.PhaseWaitingForWorker}, false, nil } // Idempotent register (the worker keys by request identity and keeps the first @@ -113,50 +103,83 @@ func (r *EventRouter) ServiceCommitRequest( // not acted on, and reports the worker's answer, so the controller never fails a request the worker // could still commit. // -// - No worker for the GitTarget: nothing holds the request and nothing can commit it, so it -// resolves at once as withdrawn. The controller stops sending attaches, so a worker that starts -// later never sees it. -// - A worker: the withdraw rides the FIFO behind every attach already sent, and is a no-op for a -// request the worker holds. Until it is handled, resolved=false and Phase says where the -// request stands; a held phase means the controller keeps waiting. +// - The worker that holds the request answers for it, even when its GitTarget is gone or the +// worker is being retired. The withdraw rides its FIFO behind every attach already sent, and +// is a no-op for a request it holds. Until it is handled, resolved=false and Phase says where +// the request stands; a held phase means the controller keeps waiting. +// - Nothing holds it, and the GitTarget is gone or names no worker: nothing can commit the +// request, so it resolves at once as withdrawn. The controller sends no attach after it starts +// withdrawing, so a worker that starts later never sees it. func (r *EventRouter) WithdrawCommitRequest( ctx context.Context, attach git.AttachCommitRequest, ) (git.FinalizeResult, bool, error) { + worker, branch, err := r.commitRequestWorker(ctx, attach) + if errors.Is(err, errGitTargetGone) { + // No worker holds the request, and its GitTarget can never start one. + return git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, true, nil + } + if err != nil { + return git.FinalizeResult{}, false, err + } + if worker == nil { + return git.FinalizeResult{ + Branch: branch, + Phase: git.PhaseWaitingForWorker, + Err: git.ErrCommitRequestWithdrawn, + }, true, nil + } + + if result, resolved := worker.LookupCommitRequestOutcome(attach.Namespace, attach.Name, attach.UID); resolved { + return result, true, nil + } + phase := worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID) + if phase.Held() { + return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil + } + worker.EnqueueWithdraw(&attach) + // A worker whose loop has exited answers the withdraw at once. + if result, resolved := worker.LookupCommitRequestOutcome(attach.Namespace, attach.Name, attach.UID); resolved { + return result, true, nil + } + return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil +} + +// commitRequestWorker finds the worker to ask about a CommitRequest, and the branch it serves. +// +// The worker that holds the request comes first, whatever its GitTarget says now: a shared worker +// outlives a deleted target, and a retired one finishes its last push after the manager stopped +// listing it. Only a request no worker holds falls back to the GitTarget's current worker, which is +// nil while that worker has not started. A GitTarget that is gone is errGitTargetGone. +func (r *EventRouter) commitRequestWorker( + ctx context.Context, + attach git.AttachCommitRequest, +) (*git.BranchWorker, string, error) { + if owner, ok := r.WorkerManager.CommitRequestOwner(attach.Namespace, attach.Name, attach.UID); ok { + return owner, owner.Branch, nil + } + var gitTarget configv1alpha3.GitTarget if err := r.Client.Get(ctx, client.ObjectKey{ Name: attach.GitTargetName, Namespace: attach.GitTargetNamespace, }, &gitTarget); err != nil { if apierrors.IsNotFound(err) { - // A deleted GitTarget has no worker left to hold the request. - return git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, true, nil + return nil, "", fmt.Errorf("%w: %s/%s", errGitTargetGone, attach.GitTargetNamespace, attach.GitTargetName) } - return git.FinalizeResult{}, false, fmt.Errorf("get GitTarget %s/%s: %w", + return nil, "", fmt.Errorf("get GitTarget %s/%s: %w", attach.GitTargetNamespace, attach.GitTargetName, err) } worker, exists := r.WorkerManager.GetWorkerForTarget( gitTarget.Spec.GitProviderRef.Name, - gitTarget.Namespace, + gitTarget.Namespace, // provider is in the same namespace as the target gitTarget.Spec.Branch, ) if !exists { - return git.FinalizeResult{ - Branch: gitTarget.Spec.Branch, - Phase: git.PhaseWaitingForWorker, - Err: git.ErrCommitRequestWithdrawn, - }, true, nil - } - - if result, resolved := worker.LookupCommitRequestOutcome(attach.Namespace, attach.Name, attach.UID); resolved { - return result, true, nil + return nil, gitTarget.Spec.Branch, nil } - worker.EnqueueWithdraw(&attach) - return git.FinalizeResult{ - Branch: gitTarget.Spec.Branch, - Phase: worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID), - }, false, nil + return worker, gitTarget.Spec.Branch, nil } // recordBackgroundResyncFailure counts a fire-and-forget resync whose apply failed or diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index 4593bb5a..9a54e55a 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -58,7 +58,7 @@ func TestServiceCommitRequest_GitTargetNotFound(t *testing.T) { _, resolved, err := router.ServiceCommitRequest(context.Background(), saveAttach("missing", "team-a")) require.Error(t, err) assert.False(t, resolved) - assert.Contains(t, err.Error(), "get GitTarget") + require.ErrorIs(t, err, errGitTargetGone) } func TestGitTargetEventStreamRegistry(t *testing.T) { @@ -456,6 +456,61 @@ func TestWithdrawCommitRequest_ALiveWorkerCancelsAWaitingRequest(t *testing.T) { require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) } +// TestWithdrawCommitRequest_ADeletedGitTargetsWorkerStillAnswers: a worker is shared by every +// GitTarget on its branch, so it outlives a deleted one. The withdraw must reach that worker, and +// must never report success while the request is still registered there. +func TestWithdrawCommitRequest_ADeletedGitTargetsWorkerStillAnswers(t *testing.T) { + scheme := eventRouterScheme(t) + provider := &configv1alpha3.GitProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-provider", Namespace: "team-a"}, + Spec: configv1alpha3.GitProviderSpec{URL: "file:///tmp/does-not-need-to-exist"}, + } + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "team-a-provider"}, + Branch: "main", + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(provider, gitTarget).Build() + workerManager := git.NewWorkerManager( + client, logr.Discard(), git.BranchWorkerLimits{}, types.SensitiveResourcePolicy{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = workerManager.Start(ctx) }() + time.Sleep(100 * time.Millisecond) // allow the manager to record its context + require.NoError(t, workerManager.EnsureWorker( + ctx, "team-a-provider", "team-a", "main", git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, "")) + worker, ok := workerManager.GetWorkerForTarget("team-a-provider", "team-a", "main") + require.True(t, ok) + + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + attach := saveAttach("team-a-config", "team-a") + attach.AttachTimeout = time.Hour + _, _, err := router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err) + require.Eventually(t, func() bool { + return worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID) == git.PhaseWaitingForWindow + }, 5*time.Second, 10*time.Millisecond) + + require.NoError(t, client.Delete(context.Background(), gitTarget)) + + var result git.FinalizeResult + require.Eventually(t, func() bool { + var resolved bool + result, resolved, err = router.WithdrawCommitRequest(context.Background(), attach) + require.NoError(t, err) + if resolved { + require.Empty(t, worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID), + "the withdraw reported success while the worker still held the registration") + } + return resolved + }, 5*time.Second, 50*time.Millisecond) + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + assert.Equal(t, "main", result.Branch, "the worker answered, not the missing GitTarget") +} + // TestDrainScopedResync_QueueFullIsDrainedNotOrphaned guards the ordering invariant behind // Failure A/B: EnqueueResync answers a dropped request on its reply channel, so the drain MUST be // started even when the request never entered the FIFO. From 3fb6666734bcb3eb638090e964fb568762b6a4c0 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:16:52 +0000 Subject: [PATCH 12/18] fix(git): scope base trust to the parent generation and admit a push only on its current root SetParentBranch ran on the controller goroutine, dropped trust once and raised a flag the event loop could undo or bypass: a push or fetch finishing after the change set trust again, the remembered new-branch parent then rooted the next write on the old parent, and a change landing between the rebuild and the push was pushed on the old root. - The parent configuration is one immutable {name, gen} snapshot. Every fetch loads it once and records what it actually used; nothing re-reads the name after the network call. - Trust is scoped to the generation the checkout was proven under, so a late setBaseTrusted(true) cannot vouch for a newer configuration, and the remembered new-branch parent counts only under the current one. - runPushCycle checks, under repoMu right before each push, that the root was chosen under the current generation; otherwise it replays first, with no extra connection. A push already admitted is not recalled: a branch it creates from the old parent stays, and an existing branch does not follow its parent. - A cycle rooted on an existing write branch is re-stamped without a fetch, because a parent cannot move it. SetParentBranch therefore no longer invalidates the base; the generation does. - A failed rebuild leaves the generations apart, which replaces the flag hand-back of dcbc8b82. Five of the seven new tests fail on the previous worker, including the review's no-op-push reproduction; the other two pin the in-flight rule. Co-Authored-By: Claude Opus 5.5 --- .coverage-baseline | 2 +- internal/git/branch_worker.go | 250 ++++++++++++---- .../git/branch_worker_parent_branch_test.go | 7 +- .../git/branch_worker_parent_change_test.go | 282 ++++++++++++++++++ internal/git/git.go | 2 +- internal/git/worker_manager.go | 5 +- 6 files changed, 478 insertions(+), 70 deletions(-) create mode 100644 internal/git/branch_worker_parent_change_test.go diff --git a/.coverage-baseline b/.coverage-baseline index fa863583..068d2ef9 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -79.2 +79.3 diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 9bb3b8ac..dcb79c02 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -57,6 +57,11 @@ var ( fetchRemoteBranchHashFn = fetchRemoteBranchHash //nolint:gochecknoglobals syncToRemoteFn = syncToRemoteFrom + // beforePushAdmission runs right before a push cycle compares its root's parent generation + // with the current one. Nil in production; a test changes the parent here to land a change + // just before admission. + //nolint:gochecknoglobals + beforePushAdmission func() ) // BranchWorker processes events for a single (GitProvider, Branch) combination. @@ -243,14 +248,18 @@ type BranchWorker struct { newBranchParent plumbing.ReferenceName newBranchParentHash plumbing.Hash - // parentBranchName is the GitTarget's spec.parentBranch: the branch a write branch the remote - // does not carry is created from. Empty means the remote's default branch. It is set by the - // reconcile and read by the event loop, hence atomic. - parentBranchName atomic.Pointer[string] - // parentChangedState is raised by SetParentBranch and consumed by the event loop: writes it - // retains were planned against the old parent, and invalidating base trust alone does not - // reach them (ensureBaseForCycle ignores trust while writes are retained). - parentChangedState atomic.Bool + // parentState is the GitTarget's spec.parentBranch, with a generation that SetParentBranch + // raises on every change. It is set by the reconcile and read by the event loop, hence atomic. + // See parentConfig. + parentState atomic.Pointer[parentConfig] + // baseParentGen is the generation of the parent configuration the checkout was last proven + // against: the snapshot whose fetch produced it, or the current one once the checkout is known + // not to depend on the parent (the write branch exists). Trust is scoped to it, so a push or a + // fetch that finishes after a change can never vouch for the new configuration. + baseParentGen atomic.Uint64 + // pushCycleRootGen is the parent generation pushCycleRootBranch was chosen under. The push + // admission check compares it with the current one. Protected by repoMu. + pushCycleRootGen uint64 // firsts surfaces the first successful commit and push at default verbosity. firsts branchWorkerLogFirsts @@ -1478,9 +1487,6 @@ func (l *branchWorkerEventLoop) resetCommitTimer(delay time.Duration) { // The case only arises with retained work. With nothing retained, commitPendingWrites' own guard // resets for us, and it can do that safely because there is nothing to lose. func (l *branchWorkerEventLoop) recoverRetainedWrites() error { - // Taken first, so a change that found nothing retained does not linger: with nothing retained - // the invalidated base makes the next cycle fetch the new parent anyway. - parentChanged := l.w.parentChangedState.Swap(false) if len(l.pendingWrites) == 0 { // Nothing is retained, so there is nothing a stale replay could strand. A replay only // ever marks itself required while writes are retained, but clear it here too so the flag @@ -1488,7 +1494,9 @@ func (l *branchWorkerEventLoop) recoverRetainedWrites() error { l.w.markReplayComplete() return nil } - dirty, needsReplay := l.w.worktreeDirty(), l.w.replayRequired() + // A parent change is read from the generations, not taken from a flag: a rebuild that fails + // leaves them apart, so the next attempt rebuilds again. + dirty, needsReplay, parentChanged := l.w.worktreeDirty(), l.w.replayRequired(), l.w.rootParentStale() if !dirty && !needsReplay && !parentChanged { return nil } @@ -1501,15 +1509,7 @@ func (l *branchWorkerEventLoop) recoverRetainedWrites() error { // fetchReasonRecovery, the same series the no-retained-writes case records in // ensureBaseForCycle: this is one event, and which half of it an operator sees must not depend // on whether a push happened to be in cooldown at the time. - err := l.invalidateAndRefresh("retained writes cannot be trusted", fetchReasonRecovery) - if err != nil && parentChanged { - // The rebuild can fail before its reset marks a replay required (reading the GitProvider or - // the credentials, say), and then nothing else remembers that the retained writes sit on - // the old parent: the next attempt would publish from it. Hand the change back until a - // rebuild succeeds. - l.w.parentChangedState.Store(true) - } - return err + return l.invalidateAndRefresh("retained writes cannot be trusted", fetchReasonRecovery) } // invalidateAndRefresh drops base trust and, when writes are retained, acts on that invalidation @@ -1839,7 +1839,13 @@ func (l *branchWorkerEventLoop) stopRefusalTimer() { // // The failure direction is the safe one. A stale false costs one fetch. A stale true is caught by // the compare-and-swap on the next push, which is the mechanism the write path already relies on. -func (w *BranchWorker) baseTrusted() bool { return w.baseTrustedState.Load() } +// +// Trust is scoped to the parent configuration: a checkout proven under an older one is not +// trusted, whatever the flag says, because the flag can be set by a push or a fetch that started +// before the change. +func (w *BranchWorker) baseTrusted() bool { + return w.baseTrustedState.Load() && w.baseParentGen.Load() == w.parentSnapshot().gen +} // worktreeDirty reports whether the worktree may hold a partial write. // @@ -1925,7 +1931,11 @@ func (w *BranchWorker) ensureBaseForCycle( repoPath string, hasPendingCommits bool, ) error { - if hasPendingCommits || (w.baseTrusted() && !w.worktreeDirty()) { + if hasPendingCommits { + return nil + } + w.restampParentGenIfIndependent() + if w.baseTrusted() && !w.worktreeDirty() { return nil } @@ -1995,6 +2005,7 @@ func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite, hasPend if !hasPendingCommits { w.pushCycleRootBranch = baseBranch w.pushCycleRootHash = baseHash + w.pushCycleRootGen = w.baseParentGen.Load() } commitsCreated, err := w.executePendingWrites(w.ctx, repo, pendingWrites) @@ -2064,15 +2075,32 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { } const maxRetries = 3 - rootHash := w.pushCycleRootHash var lastErr error for range maxRetries { + // Re-read every attempt: a replay below re-roots the cycle. + rootHash := w.pushCycleRootHash rootBranch := w.pushCycleRootBranch if rootBranch == "" { rootBranch = plumbing.NewBranchReferenceName(w.Branch) } + // The admission boundary for a parent change: compared here, under repoMu and right before + // the push, so a change made after the cycle was planned never leaves on the old root. A + // change after this point is in flight, and the push is not recalled. + if beforePushAdmission != nil { + beforePushAdmission() + } + if !w.admitPushRoot(rootBranch) { + w.Log.Info("Parent branch changed since this cycle was planned; rebuilding before the push", + "branch", w.Branch, "parentBranch", w.ParentBranch()) + w.recordFetch(fetchReasonRecovery) + if err := w.replayOntoRemote(repo, pendingWrites, auth); err != nil { + return err + } + continue + } + outcome, err := pushAtomicFn(w.ctx, repo, rootHash, rootBranch, auth) if err == nil { // A push that returns without an error is an OBSERVATION of the remote, on the @@ -2109,34 +2137,95 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { return err } w.recordPushRetry(pushRetryRemoteMoved) - - // Marked BEFORE the reset, not after it. A reset moves the branch ref and then rewrites - // the worktree, so it can fail with the ref already moved and the commits behind these - // writes already unreachable. Marking afterwards misses exactly that case, and the cost of - // marking too eagerly is one fetch on the next cycle if the reset turns out to have - // changed nothing. - w.markReplayRequired() w.recordFetch(fetchReasonContention) - pullReport, syncErr := w.syncToRemote(w.ctx, repo, auth) - if syncErr != nil { - w.invalidateBase("sync during replay failed") - return fmt.Errorf("sync remote during replay: %w", syncErr) - } - w.updateBranchMetadataFromPullReport(pullReport) - - rootBranch, rootHash, err = w.rebuildPendingWrites(repo, pendingWrites) - if err != nil { - w.invalidateBase("replay rebuild failed") + if err := w.replayOntoRemote(repo, pendingWrites, auth); err != nil { return err } - w.pushCycleRootBranch = rootBranch - w.pushCycleRootHash = rootHash - w.markReplayComplete() } + if lastErr == nil { + return fmt.Errorf("push not admitted after %d attempts: the parent branch kept changing", maxRetries) + } return fmt.Errorf("push failed after %d attempts: %w", maxRetries, lastErr) } +// replayOntoRemote syncs to the remote, rebuilds the pending writes on what it found, and makes +// that the cycle's root. The caller owns repoMu and has recorded the fetch. +func (w *BranchWorker) replayOntoRemote( + repo *gogit.Repository, + pendingWrites []PendingWrite, + auth []gitclient.Option, +) error { + // Marked BEFORE the reset, not after it. A reset moves the branch ref and then rewrites + // the worktree, so it can fail with the ref already moved and the commits behind these + // writes already unreachable. Marking afterwards misses exactly that case, and the cost of + // marking too eagerly is one fetch on the next cycle if the reset turns out to have + // changed nothing. + w.markReplayRequired() + pullReport, err := w.syncToRemote(w.ctx, repo, auth) + if err != nil { + w.invalidateBase("sync during replay failed") + return fmt.Errorf("sync remote during replay: %w", err) + } + w.updateBranchMetadataFromPullReport(pullReport) + + rootBranch, rootHash, err := w.rebuildPendingWrites(repo, pendingWrites) + if err != nil { + w.invalidateBase("replay rebuild failed") + return err + } + w.pushCycleRootBranch = rootBranch + w.pushCycleRootHash = rootHash + w.pushCycleRootGen = w.baseParentGen.Load() + w.markReplayComplete() + return nil +} + +// admitPushRoot reports whether a push rooted on rootBranch may leave under the current parent +// configuration: the root was chosen under it, or the root is the write branch itself, which a +// parent change cannot move. In that second case it re-stamps the root, at no cost. The caller +// owns repoMu. +func (w *BranchWorker) admitPushRoot(rootBranch plumbing.ReferenceName) bool { + gen := w.parentSnapshot().gen + if w.pushCycleRootGen == gen { + return true + } + if rootBranch != plumbing.NewBranchReferenceName(w.Branch) || !w.writeBranchProvenPresent() { + return false + } + w.pushCycleRootGen = gen + w.restampParentGenIfIndependent() + return true +} + +// writeBranchProvenPresent reports whether the last observation of the remote found the write +// branch. A parent change cannot alter a cycle rooted on an existing write branch, and the push's +// compare-and-swap still guards the branch if the observation went stale. +func (w *BranchWorker) writeBranchProvenPresent() bool { + observed := w.lastObservation.Load() + return observed != nil && observed.Commit != "" +} + +// restampParentGenIfIndependent carries trust across a parent change when the checkout does not +// depend on the parent: the write branch exists, and HEAD is not a local branch still rooted on a +// parent. It costs no connection, which is what keeps an existing write branch's cycles free. +func (w *BranchWorker) restampParentGenIfIndependent() { + if w.newBranchParent == "" && w.writeBranchProvenPresent() { + w.baseParentGen.Store(w.parentSnapshot().gen) + } +} + +// rootParentStale reports whether the retained cycle's root was chosen under an older parent +// configuration, and the root depends on the parent. +func (w *BranchWorker) rootParentStale() bool { + w.repoMu.Lock() + defer w.repoMu.Unlock() + if w.pushCycleRootGen == w.parentSnapshot().gen { + return false + } + return w.pushCycleRootBranch != plumbing.NewBranchReferenceName(w.Branch) || !w.writeBranchProvenPresent() +} + // remoteMovedDuringPush reports whether a failed push was contention — the remote branch no // longer sitting at the hash our commits were based on — which is the only push failure a replay // can fix. @@ -2281,6 +2370,7 @@ func (w *BranchWorker) refreshRemoteAndRebuildPendingWrites( } w.pushCycleRootBranch = rootBranch w.pushCycleRootHash = rootHash + w.pushCycleRootGen = w.baseParentGen.Load() w.markReplayComplete() return nil } @@ -2409,8 +2499,12 @@ func (w *BranchWorker) ensureWriteBranch(repo *gogit.Repository) (plumbing.Refer } // Already on the local write branch, with nothing committed on top of the parent it was - // created from: the remote still does not carry it, so the parent remains the root. - if w.newBranchParent != "" && baseHash == w.newBranchParentHash { + // created from: the remote still does not carry it, so the parent remains the root. Only while + // the parent configuration is the one that checkout was made under: after a change, the + // remembered parent is the old one, and rooting on it would let the push create the branch + // there. The root then names the local branch, which the admission check refuses. + if w.newBranchParent != "" && baseHash == w.newBranchParentHash && + w.baseParentGen.Load() == w.parentSnapshot().gen { return w.newBranchParent, w.newBranchParentHash, nil } @@ -2446,25 +2540,42 @@ func (w *BranchWorker) endPushCycle(kind PushOutcomeKind) { } } -// ParentBranch is the configured parent branch, or empty for the remote's default branch. -func (w *BranchWorker) ParentBranch() string { - if name := w.parentBranchName.Load(); name != nil { - return *name +// parentConfig is one immutable snapshot of the parent configuration. Every operation that +// consults the parent loads it once, at its start, and uses that name and generation throughout: +// nothing re-reads the current name after a network call returns, so an old lookup can never +// report a result about a newly configured parent. +type parentConfig struct { + // name is spec.parentBranch; empty means the remote's default branch. + name string + // gen rises on every change. Zero is the configuration a worker starts with. + gen uint64 +} + +// parentSnapshot returns the current parent configuration. +func (w *BranchWorker) parentSnapshot() parentConfig { + if p := w.parentState.Load(); p != nil { + return *p } - return "" + return parentConfig{} } -// SetParentBranch configures the parent branch. A change invalidates the base, so the next cycle -// fetches and compares against the new parent, and the event loop rebuilds retained writes on it; -// the worker and its clone are kept. spec.parentBranch is immutable, so a change reaches a live -// worker only when the targets on its branch were replaced. It reports whether the value changed. +// ParentBranch is the configured parent branch, or empty for the remote's default branch. +func (w *BranchWorker) ParentBranch() string { return w.parentSnapshot().name } + +// SetParentBranch configures the parent branch, and reports whether the value changed. A change +// raises the generation, which is what every later decision compares against: trust earned under +// the old one no longer counts, a cycle planned under it is rebuilt before its push is admitted, +// and retained writes are replayed on the new parent. The worker and its clone are kept, and an +// existing write branch, which a parent cannot move, keeps its trust at no cost. +// spec.parentBranch is immutable, so a change reaches a live worker only when the targets on its +// branch were replaced. func (w *BranchWorker) SetParentBranch(name string) bool { - if w.ParentBranch() == name { + current := w.parentSnapshot() + if current.name == name { return false } - w.parentBranchName.Store(&name) - w.invalidateBase("parent branch changed") - w.parentChangedState.Store(true) + w.parentState.Store(&parentConfig{name: name, gen: current.gen + 1}) + w.Log.V(1).Info("Parent branch changed", "branch", w.Branch, "parentBranch", name) return true } @@ -2474,8 +2585,9 @@ func (w *BranchWorker) SetParentBranch(name string) bool { func (w *BranchWorker) prepareBranch( ctx context.Context, repoPath string, auth []gitclient.Option, ) (*PullReport, error) { - report, err := PrepareBranchFrom(ctx, w.repo.URL, repoPath, w.Branch, w.ParentBranch(), auth) - w.noteMissingParent(err) + parent := w.parentSnapshot() + report, err := PrepareBranchFrom(ctx, w.repo.URL, repoPath, w.Branch, parent.name, auth) + w.noteFetchParent(parent, err) return report, err } @@ -2483,14 +2595,22 @@ func (w *BranchWorker) prepareBranch( func (w *BranchWorker) syncToRemote( ctx context.Context, repo *gogit.Repository, auth []gitclient.Option, ) (*PullReport, error) { - report, err := syncToRemoteFn(ctx, repo, plumbing.NewBranchReferenceName(w.Branch), w.ParentBranch(), auth) - w.noteMissingParent(err) + parent := w.parentSnapshot() + report, err := syncToRemoteFn(ctx, repo, plumbing.NewBranchReferenceName(w.Branch), parent.name, auth) + w.noteFetchParent(parent, err) return report, err } -func (w *BranchWorker) noteMissingParent(err error) { +// noteFetchParent records what a fetch under one parent snapshot proved: on success, that the +// checkout now matches that configuration; on a missing parent, the observation, naming the +// parent the fetch actually asked for. +func (w *BranchWorker) noteFetchParent(parent parentConfig, err error) { + if err == nil { + w.baseParentGen.Store(parent.gen) + return + } if errors.Is(err, ErrParentBranchNotFound) { - w.recordMissingParent(w.ParentBranch(), ObservedByFetch) + w.recordMissingParent(parent.name, ObservedByFetch) } } diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index 5d68c32b..fc915874 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -11,9 +11,11 @@ import ( "context" "errors" "path/filepath" + "strings" "testing" "time" + gogit "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,7 +33,10 @@ func (f *newBranchFixture) pushToRelease(file, content string) plumbing.Hash { func (f *newBranchFixture) commitParent(tip plumbing.Hash) plumbing.Hash { f.t.Helper() - commit, err := f.server.CommitObject(tip) + // Opened afresh: a handle opened before the pushes can miss a packfile they added. + server, err := gogit.PlainOpen(strings.TrimPrefix(f.remote, "file://")) + require.NoError(f.t, err) + commit, err := server.CommitObject(tip) require.NoError(f.t, err) require.Len(f.t, commit.ParentHashes, 1) return commit.ParentHashes[0] diff --git a/internal/git/branch_worker_parent_change_test.go b/internal/git/branch_worker_parent_change_test.go new file mode 100644 index 00000000..590e13ed --- /dev/null +++ b/internal/git/branch_worker_parent_change_test.go @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// A parent change reaches a live worker from the reconcile goroutine while the event loop may be in +// the middle of a fetch or a push. These pin the rules that make that safe: every operation uses +// one snapshot of the parent configuration, trust is scoped to the snapshot's generation, and a +// push is admitted only when its root was chosen under the current generation. A push already +// admitted is not recalled. + +import ( + "context" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// allFetchReasons is every series a fetch can be counted under. +var allFetchReasons = []string{ + fetchReasonBootstrap, fetchReasonPublication, fetchReasonRecovery, fetchReasonContention, + fetchReasonPushFailureProbe, fetchReasonRefresh, +} + +func totalFetches(t *testing.T, reader *sdkmetric.ManualReader, worker *BranchWorker) int64 { + t.Helper() + var total int64 + for _, reason := range allFetchReasons { + total += fetchCount(t, reader, worker, reason) + } + return total +} + +// onSync runs hook after the n-th sync (1-based) the worker makes, inside that fetch. +func onSync(t *testing.T, n int, hook func()) { + t.Helper() + original := syncToRemoteFn + calls := 0 + syncToRemoteFn = func(ctx context.Context, repo *gogit.Repository, target plumbing.ReferenceName, + parent string, auth []gitclient.Option) (*PullReport, error) { + report, err := original(ctx, repo, target, parent, auth) + calls++ + if calls == n { + hook() + } + return report, err + } + t.Cleanup(func() { syncToRemoteFn = original }) +} + +// pushRoots records the root of every push attempt, and runs hook inside the first one, before it +// reaches the remote. +func pushRoots(t *testing.T, hook func()) *[]plumbing.ReferenceName { + t.Helper() + original := pushAtomicFn + var roots []plumbing.ReferenceName + pushAtomicFn = func(ctx context.Context, repo *gogit.Repository, rootHash plumbing.Hash, + rootBranch plumbing.ReferenceName, auth []gitclient.Option) (PushOutcome, error) { + roots = append(roots, rootBranch) + if len(roots) == 1 && hook != nil { + hook() + } + return original(ctx, repo, rootHash, rootBranch, auth) + } + t.Cleanup(func() { pushAtomicFn = original }) + return &roots +} + +// observations collects every observation the worker reports. +func observations(f *newBranchFixture) *[]RemoteObservation { + var seen []RemoteObservation + f.worker.remoteReporter = func(o RemoteObservation) { seen = append(seen, o) } + return &seen +} + +// Test 1: a change that lands inside a fetch. The fetch finishes under the old snapshot, so its +// observation names the parent it used, and the push that follows is not admitted on that root. +func TestBranchWorker_AParentChangeDuringAFetchIsNotPushedOnTheOldParent(t *testing.T) { + t.Run("the initial fetch", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + seen := observations(f) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + onSync(t, 1, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip), "the first commit sits on the new parent") + require.NotEmpty(t, *seen) + assert.Equal(t, "main", (*seen)[0].ParentBranch, "the fetch reports the parent it actually used") + }) + + t.Run("a contention fetch", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + f.pushToMain(map[string]string{"MOVED.md": "main moved\n"}) // the push is refused: contention + seen := observations(f) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + onSync(t, 1, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) + require.NotEmpty(t, *seen) + assert.Equal(t, "main", (*seen)[0].ParentBranch, "the contention fetch reports the parent it used") + }) +} + +// Test 2, the review's reproduction: a change inside a no-op push that settles as PushNoBranch. +// The push sets trust after the change, and the remembered parent is the old one; neither may +// root the next real write. +func TestBranchWorker_AParentChangeDuringANoOpPushIsTakenByTheNextWrite(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + roots := pushRoots(t, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") // already in Git: settles without creating the branch + _, onRemote := f.featureOnRemote() + require.False(t, onRemote) + require.Len(t, *roots, 1) + + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip), "the next real write lands on the new parent") +} + +// Test 3: a change just before admission, with retained writes. No push leaves on the old root. +func TestBranchWorker_AParentChangeBeforeAdmissionReplaysOntoTheNewParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + loop.lastPushAt = time.Now() // cooldown: the write is retained + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + + changed := false + beforePushAdmission = func() { + if !changed { + changed = true + f.worker.SetParentBranch("release") + } + } + t.Cleanup(func() { beforePushAdmission = nil }) + roots := pushRoots(t, nil) + + loop.pushPending() + + require.Empty(t, loop.pendingWrites, "the replay published") + assert.NotContains(t, *roots, plumbing.NewBranchReferenceName("main"), "no push left on the old root") + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) +} + +// Test 4: a change after admission is in flight. The push that creates the branch from the old +// parent is not recalled; the branch then exists, and an existing branch does not follow its +// parent, so later writes build on it with no attempt to re-root it. +func TestBranchWorker_AParentChangeAfterAdmissionIsNotRecalled(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release\n") + mainTip := f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + roots := pushRoots(t, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + first, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, mainTip, f.commitParent(first), "the admitted push created the branch from main") + + fetches := totalFetches(t, reader, f.worker) + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + tip, _ := f.featureOnRemote() + assert.Equal(t, first, f.commitParent(tip), "later writes go to the existing branch") + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "and nothing re-reads the parent") + assert.Len(t, *roots, 2) + assert.Equal(t, plumbing.NewBranchReferenceName("feature"), (*roots)[1]) +} + +// Test 5: the new parent is missing, in cases 1 and 3. The branch is never created, the work stays +// retained, and it lands once the parent exists. +func TestBranchWorker_AParentChangeToAMissingParentKeepsTheWork(t *testing.T) { + t.Run("during a fetch", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + onSync(t, 1, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + _, onRemote := f.featureOnRemote() + require.False(t, onRemote, "no branch on the old parent") + require.NotEmpty(t, loop.pendingWrites, "the work is retained") + + release := f.pushToRelease("RELEASE.md", "now it exists\n") + loop.pushPending() + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) + }) + + t.Run("before admission", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + beforePushAdmission = func() { f.worker.SetParentBranch("release") } + t.Cleanup(func() { beforePushAdmission = nil }) + + loop.pushPending() + + _, onRemote := f.featureOnRemote() + require.False(t, onRemote) + require.Len(t, loop.pendingWrites, 1, "the work is retained") + + beforePushAdmission = nil + release := f.pushToRelease("RELEASE.md", "now it exists\n") + loop.pushPending() + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) + }) +} + +// Test 6: a parent cannot move an existing write branch, so a change costs it nothing: no fetch, +// and the next push goes to the write branch. +func TestBranchWorker_AParentChangeCostsAnExistingWriteBranchNothing(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release\n") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + liveWrite(loop, "cm1") + first, onRemote := f.featureOnRemote() + require.True(t, onRemote) + + require.True(t, f.worker.SetParentBranch("release")) + fetches := totalFetches(t, reader, f.worker) + roots := pushRoots(t, nil) + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "zero extra fetches") + require.Len(t, *roots, 1, "one push") + assert.Equal(t, plumbing.NewBranchReferenceName("feature"), (*roots)[0]) + tip, _ := f.featureOnRemote() + assert.Equal(t, first, f.commitParent(tip)) +} diff --git a/internal/git/git.go b/internal/git/git.go index 0599bb26..7fc97eca 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -147,7 +147,7 @@ func PrepareBranchFrom( } targetBranch := plumbing.NewBranchReferenceName(targetBranchName) - pullReport, err := syncToRemoteFrom(ctx, repo, targetBranch, parentBranch, auth) + pullReport, err := syncToRemoteFn(ctx, repo, targetBranch, parentBranch, auth) if err != nil { return nil, err } diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index f3e015eb..a466fa31 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -290,8 +290,9 @@ func (m *WorkerManager) RemoteForBranch(key BranchKey) (RemoteObservation, bool) // // parentBranch is the GitTarget's spec.parentBranch. It is immutable on a GitTarget, so a live // worker sees a different one only when every target on its branch was replaced before the worker -// sweep ran. That is not a replacement: the worker keeps its clone, and SetParentBranch drops its -// base trust and rebuilds any retained writes on the new parent. +// sweep ran. That is not a replacement: the worker keeps its clone, and SetParentBranch raises the +// parent generation, so nothing planned under the old parent is pushed and retained writes are +// rebuilt on the new one. // // Worker creation/start is protected by the manager lock. func (m *WorkerManager) EnsureWorker( From 9da39b21b21abd2d8ad4fe2f7aad71e5f4469d47 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:28:33 +0000 Subject: [PATCH 13/18] fix(git): never orphan a branch in a repository that is not empty Two ways an omitted parent started the write branch with no history beside refs it ignored: - A cycle planned on an empty repository (root = zero) pushed after somebody created main: no check fired, because the missing-root check needs a non-zero root and the moved check compared the absent write branch with zero. validatePushState now refuses with RepositoryNotEmptyError when the root is zero, the write branch is not advertised and the advertisement carries other refs. It reads the advertisement the push already holds, and remoteMovedDuringPush treats it as contention, so the replay resolves the new default branch and re-plans from the retained writes. - SmartFetchFrom returned the "empty repository" answer for a repository with refs but no resolvable default branch (HEAD dangling, or only tags), and the caller made HEAD unborn. It now returns DefaultBranchUnresolvedError, handled like ErrParentBranchNotFound on the fetch, the replay and the refresh: no write is planned and the parent is observed Missing. "Empty" has one definition: no hash refs at all, tags included. Default-branch resolution follows go-git, which rewrites a hash-only HEAD before Remote.List returns; that is pinned by a test. An omitted parent can now stall: the observation carries ParentRequested, and parentBranchReadiness compares it with spec.parentBranch, with a message that says to fix HEAD or set the field. Canonical git advertises neither a HEAD that resolves to nothing nor its symref, so against it the message says the remote has no default branch. Co-Authored-By: Claude Opus 5.5 --- api/v1alpha3/gittarget_types.go | 16 +- .../crd/bases/configbutler.ai_gittargets.yaml | 6 +- docs/configuration.md | 16 +- internal/controller/gittarget_controller.go | 45 ++- .../gittarget_parent_branch_test.go | 30 +- internal/git/branch_worker.go | 34 ++- internal/git/branch_worker_empty_repo_test.go | 264 ++++++++++++++++++ internal/git/git_atomic_push.go | 34 +++ internal/git/git_smart_fetch.go | 66 ++++- internal/git/refresh.go | 12 +- internal/git/remote_observation.go | 17 +- 11 files changed, 494 insertions(+), 46 deletions(-) create mode 100644 internal/git/branch_worker_empty_repo_test.go diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index 30c180d7..e54b6ac4 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -492,24 +492,26 @@ type GitTargetParentState string const ( // GitTargetParentFound means the parent branch exists; the first commit builds on its tip. GitTargetParentFound GitTargetParentState = "Found" - // GitTargetParentMissing means spec.parentBranch is not on the remote either, so the write branch - // cannot be created; the GitTarget reports ParentBranchNotFound. + // GitTargetParentMissing means the parent is not on the remote either, so the write branch + // cannot be created; the GitTarget reports ParentBranchNotFound. The parent is spec.parentBranch, + // or, when that is omitted, the default branch of a repository that is not empty but whose HEAD + // names no branch it carries. GitTargetParentMissing GitTargetParentState = "Missing" - // GitTargetParentUnborn means the repository is empty and no parent is configured, so the first - // commit starts the write branch with no history. + // GitTargetParentUnborn means the repository is empty, with no refs at all (tags included), and + // no parent is configured, so the first commit starts the write branch with no history. GitTargetParentUnborn GitTargetParentState = "Unborn" ) // GitTargetParentStatus is the parent branch of a write branch that does not exist yet. type GitTargetParentStatus struct { // State is what the write branch would be created from: Found (the parent's tip), Missing - // (spec.parentBranch is not on the remote; nothing is written) or Unborn (an empty repository - // and no parent configured; the first commit has no history). + // (the parent is not on the remote; nothing is written) or Unborn (an empty repository, with no + // refs at all, and no parent configured; the first commit has no history). // +required State GitTargetParentState `json:"state"` // Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when - // that is omitted. Empty when State is Unborn. + // that is omitted. Empty when State is Unborn, and when the remote names no default branch. // +optional Branch string `json:"branch,omitempty"` diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 763abb77..bafbcae5 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -708,7 +708,7 @@ spec: branch: description: |- Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when - that is omitted. Empty when State is Unborn. + that is omitted. Empty when State is Unborn, and when the remote names no default branch. type: string commit: description: |- @@ -718,8 +718,8 @@ spec: state: description: |- State is what the write branch would be created from: Found (the parent's tip), Missing - (spec.parentBranch is not on the remote; nothing is written) or Unborn (an empty repository - and no parent configured; the first commit has no history). + (the parent is not on the remote; nothing is written) or Unborn (an empty repository, with no + refs at all, and no parent configured; the first commit has no history). enum: - Found - Missing diff --git a/docs/configuration.md b/docs/configuration.md index 7e452008..0339ab5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -770,8 +770,14 @@ spec: - An explicit parent must exist when the write branch is created. Otherwise the target reports `Ready=False` with reason `ParentBranchNotFound`, writes nothing, and recovers once the branch is - pushed. An omitted parent never fails this way, and in an empty repository the first commit - starts a branch with no history. + pushed. +- An omitted parent is the remote's default branch, as last discovered. In an empty repository (no + refs at all, tags included) the first commit starts a branch with no history. A repository that + is not empty but whose `HEAD` names no branch it carries also reports `ParentBranchNotFound`, and + writes nothing until `HEAD` is fixed or `parentBranch` is set; that never starts an orphan. +- A detached `HEAD`, or a server that does not say what `HEAD` points at, resolves the way go-git + resolves it: to `master` when it is at `HEAD`'s commit, otherwise to the alphabetically first + branch that is. Set `parentBranch` when you need certainty rather than that guess. - Once the write branch exists, it is used as it is: new commits on the parent do not reach it. Delete the write branch after its changes are merged, and the next edit starts a fresh one. - `parentBranch` is immutable, including adding or removing it: delete and recreate the target to @@ -915,9 +921,9 @@ status: branch is read off `kubectl`. - `parent` appears only while the branch is not on the remote: it says what the branch would be created from, observed together with the branch's absence. `state` is `Found` (the first commit - builds on `commit`, the parent's tip), `Missing` (`spec.parentBranch` is not on the remote - either; the target reports `ParentBranchNotFound` and writes nothing) or `Unborn` (an empty - repository and no parent configured; the first commit starts the branch with no history). See + builds on `commit`, the parent's tip), `Missing` (the parent is not on the remote either; the + target reports `ParentBranchNotFound` and writes nothing) or `Unborn` (an empty repository, with + no refs at all, and no parent configured; the first commit starts the branch with no history). See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch). `kubectl get gittarget -o wide` shows `lastVerifiedAt` as an age, in the `Verified` column. diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index ff177e85..6066d0e4 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -813,34 +813,51 @@ func gitTargetReadinessGates( } // parentBranchReadiness reports whether the write branch can be created: False when the last -// look at the remote found neither the write branch nor the configured spec.parentBranch. +// look at the remote found neither the write branch nor the parent it would be created from. That +// parent is the configured spec.parentBranch, or, when it is omitted, the remote's default branch: +// a repository that is not empty but whose HEAD names no branch it carries refuses as well, because +// starting there would make an orphan. // // It is read from the branch worker's remote observation, so it costs no round trip and holds -// with periodic refresh disabled: every fetch on the write path records the same observation. -// An omitted parent is never missing; it falls back to the remote's default branch. An -// observation proved against another repository than the one the GitProvider names now (a -// repoint) says nothing about this one. +// with periodic refresh disabled: every fetch on the write path records the same observation. The +// observation counts only when it was made under the parent configured now, and only for the +// repository the GitProvider names now (a repoint proves nothing about the new one). func (r *GitTargetReconciler) parentBranchReadiness( target *configbutleraiv1alpha3.GitTarget, providerNS string, repo git.RepoIdentity, ) conditionValue { ok := conditionValue{Status: metav1.ConditionTrue, Reason: GitTargetReasonOK} - if target.Spec.ParentBranch == "" { - return ok - } observed, seen := r.observeRemote(target, providerNS) - if !seen || observed.ParentState != git.ParentMissing || observed.ParentBranch != target.Spec.ParentBranch || - differentRepository(observed.Repo, repo) { + if !seen || observed.ParentState != git.ParentMissing || + observed.ParentRequested != target.Spec.ParentBranch || differentRepository(observed.Repo, repo) { return ok } return conditionValue{ - Status: metav1.ConditionFalse, - Reason: GitTargetReasonParentBranchNotFound, - Message: fmt.Sprintf( + Status: metav1.ConditionFalse, + Reason: GitTargetReasonParentBranchNotFound, + Message: parentBranchNotFoundMessage(target, observed.ParentBranch), + } +} + +// parentBranchNotFoundMessage says which parent is missing and what fixes it. +func parentBranchNotFoundMessage(target *configbutleraiv1alpha3.GitTarget, missing string) string { + switch { + case target.Spec.ParentBranch != "": + return fmt.Sprintf( "Branch '%s' does not exist on the remote, and neither does its parent branch '%s' to create "+ "it from; nothing is written until one of them exists", - target.Spec.Branch, target.Spec.ParentBranch), + target.Spec.Branch, target.Spec.ParentBranch) + case missing != "": + return fmt.Sprintf( + "Branch '%s' does not exist on the remote, and the remote's default branch (HEAD -> '%s') does "+ + "not exist either; fix the remote's HEAD or set spec.parentBranch. Nothing is written until then", + target.Spec.Branch, missing) + default: + return fmt.Sprintf( + "Branch '%s' does not exist on the remote, and the remote has no default branch to create it "+ + "from; set the remote's HEAD or spec.parentBranch. Nothing is written until then", + target.Spec.Branch) } } diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index 9b55c16f..acd564a4 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -135,7 +135,7 @@ func TestEnsureEventStream_AParentBranchChangeKeepsTheWorker(t *testing.T) { } // TestParentBranchReadiness: a fetch that found neither the write branch nor its configured parent -// stalls the GitTarget as ParentBranchNotFound. An omitted parent never does. +// stalls the GitTarget as ParentBranchNotFound, for that parent only. func TestParentBranchReadiness(t *testing.T) { r, workers := startParentBranchWorkers(t) repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} @@ -171,6 +171,34 @@ func TestParentBranchReadiness(t *testing.T) { assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status) } +// TestParentBranchReadiness_AnOmittedParentCanStall: with spec.parentBranch omitted, a remote that +// is not empty but whose default branch does not resolve stalls the target too, with a message +// that names what to fix. An observation made under a configured parent says nothing about it. +func TestParentBranchReadiness_AnOmittedParentCanStall(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "", time.Now()) + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + + worker.RecordUnresolvedDefaultBranchForTest("master") + got := r.parentBranchReadiness(target, "shop", repo) + assert.Equal(t, metav1.ConditionFalse, got.Status) + assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) + assert.Contains(t, got.Message, "HEAD -> 'master'") + assert.Contains(t, got.Message, "spec.parentBranch") + + worker.RecordUnresolvedDefaultBranchForTest("") + got = r.parentBranchReadiness(target, "shop", repo) + assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) + assert.Contains(t, got.Message, "no default branch") + + worker.RecordMissingParentForTest("release") + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status, + "an observation made under a configured parent is about another configuration") +} + func TestReadyReasonIs(t *testing.T) { conditions := []metav1.Condition{{Type: ConditionTypeReady, Status: metav1.ConditionFalse, Reason: GitTargetReasonParentBranchNotFound}} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index dcb79c02..8c812423 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -2252,6 +2252,13 @@ func (w *BranchWorker) remoteMovedDuringPush( moved.Branch == plumbing.NewBranchReferenceName(w.Branch) { return true } + // An empty repository gained refs since the cycle was planned on it. The replay's fetch + // resolves the new default branch, or refuses, and the rebuild re-plans from the retained + // writes instead of reusing the orphan commits. + var notEmpty *RepositoryNotEmptyError + if errors.As(pushErr, ¬Empty) { + return true + } remoteHash, known := advertisedRootHash(pushErr, rootBranch) if !known { @@ -2609,8 +2616,12 @@ func (w *BranchWorker) noteFetchParent(parent parentConfig, err error) { w.baseParentGen.Store(parent.gen) return } - if errors.Is(err, ErrParentBranchNotFound) { - w.recordMissingParent(parent.name, ObservedByFetch) + var unresolved *DefaultBranchUnresolvedError + switch { + case errors.As(err, &unresolved): + w.recordMissingParent(parent.name, unresolved.Head) + case errors.Is(err, ErrParentBranchNotFound): + w.recordMissingParent(parent.name, parent.name) } } @@ -2990,10 +3001,14 @@ func (w *BranchWorker) recordRemoteObservation(revision string, by ObservationSo w.publishObservation(RemoteObservation{Commit: revision, At: time.Now(), By: by, Repo: w.repo}) } -// recordMissingParent records that the write branch is absent and its configured parent is too. -func (w *BranchWorker) recordMissingParent(parent string, by ObservationSource) { +// recordMissingParent records that the write branch is absent and so is the parent it would be +// created from. requested is spec.parentBranch as the observing operation read it (empty for the +// remote's default branch); branch is the parent that is missing: the configured one, or the +// branch an unresolved HEAD names. Only a look at the remote can prove it, so it is by Fetch. +func (w *BranchWorker) recordMissingParent(requested, branch string) { w.publishObservation(RemoteObservation{ - At: time.Now(), By: by, Repo: w.repo, ParentState: ParentMissing, ParentBranch: parent, + At: time.Now(), By: ObservedByFetch, Repo: w.repo, + ParentState: ParentMissing, ParentRequested: requested, ParentBranch: branch, }) } @@ -3177,7 +3192,14 @@ func (w *BranchWorker) SeedRemoteObservationForTest(revision string, at time.Tim // RecordMissingParentForTest records the observation a fetch makes when neither the write branch // nor its configured parent exists, for tests in another package. func (w *BranchWorker) RecordMissingParentForTest(parent string) { - w.recordMissingParent(parent, ObservedByFetch) + w.recordMissingParent(parent, parent) +} + +// RecordUnresolvedDefaultBranchForTest records the observation a fetch makes when the write branch +// is absent, no parent is configured, and the remote's HEAD names head, which it does not carry +// (empty: no HEAD at all), for tests in another package. +func (w *BranchWorker) RecordUnresolvedDefaultBranchForTest(head string) { + w.recordMissingParent("", head) } // BaseTrustedForTest exposes the flag to tests in another package. diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go new file mode 100644 index 00000000..1b33e9af --- /dev/null +++ b/internal/git/branch_worker_empty_repo_test.go @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// An empty repository is one whose advertisement carries no hash refs at all. These pin what the +// worker does at the edges of that definition, against canonical git over HTTP: a repository that +// stops being empty while work is planned on it (§4.2), and one that is not empty but whose +// default branch does not resolve (§4.3). Neither may ever produce an orphan branch. + +import ( + "os/exec" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// realServerNewBranch is a worker for the absent write branch `feature`, against a real server. +type realServerNewBranch struct { + *ledgerFixture + + loop *branchWorkerEventLoop +} + +func newRealServerNewBranch(t *testing.T, slug string, seed func(repoDir string)) *realServerNewBranch { + t.Helper() + f := newLedgerFixtureOnBranch(t, slug, false, "feature") + if seed != nil { + seed(f.repoDir) + } + f.worker.mapper = configMapMapper() + createGitTarget(t, f.worker, newBranchTarget, newBranchFolder, + &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneAlways}) + loop := newBranchWorkerEventLoop(f.worker, 0) + t.Cleanup(loop.stopTimers) + return &realServerNewBranch{ledgerFixture: f, loop: loop} +} + +func gitIn(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput() + require.NoError(t, err, "git %v: %s", args, out) + return strings.TrimSpace(string(out)) +} + +// ref is where a branch is on the remote, or zero when the remote does not carry it. +func (f *realServerNewBranch) ref(name string) plumbing.Hash { + f.t.Helper() + repo, err := gogit.PlainOpen(f.repoDir) + require.NoError(f.t, err) + ref, err := repo.Reference(plumbing.NewBranchReferenceName(name), true) + if err != nil { + return plumbing.ZeroHash + } + return ref.Hash() +} + +func (f *realServerNewBranch) parentOf(commit plumbing.Hash) []plumbing.Hash { + f.t.Helper() + repo, err := gogit.PlainOpen(f.repoDir) + require.NoError(f.t, err) + c, err := repo.CommitObject(commit) + require.NoError(f.t, err) + return c.ParentHashes +} + +func (f *realServerNewBranch) observed() RemoteObservation { + f.t.Helper() + o, known := f.worker.LastRemoteObservation() + require.True(f.t, known) + return o +} + +// tagOnly gives the remote a tag on a commit no branch carries: not empty, and no default branch. +func tagOnly(t *testing.T, repoDir string) { + t.Helper() + tree := gitIn(t, repoDir, "mktree") + commit := gitIn(t, repoDir, "-c", "user.name=Outside", "-c", "user.email=o@example.com", + "commit-tree", tree, "-m", "tagged") + gitIn(t, repoDir, "tag", "v1", commit) +} + +// branchOnDisk creates a branch with one README commit on the bare repository directly, which works +// whatever the repository holds already. +func branchOnDisk(t *testing.T, repoDir, branch, content string) plumbing.Hash { + t.Helper() + cmd := exec.Command("git", "-C", repoDir, "hash-object", "-w", "--stdin") + cmd.Stdin = strings.NewReader(content) + out, err := cmd.Output() + require.NoError(t, err) + mk := exec.Command("git", "-C", repoDir, "mktree") + mk.Stdin = strings.NewReader("100644 blob " + strings.TrimSpace(string(out)) + "\tREADME.md\n") + tree, err := mk.Output() + require.NoError(t, err) + commit := gitIn(t, repoDir, "-c", "user.name=Outside", "-c", "user.email=o@example.com", + "commit-tree", strings.TrimSpace(string(tree)), "-m", "outside") + gitIn(t, repoDir, "update-ref", "refs/heads/"+branch, commit) + return plumbing.NewHash(commit) +} + +// standbyRequest waits for a window and closes it at once: the write it attaches to is committed +// and, in a cooldown, retained. +func standbyRequest() *AttachCommitRequest { + req := attachReq("alice", time.Hour) + req.GitTargetName = newBranchTarget + req.MaxDuration = 0 + return req +} + +// §4.2 test 1. The work was planned on an empty repository; main appears before publication. The +// push must not create feature as an orphan: it is refused at the advertisement, and the replay +// builds on main. +func TestBranchWorker_AnEmptyRepositoryThatGainsABranchIsNotOrphaned(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newRealServerNewBranch(t, "empty-gains-main", nil) + + f.loop.lastPushAt = time.Now() // cooldown: the write is retained + req := standbyRequest() + serviceAttach(f.loop, req) + liveWrite(f.loop, "cm1") + require.Len(t, f.loop.pendingWrites, 1) + contention := fetchCount(t, reader, f.worker, fetchReasonContention) + + main := simulateClientCommitOnDisk(t, f.repoDir, "main", "README.md", "main appeared\n") + f.loop.pushPending() + + tip := f.ref("feature") + require.False(t, tip.IsZero(), "the write was published") + assert.Equal(t, []plumbing.Hash{main}, f.parentOf(tip), "on main's tip, not as a root commit") + assert.NotEmpty(t, gitIn(t, f.repoDir, "merge-base", "feature", "main")) + assert.Contains(t, gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "feature"), cmFile("cm1")) + assert.Equal(t, contention+1, fetchCount(t, reader, f.worker, fetchReasonContention), "exactly one replay") + res, resolved := outcome(t, f.worker) + require.True(t, resolved) + assert.Equal(t, FinalizeCommitted, res.Outcome) +} + +// §4.2 test 2. The same with a write that changes nothing: the branch stays absent. +func TestBranchWorker_AnEmptyRepositoryThatGainsABranchKeepsANoOpAbsent(t *testing.T) { + f := newRealServerNewBranch(t, "empty-gains-main-noop", nil) + + f.loop.lastPushAt = time.Now() + deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) + deleted.Operation = "DELETE" + deleted.Object = nil + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) + require.Len(t, f.loop.pendingWrites, 1) + + simulateClientCommitOnDisk(t, f.repoDir, "main", "README.md", "main appeared\n") + f.loop.pushPending() + + assert.Empty(t, f.loop.pendingWrites, "settled") + assert.True(t, f.ref("feature").IsZero(), "nothing to publish: the branch stays absent") +} + +// §4.2 test 3. Only a tag appears: not empty, and no default branch to start from. Nothing is +// orphaned, the work stays retained and the parent is reported missing, until a branch exists. +func TestBranchWorker_AnEmptyRepositoryThatGainsOnlyATagWaits(t *testing.T) { + f := newRealServerNewBranch(t, "empty-gains-tag", nil) + + f.loop.lastPushAt = time.Now() + liveWrite(f.loop, "cm1") + require.Len(t, f.loop.pendingWrites, 1) + + tagOnly(t, f.repoDir) + f.loop.pushPending() + + assert.True(t, f.ref("feature").IsZero(), "no orphan") + assert.Len(t, f.loop.pendingWrites, 1, "the work is retained") + o := f.observed() + assert.Equal(t, ParentMissing, o.ParentState) + assert.Empty(t, o.ParentRequested) + + main := branchOnDisk(t, f.repoDir, "main", "a branch at last\n") + f.loop.pushPending() + tip := f.ref("feature") + require.False(t, tip.IsZero()) + assert.Equal(t, []plumbing.Hash{main}, f.parentOf(tip)) +} + +// §4.2 test 4 and §4.3 test 4. A truly empty repository is still Unborn, and the first push still +// creates the root commit. +func TestBranchWorker_ATrulyEmptyRepositoryStillBootstraps(t *testing.T) { + f := newRealServerNewBranch(t, "empty-throughout", nil) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + assert.Equal(t, ParentUnborn, f.observed().ParentState) + + liveWrite(f.loop, "cm1") + + tip := f.ref("feature") + require.False(t, tip.IsZero()) + assert.Empty(t, f.parentOf(tip), "a root commit") +} + +// §4.3 test 1. HEAD names a branch the repository does not carry. That is not an empty repository, +// so nothing may start an orphan there; the fetch and the refresh both report the parent missing. +func TestBranchWorker_ADanglingRemoteHeadIsMissingNotUnborn(t *testing.T) { + f := newRealServerNewBranch(t, "dangling-head", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "symbolic-ref", "HEAD", "refs/heads/master") + }) + + err := f.worker.ensureRepositoryInitialized(f.worker.ctx) + require.ErrorIs(t, err, ErrDefaultBranchUnresolved) + o := f.observed() + assert.Equal(t, ParentMissing, o.ParentState, "the fetch: not Unborn") + assert.Empty(t, o.ParentRequested) + // Canonical git advertises neither a HEAD that resolves to nothing nor its symref, so the name + // it dangles at (master) never reaches the client: the status says the remote has no default + // branch. A server that does send the symref gets the name in the message. + assert.Empty(t, o.ParentBranch) + + provider, err := f.worker.getGitProvider(f.worker.ctx) + require.NoError(t, err) + require.NoError(t, f.loop.refreshFromRemote(provider)) + assert.Equal(t, ParentMissing, f.observed().ParentState, "the refresh agrees") + + liveWrite(f.loop, "cm1") + assert.True(t, f.ref("feature").IsZero(), "no orphan") + assert.True(t, f.loop.lastPushAt.IsZero(), "nothing was pushed") + + // §4.3 test 3: repairing HEAD is enough. + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") + liveWrite(f.loop, "cm2") + tip := f.ref("feature") + require.False(t, tip.IsZero()) + assert.Equal(t, []plumbing.Hash{f.ref("main")}, f.parentOf(tip)) +} + +// §4.3 test 2. A detached HEAD is resolved the way go-git resolves it, which this pins: master when +// it is at HEAD's commit, otherwise the alphabetically first branch that is. +func TestBranchWorker_ADetachedRemoteHeadFollowsGoGitsResolution(t *testing.T) { + for name, tc := range map[string]struct { + branches []string + want string + }{ + "master wins": {branches: []string{"trunk", "master"}, want: "master"}, + "first by name": {branches: []string{"trunk", "develop"}, want: "develop"}, + "one branch at HEAD": {branches: []string{"trunk"}, want: "trunk"}, + } { + t.Run(name, func(t *testing.T) { + f := newRealServerNewBranch(t, "detached-"+strings.ReplaceAll(name, " ", "-"), func(repoDir string) { + first := simulateClientCommitOnDisk(t, repoDir, tc.branches[0], "README.md", "x\n") + for _, b := range tc.branches[1:] { + gitIn(t, repoDir, "branch", b, first.String()) + } + gitIn(t, repoDir, "update-ref", "--no-deref", "HEAD", first.String()) + }) + + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + o := f.observed() + assert.Equal(t, ParentFound, o.ParentState) + assert.Equal(t, tc.want, o.ParentBranch, "status names the branch go-git chose") + }) + } +} diff --git a/internal/git/git_atomic_push.go b/internal/git/git_atomic_push.go index 80604dd7..711c5906 100644 --- a/internal/git/git_atomic_push.go +++ b/internal/git/git_atomic_push.go @@ -104,6 +104,22 @@ type RemoteMovedError struct { Missing bool } +// RepositoryNotEmptyError reports a push planned against an empty repository that is no longer +// empty: the cycle's root is the zero hash, the write branch is not advertised, and the +// advertisement carries other refs. Creating the branch now would make an orphan beside them, so +// the push is refused and the worker replays onto what discovery finds. +// +// It is its own type rather than a RemoteMovedError, whose Branch and Advertised mean "the root, +// compared": filling them with another branch's hash would make them lie. +type RepositoryNotEmptyError struct { + // Refs is how many hash refs the advertisement carried. + Refs int +} + +func (e *RepositoryNotEmptyError) Error() string { + return fmt.Sprintf("the repository is no longer empty: the advertisement carries %d refs", e.Refs) +} + func (e *RemoteMovedError) Error() string { if e.Missing { // Unchanged from the untyped error this replaces: it is the string in the logs operators @@ -204,6 +220,13 @@ func validatePushState( return pushPlan{settled: PushUpToDate, head: remoteHash}, nil } + if rootHash.IsZero() && !found { + if n := countHashRefs(refs); n > 0 { + logger.Info("Planned against an empty repository that has gained refs", "branch", branchName, "refs", n) + return pushPlan{}, &RepositoryNotEmptyError{Refs: n} + } + } + if err := remoteMovedSinceBase(branch, rootBranch, rootHash, refs); err != nil { logger.Info("Remote branch not in expected state", "branch", branchName, "root", rootBranch.Short()) return pushPlan{}, err @@ -222,6 +245,17 @@ func validatePushState( return pushPlan{old: oldHash, new: localHash}, nil } +// countHashRefs counts the advertised hash refs other than HEAD; see advertisesNoRefs. +func countHashRefs(refs map[plumbing.ReferenceName]plumbing.Hash) int { + n := 0 + for name := range refs { + if name != plumbing.HEAD { + n++ + } + } + return n +} + // nothingToPublish reports, for a branch the remote does not carry, that there is nothing to // create it with: no local commit at all, or a new branch (rooted on another branch, its parent) // whose local tip is still the parent's, which the caller has just confirmed. Creating it would diff --git a/internal/git/git_smart_fetch.go b/internal/git/git_smart_fetch.go index 47447e55..97b2d79e 100644 --- a/internal/git/git_smart_fetch.go +++ b/internal/git/git_smart_fetch.go @@ -21,7 +21,8 @@ import ( // Return values (example with target="refs/heads/feature"): // - "refs/heads/feature", nil: Target found on remote, fetched, ready to checkout. // - "refs/heads/main", nil: Target missing on remote, fell back to default branch. -// - "", nil: No valid branches found (empty repo). +// - "", nil: The repository is empty (see advertisesNoRefs). +// - "", ErrDefaultBranchUnresolved: Not empty, and no default branch to fall back to. func SmartFetch( ctx context.Context, repo *git.Repository, @@ -37,6 +38,59 @@ func SmartFetch( // an unborn branch in an empty repository. var ErrParentBranchNotFound = errors.New("parent branch not found on the remote") +// ErrDefaultBranchUnresolved reports a repository that is not empty but whose default branch does +// not resolve to a branch it carries, while no parent is configured and the target branch is +// absent. Starting the target branch there would make an orphan, so it is refused exactly like a +// missing configured parent. DefaultBranchUnresolvedError names what HEAD points at. +var ErrDefaultBranchUnresolved = errors.New("the remote's default branch does not resolve to a branch") + +// DefaultBranchUnresolvedError is ErrDefaultBranchUnresolved with the evidence: the branch the +// remote's HEAD names, or empty when the advertisement carries no symbolic HEAD at all. +// +// The resolution itself follows go-git. Remote.List rewrites a hash-only HEAD (a server without +// the symref capability, or a detached HEAD) into a symbolic one before returning: to master when +// master has that hash, otherwise to the alphabetically first branch that has it. So a HEAD this +// error reports is one even that heuristic could not place. +type DefaultBranchUnresolvedError struct { + // Head is the short name of the branch HEAD points at; empty when there is no symbolic HEAD. + Head string +} + +func (e *DefaultBranchUnresolvedError) Error() string { + if e.Head == "" { + return "the remote has no default branch" + } + return fmt.Sprintf("the remote's default branch (HEAD -> %s) does not exist", e.Head) +} + +// Is makes the error match ErrDefaultBranchUnresolved. +func (e *DefaultBranchUnresolvedError) Is(target error) bool { + return target == ErrDefaultBranchUnresolved +} + +// advertisesNoRefs reports an empty repository: an advertisement with no hash refs at all, no +// branch, no tag, nothing else. A symbolic HEAD pointing at an unborn branch does not count, and a +// repository with only tags is not empty. This is the one definition discovery, publication and +// the observation share. +func advertisesNoRefs(refs []*plumbing.Reference) bool { + for _, ref := range refs { + if ref.Type() == plumbing.HashReference && ref.Name() != plumbing.HEAD { + return false + } + } + return true +} + +// symbolicHeadTarget is the short name of the branch the advertisement's HEAD points at, or empty. +func symbolicHeadTarget(refs []*plumbing.Reference) string { + for _, ref := range refs { + if ref.Name() == plumbing.HEAD && ref.Type() == plumbing.SymbolicReference { + return ref.Target().Short() + } + } + return "" +} + // SmartFetchFrom is SmartFetch with an explicit parent: the branch to fall back to when the target // branch is absent. Empty means the remote's default branch. A configured parent that is absent is // an error rather than a fallback, so a typo can never start an orphan branch; it does not matter @@ -107,12 +161,18 @@ func SmartFetchFrom( } // fallbackBranch is the branch to fall back on, and to fetch as the safety net: the configured -// parent, or the remote's default branch when none is configured. A configured parent the remote -// does not carry is an error only while the target is absent too. +// parent, or the remote's default branch when none is configured. A parent the remote does not +// carry is an error only while the target is absent too: a configured one is +// ErrParentBranchNotFound, and an unresolved default branch in a repository that is not empty is +// ErrDefaultBranchUnresolved, because an unborn start there would be an orphan beside the refs it +// ignored. func fallbackBranch( refs []*plumbing.Reference, parent, headFull, headShort string, targetExists bool, ) (string, string, error) { if parent == "" { + if headFull == "" && !targetExists && !advertisesNoRefs(refs) { + return "", "", &DefaultBranchUnresolvedError{Head: symbolicHeadTarget(refs)} + } return headFull, headShort, nil } full, short := configuredParent(refs, parent) diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 7ce7b293..9c680a9d 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -258,7 +258,12 @@ func (w *BranchWorker) recordAdvertisement(advertisement remoteAdvertisement, pa } if advertisement.parent.IsZero() { if parentBranch != "" { - w.recordMissingParent(parentBranch, ObservedByFetch) + w.recordMissingParent(parentBranch, parentBranch) + return true + } + if !advertisement.empty { + // Not empty, and its HEAD resolves to no branch: as missing as a configured parent. + w.recordMissingParent("", advertisement.parentBranch) return true } w.recordAbsentBranch("", "", ObservedByFetch) // an empty repository: no parent at all @@ -377,6 +382,8 @@ type remoteAdvertisement struct { parent plumbing.Hash // parentBranch is that parent's short name; empty when the remote's HEAD names no branch. parentBranch string + // empty records an advertisement with no hash refs at all: see advertisesNoRefs. + empty bool } // advertiseRemoteBranch asks the remote where a branch and its parent are, and transfers nothing @@ -396,6 +403,7 @@ func advertiseRemoteBranch( if err != nil { return remoteAdvertisement{}, err } + empty := advertisesNoRefs(refs) hashes := map[plumbing.ReferenceName]plumbing.Hash{} var defaultBranch plumbing.ReferenceName for _, ref := range refs { @@ -410,7 +418,7 @@ func advertiseRemoteBranch( defaultBranch = plumbing.NewBranchReferenceName(parentBranch) } return remoteAdvertisement{ - branch: hashes[branch], parent: hashes[defaultBranch], parentBranch: defaultBranch.Short(), + branch: hashes[branch], parent: hashes[defaultBranch], parentBranch: defaultBranch.Short(), empty: empty, }, nil } diff --git a/internal/git/remote_observation.go b/internal/git/remote_observation.go index 2da85893..cef91b8f 100644 --- a/internal/git/remote_observation.go +++ b/internal/git/remote_observation.go @@ -54,9 +54,15 @@ type RemoteObservation struct { // together with its absence. Empty while the branch exists. ParentState ParentState // ParentBranch and ParentCommit name that parent and where it is. ParentCommit is empty - // unless ParentState is ParentFound; ParentBranch is empty for ParentUnborn. + // unless ParentState is ParentFound; ParentBranch is empty for ParentUnborn. For an omitted + // parent that is ParentMissing, ParentBranch is the branch the remote's HEAD names, or empty + // when it names none. ParentBranch string ParentCommit string + // ParentRequested is spec.parentBranch as the observing operation read it: empty for the + // remote's default branch. It is set on ParentMissing, so a reader can tell whether the + // observation is about the parent configured now. + ParentRequested string } // ParentState is what a branch the remote does not carry would be created from. @@ -65,11 +71,12 @@ type ParentState string const ( // ParentFound means the parent branch exists, and the branch's first commit builds on its tip. ParentFound ParentState = "Found" - // ParentMissing means the configured parent branch is not on the remote either, so the branch - // cannot be created and nothing is written until one of them exists. + // ParentMissing means the parent is not on the remote either, so the branch cannot be created + // and nothing is written until one of them exists: the configured parent branch is absent, or, + // with none configured, the repository is not empty but its default branch does not resolve. ParentMissing ParentState = "Missing" - // ParentUnborn means the repository offers no branch to start from (it is empty) and none was - // configured, so the first commit starts the branch with no history. + // ParentUnborn means the repository is empty, with no hash refs at all (tags included), and + // no parent was configured, so the first commit starts the branch with no history. ParentUnborn ParentState = "Unborn" ) From c44732a14337ad61e808b7a0f9b49af836876cbc Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:47:10 +0000 Subject: [PATCH 14/18] fix(git): make recovery from a missing parent an obligation the worker drives While a parent branch was missing, every reconcile forced a recheck (restarting every stream of the target), indefinitely for a typo, and recovery still depended on an observation flip: the refresher skips a branch holding work, a failed push stops the push timer, refresh can be off, and a first attempt that fails transiently after the parent reappears stranded the work. The worker now latches an obligation the moment a cycle fails on the parent, and clears it only when the work is published: - retained writes stay retained until pushPending publishes them; - writes it had to drop are remembered per (GitTarget, collection) scope, and a scope clears only when a resync of it is published (pushed, or settled as a no-op), not on the resync reply. The worker drives it on its own deadline (10s, doubling, capped at 5m, one per worker): one advertisement while the parent is missing; once found, it retries the retained writes and raises a per-target SnapshotRequestSeq for the scopes still owed. Before the deadline a refresh tick costs nothing and a live write or resync fails without a fetch; reconciles never probe. A parent change makes the probe due at once. The controller forces one recheck per rise of the sequence (remembered in memory, like reconcile requests) instead of on every reconcile while the Ready reason is ParentBranchNotFound, and reports Ready=False/Reconciling with reason RecoveringParentBranch while the parent is back but work is owed. Ledger rows 1-12 are unchanged; new rows: standby no-op live write (1), standby first write after the parent moved (6, as row 6), parent probe while missing (1 advertisement, 0 fetches). e2e: a third context recovers from ParentBranchNotFound with no cluster edit after the parent is pushed. The refresher-timing step ("contention rose, refresh did not") is removed: with refresh on, a refresh can find B first; the worker tests prove publication alone finds a moved parent. Co-Authored-By: Claude Opus 5.5 --- .coverage-baseline | 2 +- docs/configuration.md | 9 +- internal/controller/condition_helper.go | 6 - internal/controller/gittarget_controller.go | 13 +- .../gittarget_parent_branch_test.go | 55 ++- .../controller/gittarget_parent_recovery.go | 76 ++++ internal/git/branch_worker.go | 52 ++- internal/git/branch_worker_empty_repo_test.go | 4 +- .../git/branch_worker_parent_branch_test.go | 1 + .../git/branch_worker_parent_change_test.go | 14 +- internal/git/git_roundtrip_ledger_test.go | 85 +++- internal/git/parent_recovery.go | 377 ++++++++++++++++++ internal/git/parent_recovery_test.go | 288 +++++++++++++ internal/git/refresh.go | 11 +- internal/git/resync_flush.go | 7 + .../git/testdata/git-roundtrip-ledger.golden | 3 + test/e2e/new_branch_parent_e2e_test.go | 227 +++++++---- 17 files changed, 1128 insertions(+), 102 deletions(-) create mode 100644 internal/controller/gittarget_parent_recovery.go create mode 100644 internal/git/parent_recovery.go create mode 100644 internal/git/parent_recovery_test.go diff --git a/.coverage-baseline b/.coverage-baseline index 068d2ef9..feb00d3b 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -79.3 +79.4 diff --git a/docs/configuration.md b/docs/configuration.md index 0339ab5c..b704a8fa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -769,8 +769,13 @@ spec: ``` - An explicit parent must exist when the write branch is created. Otherwise the target reports - `Ready=False` with reason `ParentBranchNotFound`, writes nothing, and recovers once the branch is - pushed. + `Ready=False` with reason `ParentBranchNotFound` and writes nothing. +- It recovers on its own once the parent is pushed, with no edit to the cluster and with periodic + refresh on or off. The branch worker looks for the parent with one ref advertisement after 10s, + then after twice as long each time, up to every 5 minutes, shared by every target on the branch. + When the parent is back it publishes the writes it held and asks for a fresh snapshot of the ones + it had to drop; until those are published the target reports `Ready=False` with reason + `RecoveringParentBranch`. - An omitted parent is the remote's default branch, as last discovered. In an empty repository (no refs at all, tags included) the first commit starts a branch with no history. A repository that is not empty but whose `HEAD` names no branch it carries also reports `ParentBranchNotFound`, and diff --git a/internal/controller/condition_helper.go b/internal/controller/condition_helper.go index a6a288de..e523eb31 100644 --- a/internal/controller/condition_helper.go +++ b/internal/controller/condition_helper.go @@ -44,9 +44,3 @@ func conditionIsTrue(conditions []metav1.Condition, conditionType string) bool { func conditionIsFalse(conditions []metav1.Condition, conditionType string) bool { return apimeta.IsStatusConditionFalse(conditions, conditionType) } - -// readyReasonIs reports whether the Ready condition carries the given reason. -func readyReasonIs(conditions []metav1.Condition, reason string) bool { - ready := apimeta.FindStatusCondition(conditions, ConditionTypeReady) - return ready != nil && ready.Reason == reason -} diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 6066d0e4..5b50e503 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -144,6 +144,10 @@ type GitTargetReconciler struct { // value is usable. reconcileRequests reconcileRequestTracker + // snapshotRequests remembers which of a branch worker's snapshot requests were acted on, so a + // missing parent's recovery forces one recheck per request rather than one per reconcile. + snapshotRequests snapshotRequestTracker + // remotePublications is what each GitTarget last wrote to status.remote: when, and about which // repository. It is internal by design; see remotePublicationLedger. remotePublications remotePublicationLedger @@ -185,8 +189,9 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( st := beginStatus(r.Client, r.Recorder, &target) gitPathWasRefused := conditionIsFalse(target.Status.Conditions, GitTargetConditionGitPathAccepted) // Writes are dropped while the parent is missing, so its return is recovered the way a refused - // folder is: by a recheck that re-derives what the cluster holds. - parentWasMissing := readyReasonIs(target.Status.Conditions, GitTargetReasonParentBranchNotFound) + // folder is: by a recheck that re-derives what the cluster holds. The branch worker decides + // when, and asks; this acts once per request. + parentRecheck, recoveringParent := r.parentRecovery(&target, target.Namespace) providerNS := target.Namespace // One read of the GitProvider for everything below it; see getGitProvider. @@ -272,7 +277,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // A standing reconcile request forces the same re-check a refused Git path does: the watch // plane re-anchors the target's streams, which is what makes it re-read the folder rather than // wait for the periodic pass. Taken once per distinct annotation value. - forceRecheck := gitPathWasRefused || parentWasMissing || r.reconcileRequests.take( + forceRecheck := gitPathWasRefused || parentRecheck || r.reconcileRequests.take( types.NewResourceReference(target.Name, target.Namespace), reconcileRequestedAt(&target)) observed := r.observeDataPlane(&target, sourceProvider, forceRecheck, log) st.setValue(GitTargetConditionStreamsRunning, observed.axes.Streams) @@ -285,6 +290,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( convergeAsSuspended(rd, &target) gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS, repoIdentityOf(gitProvider)), refs.gitProvider, refs.clusterProvider, refs.sourceCluster) + parentRecoveryReadiness(rd, recoveringParent) st.applyReadiness(rd) if err := st.commit(ctx); err != nil { @@ -1459,6 +1465,7 @@ func (r *GitTargetReconciler) cleanupDeletedGitTarget( // condition gauge is released on the same terms and for a sharper reason: a condition series // that outlives its object reports Ready=False forever and the alert on it never clears. r.reconcileRequests.forget(gitDest) + r.snapshotRequests.forget(gitDest) // Same terms: the publication ledger is this reconciler's memory of what the object's status // said, and an entry that outlives the object is a rate limit held against a name that may be // recreated tomorrow with nothing published. diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index acd564a4..134526c1 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -199,12 +199,55 @@ func TestParentBranchReadiness_AnOmittedParentCanStall(t *testing.T) { "an observation made under a configured parent is about another configuration") } -func TestReadyReasonIs(t *testing.T) { - conditions := []metav1.Condition{{Type: ConditionTypeReady, Status: metav1.ConditionFalse, - Reason: GitTargetReasonParentBranchNotFound}} - assert.True(t, readyReasonIs(conditions, GitTargetReasonParentBranchNotFound)) - assert.False(t, readyReasonIs(conditions, GitTargetReasonTargetConflict)) - assert.False(t, readyReasonIs(nil, GitTargetReasonParentBranchNotFound)) +// TestParentRecovery_ASnapshotRequestForcesOneRecheck: the controller acts on the branch worker's +// snapshot requests, not on the Ready reason. A request forces exactly one recheck; requests made +// entirely between two reconciles are still acted on; an unchanged sequence forces nothing. +func TestParentRecovery_ASnapshotRequestForcesOneRecheck(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + ref := types.NewResourceReference(target.Name, target.Namespace) + + recheck, _ := r.parentRecovery(target, "shop") + assert.False(t, recheck, "nothing asked") + + worker.BumpSnapshotRequestForTest(ref) + recheck, _ = r.parentRecovery(target, "shop") + assert.True(t, recheck) + recheck, _ = r.parentRecovery(target, "shop") + assert.False(t, recheck, "exactly one recheck per request") + + worker.BumpSnapshotRequestForTest(ref) + worker.BumpSnapshotRequestForTest(ref) // a whole episode between two reconciles + recheck, _ = r.parentRecovery(target, "shop") + assert.True(t, recheck, "still acted on") + + for range 10 { + recheck, _ = r.parentRecovery(target, "shop") + assert.False(t, recheck) + } + + r.snapshotRequests.forget(ref) + recheck, _ = r.parentRecovery(target, "shop") + assert.True(t, recheck, "a forgotten target acts on the standing request again") +} + +func TestParentRecoveryReadiness_IsProgressNotAStall(t *testing.T) { + rd := newGitTargetReadiness() + parentRecoveryReadiness(rd, true) + trio := rd.trio() + assert.Equal(t, metav1.ConditionFalse, trio.Ready.Status) + assert.Equal(t, GitTargetReasonRecoveringParentBranch, trio.Ready.Reason) + assert.Equal(t, metav1.ConditionTrue, trio.Reconciling.Status) + assert.NotEqual(t, metav1.ConditionTrue, trio.Stalled.Status) + assert.Equal(t, RequeueStreamSettleInterval, gitTargetRequeue(rd), "it keeps the not-Ready requeue") + + rd = newGitTargetReadiness() + parentRecoveryReadiness(rd, false) + assert.True(t, rd.converged()) } // TestCheckForConflicts_ParentBranchAppliesToInvalidPaths: a target whose path its writer will diff --git a/internal/controller/gittarget_parent_recovery.go b/internal/controller/gittarget_parent_recovery.go new file mode 100644 index 00000000..33266bf4 --- /dev/null +++ b/internal/controller/gittarget_parent_recovery.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// GitTargetReasonRecoveringParentBranch is the Ready reason while the parent branch has been found +// again but the branch worker has not yet published the work it held back. It is progress, not a +// stall, and keeps the not-Ready requeue cadence so the controller acts on the worker's snapshot +// requests within one requeue. +const GitTargetReasonRecoveringParentBranch = "RecoveringParentBranch" + +// snapshotRequestTracker remembers, per GitTarget, the last snapshot request of its branch worker +// that this reconciler acted on. A missing-parent episode that starts and ends between two +// reconciles still leaves the worker's sequence higher, so it is never lost; an unchanged sequence +// forces nothing. Its memory is empty after a restart, which is harmless: every stream starts with +// an initial list, which is itself the snapshot. Its zero value is usable. +type snapshotRequestTracker struct { + mu sync.Mutex + acted map[string]uint64 +} + +// take reports whether seq is a request not acted on yet, and records it as acted on. +func (t *snapshotRequestTracker) take(ref types.ResourceReference, seq uint64) bool { + t.mu.Lock() + defer t.mu.Unlock() + if seq == 0 || t.acted[ref.Key()] >= seq { + return false + } + if t.acted == nil { + t.acted = map[string]uint64{} + } + t.acted[ref.Key()] = seq + return true +} + +// forget drops a deleted object's record. +func (t *snapshotRequestTracker) forget(ref types.ResourceReference) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.acted, ref.Key()) +} + +// parentRecovery reads the branch worker's obligation for work a missing parent held back. The +// first result is whether it asks for a new snapshot of this target (true once per rise of its +// sequence), the second whether the parent is back while work is still owed. It reads memory only, +// and costs no connection. +func (r *GitTargetReconciler) parentRecovery( + target *configbutleraiv1alpha3.GitTarget, + providerNS string, +) (bool, bool) { + if r.WorkerManager == nil { + return false, false + } + worker, ok := r.WorkerManager.GetWorkerForTarget(target.Spec.GitProviderRef.Name, providerNS, target.Spec.Branch) + if !ok { + return false, false + } + ref := types.NewResourceReference(target.Name, target.Namespace) + open, found := worker.ParentRecovery() + return r.snapshotRequests.take(ref, worker.SnapshotRequestSeq(ref)), open && found +} + +// parentRecoveryReadiness is the progress gate for a target whose parent is back but whose +// held-back work is not published yet. +func parentRecoveryReadiness(rd *readiness, recovering bool) { + rd.progressingIf(recovering, metav1.ConditionFalse, GitTargetReasonRecoveringParentBranch, + "The parent branch is on the remote again; the branch worker is publishing the work it held back") +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 8c812423..bea118ec 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -62,6 +62,8 @@ var ( // just before admission. //nolint:gochecknoglobals beforePushAdmission func() + //nolint:gochecknoglobals + advertiseRemoteBranchFn = advertiseRemoteBranch ) // BranchWorker processes events for a single (GitProvider, Branch) combination. @@ -321,6 +323,18 @@ type BranchWorker struct { // crOwners is the manager's table of which worker holds each request. See commitRequestOwners. // Set by the WorkerManager before Start; nil for a worker built on its own. crOwners *commitRequestOwners + + // clock is the time source of the parent-recovery schedule; nil is time.Now. See now(). + clock func() time.Time + // parentRecoveryOpen, parentRecoveryFound and parentProbeHold mirror the event loop's + // recovery latch for readers outside it. See parent_recovery.go. + parentRecoveryOpen atomic.Bool + parentRecoveryFound atomic.Bool + parentProbeHold atomic.Pointer[parentProbeHold] + // snapshotRequests counts, per GitTarget, the snapshots the worker has asked for. See + // SnapshotRequestSeq. + snapshotRequestsMu sync.Mutex + snapshotRequests map[itypes.ResourceReference]uint64 } // branchWorkerLogFirsts logs the first successful commit and push of a worker's @@ -1102,6 +1116,10 @@ type branchWorkerEventLoop struct { // that no window reaches still resolves on time. An attached request needs no timer of its own: // its window's timers close it. attachTimer *time.Timer + + // recovery is the obligation to write what a missing parent branch held back, and the probe + // that drives it. See parent_recovery.go. + recovery parentRecovery } // commitWindowDefaults are a GitTarget's commit window timers when it declares none. @@ -1147,6 +1165,9 @@ func (l *branchWorkerEventLoop) run() { case <-refusalC: l.refusalTimer = nil l.flushPendingRefusalTouch() + case <-l.recoveryTimerC(): + l.recovery.timer = nil + l.runParentProbe() } // After every wake: bind any waiting CommitRequest to an open window, // resolve any whose attach deadline has passed, and re-arm the deadline timer. @@ -1362,6 +1383,7 @@ func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) l.w.Log.Error(err, "Failed to recover a dirty worktree; dropping atomic request", "events", len(request.Events)) + l.noteParentUnavailable(err, atomicScope(request)) return } @@ -1389,6 +1411,7 @@ func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) l.w.Log.Error(err, "Atomic commit failed; dropping request", "events", len(request.Events)) } + l.noteParentUnavailable(err, atomicScope(request)) return } @@ -1590,7 +1613,7 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali l.w.recordCommitFailure(commitFailureKindWindow, commitFailureError) l.w.Log.Error(err, "Failed to recover a dirty worktree; dropping open window", "reason", string(reason), "windowTarget", windowTarget) - l.dropOpenWindow(pendingCR, err) + l.dropFailedWindow(err, pendingCR, err, windowScopes(targetNamespace, targetName, events)) return false } @@ -1639,7 +1662,8 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali "windowTarget", windowTarget, "events", len(events)) } - l.dropOpenWindow(pendingCR, fmt.Errorf("commit failed: %w", err)) + l.dropFailedWindow(err, pendingCR, fmt.Errorf("commit failed: %w", err), + windowScopes(targetNamespace, targetName, events)) return false } @@ -1679,6 +1703,15 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali // dropOpenWindow discards a window whose finalize failed, resolving any attached // CommitRequest as Failed so the controller does not poll forever. +// dropFailedWindow drops a window whose write failed with err, and when the parent branch is what +// failed it, remembers the scopes the dropped writes belong to so they are recovered. +func (l *branchWorkerEventLoop) dropFailedWindow( + err error, pendingCR *commitRequestID, cause error, scopes []recoveryScope, +) { + l.noteParentUnavailable(err, scopes...) + l.dropOpenWindow(pendingCR, cause) +} + func (l *branchWorkerEventLoop) dropOpenWindow(pendingCR *commitRequestID, cause error) { l.openWindow = nil l.windowBytes = 0 @@ -1725,6 +1758,11 @@ func (l *branchWorkerEventLoop) pushPending() { l.stopPushTimer() return } + if l.w.awaitingParentProbe() { + // The parent is known missing and the probe is not due: it publishes these when it is. + l.stopPushTimer() + return + } // A reset may have discarded the local commits behind these writes while the replay that // rebuilds them did not finish. Pushing now would find the branch already at the remote tip, @@ -1734,10 +1772,12 @@ func (l *branchWorkerEventLoop) pushPending() { l.w.Log.Error(err, "Cannot publish until the retained writes are rebuilt; keeping them", "pendingWrites", len(l.pendingWrites)) l.stopPushTimer() + l.noteParentUnavailable(err) return } if err := l.w.pushPendingCommits(l.pendingWrites); err != nil { + l.noteParentUnavailable(err) l.w.Log.Error(err, "Push failed; pending writes retained for retry", "pendingWrites", len(l.pendingWrites)) // Leave pendingWrites in place; do NOT advance lastPushAt — the @@ -1755,6 +1795,7 @@ func (l *branchWorkerEventLoop) pushPending() { l.pendingWritesBytes = 0 l.lastPushAt = time.Now() l.stopPushTimer() + l.noteRecoveryPublished() } // resolvePushedCommitRequests settles every CommitRequest carried by a just-pushed write, now @@ -1825,6 +1866,10 @@ func (l *branchWorkerEventLoop) stopTimers() { l.stopPushTimer() l.stopAttachTimer() l.stopRefusalTimer() + if l.recovery.timer != nil { + l.recovery.timer.Stop() + l.recovery.timer = nil + } } func (l *branchWorkerEventLoop) stopRefusalTimer() { @@ -1938,6 +1983,9 @@ func (w *BranchWorker) ensureBaseForCycle( if w.baseTrusted() && !w.worktreeDirty() { return nil } + if w.awaitingParentProbe() { + return errAwaitingParentProbe + } // Resolve credentials only on the branch that touches the remote. A cycle planning on a // trusted base never reads the credentials Secret at all, which is a saved API GET per cycle diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go index 1b33e9af..c169f035 100644 --- a/internal/git/branch_worker_empty_repo_test.go +++ b/internal/git/branch_worker_empty_repo_test.go @@ -180,7 +180,8 @@ func TestBranchWorker_AnEmptyRepositoryThatGainsOnlyATagWaits(t *testing.T) { assert.Empty(t, o.ParentRequested) main := branchOnDisk(t, f.repoDir, "main", "a branch at last\n") - f.loop.pushPending() + pastProbeDeadline(f.worker) + f.loop.runParentProbe() tip := f.ref("feature") require.False(t, tip.IsZero()) assert.Equal(t, []plumbing.Hash{main}, f.parentOf(tip)) @@ -229,6 +230,7 @@ func TestBranchWorker_ADanglingRemoteHeadIsMissingNotUnborn(t *testing.T) { // §4.3 test 3: repairing HEAD is enough. gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") + pastProbeDeadline(f.worker) liveWrite(f.loop, "cm2") tip := f.ref("feature") require.False(t, tip.IsZero()) diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go index fc915874..e5e0617c 100644 --- a/internal/git/branch_worker_parent_branch_test.go +++ b/internal/git/branch_worker_parent_branch_test.go @@ -141,6 +141,7 @@ func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testi assert.Equal(t, ParentMissing, observed.ParentState) r1 := f.pushToRelease("RELEASE.md", "now it exists\n") + pastProbeDeadline(f.worker) // until then a known-missing parent is not fetched again liveWrite(loop, "cm2") tip, onRemote := f.featureOnRemote() diff --git a/internal/git/branch_worker_parent_change_test.go b/internal/git/branch_worker_parent_change_test.go index 590e13ed..c697f91e 100644 --- a/internal/git/branch_worker_parent_change_test.go +++ b/internal/git/branch_worker_parent_change_test.go @@ -73,6 +73,12 @@ func pushRoots(t *testing.T, hook func()) *[]plumbing.ReferenceName { return &roots } +// pastProbeDeadline moves the worker's clock past any parent-probe deadline, so a test can stand in +// for the time a held-back parent waits before it is looked for again. +func pastProbeDeadline(w *BranchWorker) { + w.clock = func() time.Time { return time.Now().Add(time.Hour) } +} + // observations collects every observation the worker reports. func observations(f *newBranchFixture) *[]RemoteObservation { var seen []RemoteObservation @@ -222,9 +228,10 @@ func TestBranchWorker_AParentChangeToAMissingParentKeepsTheWork(t *testing.T) { require.NotEmpty(t, loop.pendingWrites, "the work is retained") release := f.pushToRelease("RELEASE.md", "now it exists\n") - loop.pushPending() + pastProbeDeadline(f.worker) + loop.runParentProbe() tip, onRemote := f.featureOnRemote() - require.True(t, onRemote) + require.True(t, onRemote, "the probe published the retained work") assert.Equal(t, release, f.commitParent(tip)) }) @@ -247,7 +254,8 @@ func TestBranchWorker_AParentChangeToAMissingParentKeepsTheWork(t *testing.T) { beforePushAdmission = nil release := f.pushToRelease("RELEASE.md", "now it exists\n") - loop.pushPending() + pastProbeDeadline(f.worker) + loop.runParentProbe() tip, onRemote := f.featureOnRemote() require.True(t, onRemote) assert.Equal(t, release, f.commitParent(tip)) diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 0c2c1400..4b22953d 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -61,6 +61,8 @@ type ledgerFixture struct { // pending is the retained-write slice the event loop would own. The ops drive commit and push // directly, so they carry it here. pending []PendingWrite + // loop is the event loop, for the rows whose state lives on it. + loop *branchWorkerEventLoop } // newLedgerFixture builds that world. seeded chooses between an empty remote and one that already @@ -189,6 +191,9 @@ type ledgerOp struct { slug string // seeded starts the remote with a commit on main. seeded bool + // branch is the worker's write branch; empty is main. A branch the remote does not carry is a + // target on standby. + branch string // prime puts the fixture in the state the row describes. It runs BEFORE the measurement // starts, so a row about steady-state publication is not charged for the initial clone. prime func(f *ledgerFixture) @@ -366,6 +371,63 @@ func ledgerOperations() []ledgerOp { <-done }, }, + { + // H3: a target on standby, and a live write that changes nothing (a deployment's echo, + // here the delete of an object Git never had). The branch stays absent, and the push + // advertisement that confirms the parent is the only connection. + name: "13. standby, a live write that changes nothing", + slug: "standby-noop", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) }, + run: func(f *ledgerFixture) { + deleted := configMapEvent("never-written", "alice", "team-a") + deleted.Operation = "DELETE" + deleted.Object = nil + pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, []Event{deleted}) + require.NoError(f.t, err) + require.NoError(f.t, f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + f.pending = append(f.pending, *pendingWrite) + f.push() + }, + }, + { + // H5: a target on standby whose parent moved since its checkout. The advertisement + // refuses the stale root client-side, one contention fetch replays onto the new tip, + // and the push creates the branch there. It is row 6, for a new branch. + name: "14. standby, the first write after the parent moved", + slug: "standby-parent-moved", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { + require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + f.contend("MOVED.md", "the parent moved\n") + }, + run: func(f *ledgerFixture) { f.publish("first") }, + }, + { + // §4.4: while the parent is missing, the worker's probe is one advertisement per + // deadline and never a fetch. How often it runs is test 4 of parent_recovery_test.go. + name: "15. parent probe while the parent is missing", + slug: "parent-probe", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { + f.worker.SetParentBranch("release") + f.loop = newBranchWorkerEventLoop(f.worker, time.Hour) + f.t.Cleanup(f.loop.stopTimers) + pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, + []Event{configMapEvent("held", "alice", "team-a")}) + require.NoError(f.t, err) + err = f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false) + require.ErrorIs(f.t, err, ErrParentBranchNotFound) + f.loop.noteParentUnavailable(err) + }, + run: func(f *ledgerFixture) { + f.worker.clock = func() time.Time { return time.Now().Add(time.Hour) } + f.loop.runParentProbe() + }, + }, } } @@ -420,7 +482,11 @@ func TestGitRoundTripLedger(t *testing.T) { for _, op := range ops { row := ledgerRow{Operation: op.name} ok := t.Run(op.name, func(t *testing.T) { - f := newLedgerFixture(t, op.slug, op.seeded) + branch := op.branch + if branch == "" { + branch = "main" + } + f := newLedgerFixtureOnBranch(t, op.slug, op.seeded, branch) if op.prime != nil { op.prime(f) } @@ -442,19 +508,20 @@ func TestGitRoundTripLedger(t *testing.T) { // A harness that measures nothing measures nothing consistently, and a golden file is happy // to record that. Row 12 is meant to be zero; everything else is not. - for _, row := range rows[:len(rows)-1] { + const idleRow = "12. an idle target held across several commit windows" + for _, row := range rows { + if row.Operation == idleRow { + // Row 12 is a property, not a measurement: an idle target is silent. It is asserted + // here directly so it can never be "accepted" by regenerating the golden file. + require.Zero(t, row.Snapshot.connections(), + "an idle target must not talk to the Git host at all; it opened %s", row.Snapshot.exactBytes()) + continue + } require.Positive(t, row.Snapshot.connections(), "%s recorded no traffic at all, so the harness — not the code — is what this row is "+ "describing", row.Operation) } - // Row 12 is a property, not a measurement: an idle target is silent. It is asserted here - // directly so it can never be "accepted" by regenerating the golden file. - idle := rows[len(rows)-1] - require.Equal(t, "12. an idle target held across several commit windows", idle.Operation) - require.Zero(t, idle.Snapshot.connections(), - "an idle target must not talk to the Git host at all; it opened %s", idle.Snapshot.exactBytes()) - assertLedgerGolden(t, rows) } diff --git a/internal/git/parent_recovery.go b/internal/git/parent_recovery.go new file mode 100644 index 00000000..a22a51e7 --- /dev/null +++ b/internal/git/parent_recovery.go @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "errors" + "fmt" + "time" + + "github.com/go-git/go-git/v6/plumbing" + + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// This file is the worker's recovery from a parent branch that is missing or does not resolve: +// work it could not write then, and the obligation to write it once the parent exists. +// +// An observation flip is not an obligation. "The parent is Found again" says nothing about whether +// the writes held back meanwhile have landed: the first attempt after it can fail transiently, the +// push timer is stopped by a failed push, the refresher skips a branch that holds work, and with +// --git-refresh-interval=0 there is no refresher at all. So the worker latches an obligation the +// moment a cycle fails on the parent, and clears it only when the work is published. +// +// The obligation has two halves: +// +// - Retained writes: the loop's pendingWrites. Nothing extra is needed to remember them; the +// obligation is open while any are held, and pushPending publishing them is what closes it. +// - Dropped writes, per (GitTarget, collection) scope: a live window or a resync the worker had +// to drop because the parent was missing. The worker cannot re-derive what the cluster holds, +// so it asks the controller for a snapshot (SnapshotRequestSeq), and a scope clears only when +// a resync of it is published: pushed, or settled as a no-op. A successful resync reply alone +// is not enough, because its write may still be waiting for the push. Scopes are per +// collection because a ConfigMap snapshot of target A proves nothing about a Deployment of A, +// or about target B on the same worker. +// +// The worker probes for the parent itself, on the event loop, with one advertisement per deadline: +// 10s, doubling, capped at 5m, shared by every target on the worker. Until the parent is found, +// nothing else may spend a connection on it: a refresh tick before the deadline is a no-op, and a +// live write or resync fails at once without a fetch. Reconciles never probe. + +const ( + parentProbeInitialBackoff = 10 * time.Second + parentProbeMaxBackoff = 5 * time.Minute + parentProbeBackoffFactor = 2 +) + +// errAwaitingParentProbe is the answer to a cycle that would fetch for a parent already known to be +// missing, before the next probe is due. It is a missing parent as far as every caller is +// concerned, and costs no connection. +var errAwaitingParentProbe = fmt.Errorf("%w (unchanged since the last look; waiting for the next probe)", + ErrParentBranchNotFound) + +// isParentUnavailable reports an error caused by the parent a new write branch would be created +// from: a configured parent that is absent, or a default branch that does not resolve. +func isParentUnavailable(err error) bool { + return errors.Is(err, ErrParentBranchNotFound) || errors.Is(err, ErrDefaultBranchUnresolved) +} + +// recoveryScope is one slice of a GitTarget whose writes were dropped. A zero collection is a write +// whose collection is unknown, which any resync of the target covers. +type recoveryScope struct { + target itypes.ResourceReference + collection itypes.CollectionKey +} + +// parentRecovery is the loop's half of the obligation. Loop-goroutine only. +type parentRecovery struct { + // active is the latch. + active bool + // found records that the parent was seen since the latch was set (or since it was last lost). + found bool + // gen is the parent generation the latch was set under; a change makes the probe due at once. + gen uint64 + + // scopes are the dropped writes still owed a published snapshot. + scopes map[recoveryScope]struct{} + // awaitingPush are scopes a resync has re-derived into a commit that is not pushed yet. + awaitingPush map[recoveryScope]struct{} + + backoff time.Duration + nextProbeAt time.Time + timer *time.Timer +} + +// now is the loop's clock, injectable so the probe schedule can be tested without sleeping. +func (w *BranchWorker) now() time.Time { + if w.clock != nil { + return w.clock() + } + return time.Now() +} + +// noteParentUnavailable latches the obligation when err was caused by the parent, and remembers +// the scopes whose writes were dropped for it. Any other error is not this obligation's concern. +func (l *branchWorkerEventLoop) noteParentUnavailable(err error, scopes ...recoveryScope) { + if !isParentUnavailable(err) { + return + } + r := &l.recovery + switch { + case !r.active: + r.active = true + r.backoff = parentProbeInitialBackoff + r.nextProbeAt = l.w.now().Add(r.backoff) + r.gen = l.w.parentSnapshot().gen + case r.found: + // Found, then lost again on the attempt that followed: wait a whole backoff before asking. + r.nextProbeAt = l.w.now().Add(r.backoff) + } + r.found = false + for _, scope := range scopes { + if r.scopes == nil { + r.scopes = map[recoveryScope]struct{}{} + } + r.scopes[scope] = struct{}{} + } + l.armRecoveryTimer() + l.publishRecovery() +} + +// windowScopes are the scopes of a dropped live window: one per collection its events came from. +func windowScopes(namespace, name string, events []Event) []recoveryScope { + target := itypes.NewResourceReference(name, namespace) + seen := map[itypes.CollectionKey]bool{} + var scopes []recoveryScope + for i := range events { + collection := events[i].SourceCollection + if !seen[collection] { + seen[collection] = true + scopes = append(scopes, recoveryScope{target: target, collection: collection}) + } + } + return scopes +} + +// atomicScope is the scope of a dropped atomic write. +func atomicScope(request *WriteRequest) recoveryScope { + name, namespace := atomicRefusalTarget(request) + return recoveryScope{target: itypes.NewResourceReference(name, namespace), collection: request.sourceCollection()} +} + +// resyncScope is the scope of a resync. +func resyncScope(req *ResyncRequest) recoveryScope { + return recoveryScope{ + target: itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), + collection: req.refusalCollection(), + } +} + +// noteResyncApplied records that a resync of one scope succeeded. A resync succeeds only after +// fetching the parent, so it is also proof the parent is back. A no-op settles the scope now; one +// that committed settles it when its commit is pushed. +func (l *branchWorkerEventLoop) noteResyncApplied(namespace, name string, collection itypes.CollectionKey, + committed bool) { + r := &l.recovery + if !r.active { + return + } + target := itypes.NewResourceReference(name, namespace) + for scope := range r.scopes { + if scope.target != target || !resyncCovers(collection, scope.collection) { + continue + } + if committed { + if r.awaitingPush == nil { + r.awaitingPush = map[recoveryScope]struct{}{} + } + r.awaitingPush[scope] = struct{}{} + } else { + delete(r.scopes, scope) + } + } + l.closeRecoveryIfDone() +} + +// resyncCovers reports whether a resync of one collection re-derives a dropped scope: the same +// collection, or either side unknown (a resync of the whole target, or a write whose collection is +// not known). +func resyncCovers(resynced, dropped itypes.CollectionKey) bool { + var unknown itypes.CollectionKey + return resynced == dropped || resynced == unknown || dropped == unknown +} + +// noteRecoveryPublished runs after every successful push: the scopes whose snapshot it carried are +// settled, and the retained writes it published were the other half. +func (l *branchWorkerEventLoop) noteRecoveryPublished() { + r := &l.recovery + if !r.active { + return + } + for scope := range r.awaitingPush { + delete(r.scopes, scope) + } + r.awaitingPush = nil + l.closeRecoveryIfDone() +} + +// closeRecoveryIfDone clears the latch once nothing is owed. +func (l *branchWorkerEventLoop) closeRecoveryIfDone() { + r := &l.recovery + if !r.active || len(l.pendingWrites) > 0 || len(r.scopes) > 0 { + return + } + if r.timer != nil { + r.timer.Stop() + } + *r = parentRecovery{} + l.publishRecovery() +} + +// probeDue reports whether the next background look at the remote may be spent now. +func (l *branchWorkerEventLoop) probeDue() bool { + r := &l.recovery + return r.active && (!l.w.now().Before(r.nextProbeAt) || r.gen != l.w.parentSnapshot().gen) +} + +// runParentProbe is the deadline: while the parent is missing it spends one advertisement; once it +// is found it retries the retained writes and asks again for the snapshots still owed. Whatever +// remains open is scheduled for the next deadline. +func (l *branchWorkerEventLoop) runParentProbe() { + r := &l.recovery + if !r.active { + return + } + if gen := l.w.parentSnapshot().gen; gen != r.gen { + r.gen, r.found, r.backoff = gen, false, parentProbeInitialBackoff + } + if !r.found { + missing, err := l.probeParent() + if err != nil || missing { + if err != nil { + l.w.Log.V(1).Info("Parent branch probe failed", "branch", l.w.Branch, "error", err.Error()) + } + l.scheduleNextProbe() + return + } + r.found = true + r.backoff = parentProbeInitialBackoff + l.publishRecovery() + l.w.Log.Info("The parent branch is on the remote again; recovering held-back work", + "branch", l.w.Branch, "parentBranch", l.w.ParentBranch(), + "pendingWrites", len(l.pendingWrites), "scopes", len(r.scopes)) + } + if len(l.pendingWrites) > 0 { + l.pushPending() + } + if r.active && r.found { + for target := range l.recoveryTargets() { + l.w.bumpSnapshotRequest(target) + } + } + if r.active { + l.scheduleNextProbe() + } +} + +// recoveryTargets are the GitTargets owed a snapshot. +func (l *branchWorkerEventLoop) recoveryTargets() map[itypes.ResourceReference]struct{} { + targets := map[itypes.ResourceReference]struct{}{} + for scope := range l.recovery.scopes { + if _, waiting := l.recovery.awaitingPush[scope]; !waiting { + targets[scope.target] = struct{}{} + } + } + return targets +} + +// scheduleNextProbe doubles the backoff and moves the deadline that far on: the first deadline is +// parentProbeInitialBackoff after the latch, the next twice that after it, up to the cap. +func (l *branchWorkerEventLoop) scheduleNextProbe() { + r := &l.recovery + r.backoff = min(parentProbeBackoffFactor*r.backoff, parentProbeMaxBackoff) + r.nextProbeAt = l.w.now().Add(r.backoff) + l.armRecoveryTimer() + l.publishRecovery() +} + +// probeParent reads one advertisement and records what it says. It reports whether the parent is +// still missing; a write branch that now exists is not. +func (l *branchWorkerEventLoop) probeParent() (bool, error) { + w := l.w + provider, err := w.getGitProvider(w.ctx) + if err != nil { + return true, fmt.Errorf("get GitProvider: %w", err) + } + auth, err := getAuthFromSecret(w.ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) + if err != nil { + return true, fmt.Errorf("get auth: %w", err) + } + parent := w.parentSnapshot() + advertisement, err := advertiseRemoteBranchFn( + w.repo.URL, + plumbing.NewBranchReferenceName(w.Branch), + parent.name, + auth, + ) + if err != nil { + return true, fmt.Errorf("read the remote advertisement: %w", err) + } + return w.recordAdvertisement(advertisement, parent.name), nil +} + +func (l *branchWorkerEventLoop) armRecoveryTimer() { + r := &l.recovery + if r.timer != nil { + r.timer.Stop() + r.timer = nil + } + if !r.active { + return + } + r.timer = time.NewTimer(max(r.nextProbeAt.Sub(l.w.now()), 0)) +} + +func (l *branchWorkerEventLoop) recoveryTimerC() <-chan time.Time { + if l.recovery.timer == nil { + return nil + } + return l.recovery.timer.C +} + +// publishRecovery mirrors the latch for the readers outside the loop: the controller, and the +// fetch gate the write path consults. +func (l *branchWorkerEventLoop) publishRecovery() { + r := &l.recovery + l.w.parentRecoveryOpen.Store(r.active) + l.w.parentRecoveryFound.Store(r.active && r.found) + if r.active && !r.found { + l.w.parentProbeHold.Store(&parentProbeHold{until: r.nextProbeAt, gen: r.gen}) + } else { + l.w.parentProbeHold.Store(nil) + } +} + +// parentProbeHold is the window in which a missing parent is not looked for again. +type parentProbeHold struct { + until time.Time + gen uint64 +} + +// awaitingParentProbe reports that the parent is known missing and the next probe is not due, so +// a cycle that would fetch for it must not. A parent change lifts it. +func (w *BranchWorker) awaitingParentProbe() bool { + hold := w.parentProbeHold.Load() + return hold != nil && hold.gen == w.parentSnapshot().gen && w.now().Before(hold.until) +} + +// bumpSnapshotRequest asks the controller for a fresh snapshot of one GitTarget. +func (w *BranchWorker) bumpSnapshotRequest(target itypes.ResourceReference) { + w.snapshotRequestsMu.Lock() + defer w.snapshotRequestsMu.Unlock() + if w.snapshotRequests == nil { + w.snapshotRequests = map[itypes.ResourceReference]uint64{} + } + w.snapshotRequests[target]++ +} + +// BumpSnapshotRequestForTest asks for a snapshot of the GitTarget, as the recovery probe does, for +// tests in another package. +func (w *BranchWorker) BumpSnapshotRequestForTest(target itypes.ResourceReference) { + w.bumpSnapshotRequest(target) +} + +// SnapshotRequestSeq is how many times the worker has asked for a fresh snapshot of the GitTarget, +// to re-derive writes it dropped while the parent branch was missing. The controller forces one +// recheck each time it rises, and remembers the value it acted on. +func (w *BranchWorker) SnapshotRequestSeq(target itypes.ResourceReference) uint64 { + w.snapshotRequestsMu.Lock() + defer w.snapshotRequestsMu.Unlock() + return w.snapshotRequests[target] +} + +// ParentRecovery reports whether the worker still owes work held back by a missing parent (the +// first result), and whether the parent has been found since (the second). +func (w *BranchWorker) ParentRecovery() (bool, bool) { + return w.parentRecoveryOpen.Load(), w.parentRecoveryFound.Load() +} diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go new file mode 100644 index 00000000..9c32f481 --- /dev/null +++ b/internal/git/parent_recovery_test.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// Work a missing parent branch held back is an obligation, not an observation: it stays open until +// the work is published, so a transient failure after the parent reappears cannot strand it, and +// it is driven by the worker on its own deadline, whatever the refresh configuration. These pin +// the obligation, its per-scope bookkeeping, and the probe budget. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// testClock is a clock the test moves by hand. +type testClock struct { + mu sync.Mutex + now time.Time +} + +func useTestClock(w *BranchWorker) *testClock { + c := &testClock{now: time.Now()} + w.clock = c.Now + return c +} + +func (c *testClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *testClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +// failNextSync makes the next sync the worker attempts fail as a network error would. +func failNextSync(t *testing.T) { + t.Helper() + original := syncToRemoteFn + failed := false + syncToRemoteFn = func(ctx context.Context, repo *gogit.Repository, target plumbing.ReferenceName, + parent string, auth []gitclient.Option) (*PullReport, error) { + if !failed { + failed = true + return nil, errors.New("connection reset by peer") + } + return original(ctx, repo, target, parent, auth) + } + t.Cleanup(func() { syncToRemoteFn = original }) +} + +// Test 1. Writes retained while the parent is missing are published once it exists, through the +// worker's own deadline, even when the first attempt after the parent reappears fails. No cluster +// edit is involved, and it works with the refresher on or off. +func TestParentRecovery_RetainedWritesLandAfterAFailedFirstAttempt(t *testing.T) { + for name, viaRefresh := range map[string]bool{"refresh off": false, "refresh on": true} { + t.Run(name, func(t *testing.T) { + f := newRealServerNewBranch( + t, + "recovery-retained-"+map[bool]string{false: "timer", true: "refresh"}[viaRefresh], + func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "release", "main") + }, + ) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + deadline := func() { + if viaRefresh { + f.loop.handleRefreshRequest(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + return + } + f.loop.runParentProbe() // what the probe timer does + } + + f.loop.lastPushAt = time.Now() + liveWrite(f.loop, "cm1") + require.Len(t, f.loop.pendingWrites, 1) + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") // the parent goes away + f.loop.pushPending() + require.Len(t, f.loop.pendingWrites, 1, "kept") + open, found := f.worker.ParentRecovery() + require.True(t, open) + require.False(t, found) + + if viaRefresh { + before := f.mark() + deadline() + assert.Zero(t, f.mark().since(before).connections(), "a refresh tick before the deadline costs nothing") + } + + release := simulateClientCommitOnDisk(t, f.repoDir, "release", "RELEASE.md", "back\n") + failNextSync(t) + clock.advance(parentProbeMaxBackoff) + deadline() + + assert.True(t, f.ref("feature").IsZero(), "the first attempt failed") + require.Len(t, f.loop.pendingWrites, 1, "and the work is still held") + open, found = f.worker.ParentRecovery() + require.True(t, open, "the obligation outlives a failed attempt") + assert.True(t, found) + + clock.advance(parentProbeMaxBackoff) + deadline() + + tip := f.ref("feature") + require.False(t, tip.IsZero(), "the next deadline published it, with no cluster edit") + assert.Equal(t, []plumbing.Hash{release}, f.parentOf(tip)) + assert.Empty(t, f.loop.pendingWrites) + open, _ = f.worker.ParentRecovery() + assert.False(t, open, "published: nothing is owed") + }) + } +} + +func refreshTarget() itypes.ResourceReference { + return itypes.NewResourceReference(newBranchTarget, "default") +} + +// Test 2. Writes dropped for two targets on one worker. Each scope clears only when a resync of it +// is published; a resync that committed but is not pushed yet does not clear it, and a target +// whose resync failed is asked again on the next deadline. +func TestParentRecovery_DroppedWritesClearPerScopeWhenPublished(t *testing.T) { + f := newNewBranchFixture(t, nil) + clock := useTestClock(f.worker) + createPlainGitTarget(t, f.worker, "other-target", "other") + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + a := itypes.NewResourceReference(newBranchTarget, "default") + b := itypes.NewResourceReference("other-target", "default") + collectionA := itypes.CollectionKey{Resource: "configmaps", Namespace: "default"} + collectionB := itypes.CollectionKey{Resource: "configmaps", Namespace: "default", LabelSelector: "team=b"} + write := func(target string, collection itypes.CollectionKey, name string) { + event := configMapTargetEvent(name, "alice", target) + event.SourceCollection = collection + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + } + resync := func(target itypes.ResourceReference, collection itypes.CollectionKey, name string) error { + scope := ResyncScope{Collection: collection} + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCM(name, "blue")}, + ResourceVersion: "1", + GitTargetName: target.Name, + GitTargetNamespace: target.Namespace, + Scope: &scope, + Result: make(chan ResyncResult, 1), + } + loop.applyResync(req) + return (<-req.Result).Err + } + + write(newBranchTarget, collectionA, "a1") + write("other-target", collectionB, "b1") + assert.Empty(t, loop.pendingWrites, "both were dropped") + assert.Len(t, loop.recovery.scopes, 2) + + f.pushToRelease("RELEASE.md", "release\n") + clock.advance(parentProbeMaxBackoff) + loop.runParentProbe() + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(a)) + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(b)) + + loop.lastPushAt = time.Now() // cooldown: A's resync commits, and its push waits + require.NoError(t, resync(a, collectionA, "a1")) + require.Len(t, loop.pendingWrites, 1) + assert.Contains(t, loop.recovery.scopes, recoveryScope{target: a, collection: collectionA}, + "a resync that is not pushed yet does not clear its scope") + + failNextSync(t) + require.Error(t, resync(b, collectionB, "b1")) + + clock.advance(parentProbeMaxBackoff) + loop.runParentProbe() // publishes A's commit, and asks for B again + + assert.Empty(t, loop.pendingWrites) + assert.NotContains(t, loop.recovery.scopes, recoveryScope{target: a, collection: collectionA}, "published") + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(a), "A is not asked again") + assert.Equal(t, uint64(2), f.worker.SnapshotRequestSeq(b), "B is") + + require.NoError(t, resync(b, collectionB, "b1")) + loop.pushPending() + open, _ := f.worker.ParentRecovery() + assert.False(t, open, "every scope is published") +} + +// Test 4. However many targets share the worker, and however often refreshes tick and live writes +// arrive, a parent that stays missing is asked about at most once per deadline of the shared +// schedule, and never fetched. +func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newRealServerNewBranch(t, "recovery-probe-budget", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + }) + clock := useTestClock(f.worker) + createPlainGitTarget(t, f.worker, "second", "second") + createPlainGitTarget(t, f.worker, "third", "third") + f.worker.SetParentBranch("release") + targets := []string{newBranchTarget, "second", "third"} + + liveWrite(f.loop, "first") // the one fetch that finds the parent missing + open, _ := f.worker.ParentRecovery() + require.True(t, open) + + fetches := totalFetches(t, reader, f.worker) + before := f.mark() + probes := 0 + const window = 10 * time.Minute + for elapsed := time.Duration(0); elapsed < window; elapsed += 2 * time.Second { + clock.advance(2 * time.Second) + if f.loop.probeDue() { + probes++ + f.loop.runParentProbe() // the timer + } + for _, target := range targets { + f.loop.handleRefreshRequest(&RefreshRequest{ + Target: itypes.NewResourceReference(target, "default"), MaxAge: time.Nanosecond, + }) + event := configMapTargetEvent("cm-"+target, "alice", target) + f.loop.handleQueueItem( + WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}, + ) + } + } + + // 10s, then doubling to the 5m cap: deadlines at 10s, 30s, 70s, 150s and 310s. + assert.Equal(t, 5, probes) + assert.LessOrEqual(t, f.mark().since(before).connections(), int64(probes), + "one advertisement per deadline, whatever else happens") + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "and no fetch at all") +} + +// Test 5. An unresolved default branch, once repaired, recovers through the same path: the probe +// finds it and asks for the snapshot the dropped write needs. +func TestParentRecovery_ARepairedRemoteHeadAsksForTheSnapshot(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-dangling-head", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "symbolic-ref", "HEAD", "refs/heads/master") + }) + clock := useTestClock(f.worker) + + liveWrite(f.loop, "cm1") + assert.True(t, f.ref("feature").IsZero()) + + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") + clock.advance(parentProbeMaxBackoff) + f.loop.runParentProbe() + + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(refreshTarget())) + open, found := f.worker.ParentRecovery() + assert.True(t, open, "open until the snapshot is published") + assert.True(t, found) +} + +func TestParentRecovery_AParentChangeMakesTheProbeDue(t *testing.T) { + f := newNewBranchFixture(t, nil) + useTestClock(f.worker) + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + liveWrite(loop, "cm1") + require.False(t, loop.probeDue()) + require.True(t, f.worker.awaitingParentProbe()) + + f.worker.SetParentBranch("other") + + assert.True(t, loop.probeDue()) + assert.False(t, f.worker.awaitingParentProbe(), "a new parent is looked for at once") +} diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 9c680a9d..c4f71806 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -94,6 +94,15 @@ func (l *branchWorkerEventLoop) handleRefreshRequest(req *RefreshRequest) { // do no work at all, two lines below. observed, known := w.LastRemoteObservation() + // While work is held back by a missing parent, the refresher shares the worker's one probe + // deadline: a tick before it costs nothing, and a tick after it is the probe. + if l.recovery.active { + if l.probeDue() { + l.runParentProbe() + } + return + } + // 3. Not idle, so not the target this exists for. A reset here would destroy retained // commits, and the worktree may hold a partial write — which is also why this is the one exit // that does not re-read the folder: a layout resolved from a half-written tree is worse than @@ -191,7 +200,7 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr // declared but never published to, and one whose GitProvider was recreated against a // different repository. parentBranch := w.ParentBranch() - advertisement, err := advertiseRemoteBranch( + advertisement, err := advertiseRemoteBranchFn( w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), parentBranch, auth) if err != nil { return fmt.Errorf("read the remote advertisement: %w", err) diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index 52dff404..56f49f39 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -94,6 +94,10 @@ func resyncHealKey(req *ResyncRequest) healKey { // on that flag, so on a target holding work a bare invalidation reached nothing at all and the // snapshot judged a tree nobody had read. That is the exact hazard this function exists to close. func (l *branchWorkerEventLoop) prepareBaseForResync(req *ResyncRequest) error { + // A parent known to be missing is not fetched again before its probe is due. + if l.w.awaitingParentProbe() { + return errAwaitingParentProbe + } // The invalidation is skipped when RefreshRemote already fetched a moment ago, which would // otherwise make a forced recheck pay twice. if req.RefreshRemote { @@ -159,6 +163,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { closedWindow = l.finalizeOpenWindowWithReason(windowFinalizeReasonResyncBeforeApply) } if err := l.prepareBaseForResync(req); err != nil { + l.noteParentUnavailable(err, resyncScope(req)) req.reply(ResyncResult{Err: err}) return } @@ -186,6 +191,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { refusalObservationForDesired(req.Desired, refused), req.refusalCollection()) } l.w.Log.Error(err, "Resync commit failed; dropping request", "resources", len(req.Desired)) + l.noteParentUnavailable(err, resyncScope(req)) req.reply(ResyncResult{Err: err}) return } @@ -206,6 +212,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { l.pendingWrites = append(l.pendingWrites, *pendingWrite) l.pendingWritesBytes += pendingWrite.ByteSize } + l.noteResyncApplied(req.GitTargetNamespace, req.GitTargetName, req.refusalCollection(), committed) // Schedule a push whenever this request CLOSED a live window — that window's // commit is now in pendingWrites and must reach the remote — or the resync itself // committed. Any finalize that closes a window must schedule its push, or the diff --git a/internal/git/testdata/git-roundtrip-ledger.golden b/internal/git/testdata/git-roundtrip-ledger.golden index b84be853..0319eb6e 100644 --- a/internal/git/testdata/git-roundtrip-ledger.golden +++ b/internal/git/testdata/git-roundtrip-ledger.golden @@ -30,3 +30,6 @@ operation conn fetch-open 10. a resync (snapshot) 4 2 -/<1K 0 1 -/<1K 1 <1K/<1K 11. publication onto a branch the remote does not have 2 0 0 1 -/<1K 1 <1K/<1K 12. an idle target held across several commit windows 0 0 0 0 0 +13. standby, a live write that changes nothing 1 0 0 1 -/<1K 0 +14. standby, the first write after the parent moved 6 2 -/<1K 1 <1K/<1K 2 -/<1K 1 <4K/<1K +15. parent probe while the parent is missing 1 1 -/<1K 0 0 0 diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go index 2d254a6c..1f5ece94 100644 --- a/test/e2e/new_branch_parent_e2e_test.go +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -27,13 +27,15 @@ import ( // initial snapshot clones main at A and trusts it before B is pushed. A cold worker would fetch on // its first write and land on B without exercising the check at all. // -// What fixed it is read off the fetch counter: the moved parent must be recovered by a -// `contention` fetch (the rejected push), with no `refresh` fetch in between. That takes the -// periodic refresher out of the proof without redeploying the controller with it disabled, which -// would disturb every other spec on the leg. +// The spec asserts ancestry and content, not which fetch found B: with periodic refresh on, a +// refresh can legitimately find B before the publication does. That publication alone finds a +// moved parent, with no refresher at all, is proved deterministically by the worker tests +// TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip and +// TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent. // // It runs twice: once with spec.parentBranch omitted (the remote's default branch, main), and once -// naming `release`, which the GitProvider does not allow writing to because it is only read. +// naming `release`, which the GitProvider does not allow writing to because it is only read. A +// third context names a parent that does not exist yet, and recovers once it is pushed. var _ = Describe("New write branch starts from its parent", Label("manager"), func() { Context("with spec.parentBranch omitted", Ordered, func() { describeNewWriteBranchSpec("", "main", "parent-branch") @@ -41,15 +43,158 @@ var _ = Describe("New write branch starts from its parent", Label("manager"), fu Context("with spec.parentBranch naming release", Ordered, func() { describeNewWriteBranchSpec("release", "release", "parent-release") }) + Context("with spec.parentBranch naming a branch that does not exist yet", Ordered, func() { + describeMissingParentRecoverySpec() + }) }) +const parentBranchSelectorLabel = "e2e.configbutler.ai/parent-branch" + +// applyParentConfigMap applies a ConfigMap the parent-branch specs' WatchRule selects. +func applyParentConfigMap(namespace, name string) { + GinkgoHelper() + manifest := fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s + labels: + %s: "yes" +data: + key: %s +`, name, namespace, parentBranchSelectorLabel, name) + out, err := kubectlRunWithStdin(namespace, manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply ConfigMap %q: %s", name, out) +} + +// applyParentSpecProvider declares the GitProvider the parent-branch specs write through, allowing +// main and the write branch only: a parent other than main is read, never written. +func applyParentSpecProvider(namespace, providerName string, repo *RepoArtifacts, writeBranch string) { + GinkgoHelper() + provider := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: GitProvider +metadata: + name: %s + namespace: %s +spec: + url: %s + allowedBranches: ["main", %q] + secretRef: + name: %s +`, providerName, namespace, repo.RepoURLHTTP, writeBranch, repo.GitSecretHTTP) + out, err := kubectlRunWithStdin(namespace, provider, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply GitProvider: %s", out) + verifyResourceStatus("gitprovider", providerName, namespace, "True", "Succeeded", "") +} + +// applyParentSpecWatchRule declares the WatchRule that selects the specs' labelled ConfigMaps. +func applyParentSpecWatchRule(namespace, ruleName, destName string) { + GinkgoHelper() + rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + gitTargetRef: + name: %s + rules: + - apiGroups: [""] + resources: ["configmaps"] + objectSelector: + matchLabels: + %s: "yes" +`, ruleName, namespace, destName, parentBranchSelectorLabel) + out, err := kubectlRunWithStdin(namespace, rule, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply WatchRule: %s", out) +} + +// describeMissingParentRecoverySpec: a configured parent that does not exist refuses, writes +// nothing and creates no branch. Once the parent is pushed, the target recovers with no further +// cluster edit: the branch worker finds the parent on its own probe, asks for a fresh snapshot, +// and the edit made while the parent was missing lands on the parent's tip. +func describeMissingParentRecoverySpec() { + const ( + providerName = "missing-parent-provider" + destName = "missing-parent-dest" + ruleName = "missing-parent-rule" + gitPath = "apps/missing-parent" + ) + var ( + testNs string + repo *RepoArtifacts + writeBranch string + ) + + BeforeAll(func() { + testNs = testNamespaceFor("manager-missing-parent") + _, _ = kubectlRun("create", "namespace", testNs) + writeBranch = fmt.Sprintf("reverser-e2e-%d", GinkgoRandomSeed()) + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-missing-parent-%d", GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + applyParentSpecProvider(testNs, providerName, repo, writeBranch) + }) + + AfterAll(func() { + cleanupWatchRule(ruleName, testNs) + cleanupGitTarget(destName, testNs) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", providerName, "--ignore-not-found=true") + cleanupNamespace(testNs) + }) + + It("refuses while the parent is missing, then recovers on its own once it is pushed", func() { + commitFilesToBranchFromOutside(repo, testNs, "main", "e2e: seed main", map[string]string{ + "README.md": "main is not the parent here\n", + }) + createGitTargetWithParentBranch(destName, testNs, providerName, gitPath, writeBranch, "release") + applyParentSpecWatchRule(testNs, ruleName, destName) + + By("an edit made while release does not exist is not written") + applyParentConfigMap(testNs, "made-while-missing") + Eventually(func(g Gomega) { + g.Expect(readyReasonOf(g, destName, testNs)).To(Equal("ParentBranchNotFound")) + }, 3*time.Minute, 3*time.Second).Should(Succeed()) + Expect(remoteBranchHeadOf(Default, repo.CheckoutDir, writeBranch)).To(BeEmpty(), "no orphan branch") + + By("release is pushed from outside; nothing in the cluster changes after this") + releaseTip := commitFilesToBranchFromOutside(repo, testNs, "release", "e2e: release appears", + map[string]string{"RELEASE.md": "release\n"}) + + By("the write branch is created on release's tip, carrying the edit made while it was missing") + editPath := path.Join(gitPath, testNs, "configmaps", "made-while-missing.yaml") + Eventually(func(g Gomega) { + tip := remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch) + g.Expect(tip).NotTo(BeEmpty(), "the recovery creates the write branch") + _, fetchErr := gitRun(repo.CheckoutDir, "fetch", "origin", writeBranch) + g.Expect(fetchErr).NotTo(HaveOccurred()) + first, revErr := gitRun(repo.CheckoutDir, "rev-list", "--max-parents=1", "--reverse", + "origin/release.."+tip) + g.Expect(revErr).NotTo(HaveOccurred()) + commits := strings.Fields(first) + g.Expect(commits).NotTo(BeEmpty()) + parentOfFirst, revErr := gitRun(repo.CheckoutDir, "rev-parse", commits[0]+"^") + g.Expect(revErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(parentOfFirst)).To(Equal(releaseTip), + "the branch's first commit sits on release's tip") + _, showErr := gitRun(repo.CheckoutDir, "cat-file", "-e", tip+":"+editPath) + g.Expect(showErr).NotTo(HaveOccurred(), "the edit made while release was missing is on the branch") + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(readyReasonOf(g, destName, testNs)).NotTo( + BeElementOf("ParentBranchNotFound", "RecoveringParentBranch")) + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + }) +} + func describeNewWriteBranchSpec(parentBranch, parent, slug string) { const ( - providerName = "parent-provider" - destName = "parent-dest" - ruleName = "parent-rule" - gitPath = "apps/parent" - selectorLabel = "e2e.configbutler.ai/parent-branch" + providerName = "parent-provider" + destName = "parent-dest" + ruleName = "parent-rule" + gitPath = "apps/parent" ) var ( testNs string @@ -59,18 +204,7 @@ func describeNewWriteBranchSpec(parentBranch, parent, slug string) { applyConfigMap := func(name string) { GinkgoHelper() - manifest := fmt.Sprintf(`apiVersion: v1 -kind: ConfigMap -metadata: - name: %s - namespace: %s - labels: - %s: "yes" -data: - key: %s -`, name, testNs, selectorLabel, name) - out, err := kubectlRunWithStdin(testNs, manifest, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred(), "apply ConfigMap %q: %s", name, out) + applyParentConfigMap(testNs, name) } // mirroredDocument is the document the controller writes for applyConfigMap(name). @@ -84,19 +218,13 @@ metadata: namespace: %s data: key: %s -`, selectorLabel, name, testNs, name) +`, parentBranchSelectorLabel, name, testNs, name) } branchSeries := func(metric, extra string) string { return fmt.Sprintf(`sum(%s{provider_namespace=%q,branch=%q%s}) or vector(0)`, metric, testNs, writeBranch, extra) } - fetches := func(reason string) float64 { - GinkgoHelper() - v, err := queryPrometheus(branchSeries("gitopsreverser_git_fetches_total", fmt.Sprintf(",reason=%q", reason))) - Expect(err).NotTo(HaveOccurred()) - return v - } pushes := func() float64 { GinkgoHelper() v, err := queryPrometheus(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`)) @@ -115,20 +243,7 @@ data: Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") applySOPSAgeKeyToNamespace(testNs) - provider := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 -kind: GitProvider -metadata: - name: %s - namespace: %s -spec: - url: %s - allowedBranches: ["main", %q] - secretRef: - name: %s -`, providerName, testNs, repo.RepoURLHTTP, writeBranch, repo.GitSecretHTTP) - out, err := kubectlRunWithStdin(testNs, provider, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred(), "apply GitProvider: %s", out) - verifyResourceStatus("gitprovider", providerName, testNs, "True", "Succeeded", "") + applyParentSpecProvider(testNs, providerName, repo, writeBranch) }) AfterAll(func() { @@ -152,23 +267,7 @@ spec: By("creating a GitTarget on an absent write branch, and a WatchRule that selects nothing yet") createGitTargetWithParentBranch(destName, testNs, providerName, gitPath, writeBranch, parentBranch) - rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 -kind: WatchRule -metadata: - name: %s - namespace: %s -spec: - gitTargetRef: - name: %s - rules: - - apiGroups: [""] - resources: ["configmaps"] - objectSelector: - matchLabels: - %s: "yes" -`, ruleName, testNs, destName, selectorLabel) - out, err := kubectlRunWithStdin(testNs, rule, "apply", "-f", "-") - Expect(err).NotTo(HaveOccurred(), "apply WatchRule: %s", out) + applyParentSpecWatchRule(testNs, ruleName, destName) verifyResourceStatus("watchrule", ruleName, testNs, "True", "Succeeded", "") waitForWatchRuleStreamsRunning(ruleName, testNs) @@ -205,7 +304,6 @@ spec: "a write that commits nothing must not create the write branch") By(parent + " moves to B from outside, touching the target's folder") - contentionBefore, refreshBefore := fetches("contention"), fetches("refresh") hashB := commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: B moves the parent", map[string]string{ path.Join(gitPath, "NOTES.md"): "added on the parent at B\n", }) @@ -229,13 +327,6 @@ spec: g.Expect(showErr).NotTo(HaveOccurred(), "%s must be on the new branch", file) } }, 2*time.Minute, 3*time.Second).Should(Succeed()) - - By("and the publication check, not the refresher, is what found B") - waitForMetricWithTimeout(branchSeries("gitopsreverser_git_fetches_total", `,reason="contention"`), - func(v float64) bool { return v > contentionBefore }, - "the rejected push reset onto the moved parent", time.Minute) - Expect(fetches("refresh")).To(Equal(refreshBefore), - "no refresh fetch may stand in for the publication check") }) } From 54d5e68ca5512a509fc0b80999a1c9703fbe6616 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:52:58 +0000 Subject: [PATCH 15/18] fix(git): follow a switched default branch, and replay onto a write branch created during our upload R5. With spec.parentBranch omitted the parent is the default branch as last discovered, but the refresher compared the parent's hash only: a HEAD switched from main to trunk at the same commit was never followed, and no-op pushes kept recording main while refreshes recorded trunk. followParent now also compares the advertised parent branch with the one the checkout is rooted on, and follows a mismatch with one fetch. Publication still cannot see HEAD; that is documented. R6. When somebody created the write branch between our advertisement and our upload, the server refused the upload (Old was zero) as an untyped error, the fallback probed the root only, found the parent unmoved, and the work never replayed. On that path only, the fallback now reads both refs from one advertisement and treats a write branch that is now there as contention. Its ledger row measures 8 connections: the refused upload, one advertisement, the contention fetch, the push. Also rewords the "parent branch" vocabulary rule: a bare "parent" is fine where the subject is already a branch, such as status.remote.parent. Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 3 + docs/definitions.md | 8 +- internal/git/branch_worker.go | 26 ++++++ internal/git/branch_worker_empty_repo_test.go | 81 +++++++++++++++++++ internal/git/git_atomic_push.go | 9 +++ internal/git/git_roundtrip_ledger_test.go | 19 +++++ internal/git/refresh.go | 28 ++++++- .../git/testdata/git-roundtrip-ledger.golden | 1 + 8 files changed, 169 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b704a8fa..e9288831 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -783,6 +783,9 @@ spec: - A detached `HEAD`, or a server that does not say what `HEAD` points at, resolves the way go-git resolves it: to `master` when it is at `HEAD`'s commit, otherwise to the alphabetically first branch that is. Set `parentBranch` when you need certainty rather than that guess. +- When the remote's `HEAD` switches to another branch, the next discovery (a refresh, or any fetch) + follows it. A push cannot see `HEAD`, so with periodic refresh off a write branch can still start + from the previous default branch until then. - Once the write branch exists, it is used as it is: new commits on the parent do not reach it. Delete the write branch after its changes are merged, and the next edit starts a fresh one. - `parentBranch` is immutable, including adding or removing it: delete and recreate the target to diff --git a/docs/definitions.md b/docs/definitions.md index 0df60e75..789611c5 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -129,9 +129,11 @@ branch)` tuple. Every write to that branch goes through it. Two words, lowercase **Parent branch.** The branch a write branch is created from, and compared against while the write branch does not exist on the remote: `spec.parentBranch`, or the remote's default branch when that -is omitted. Always with its noun: "parent" alone already names a parent resource. Not "base", which -names a kustomize read-only base and base trust, and not "source", which names the source cluster. -Once the write branch exists, the parent is not followed. +is omitted. Say "parent branch"; a bare "parent" is fine where the subject is already a branch, such +as the `status.remote.parent` stanza or a sentence that has just named the parent branch. Use the +full term wherever a parent *resource* could be meant. Not "base", which names a kustomize read-only +base and base trust, and not "source", which names the source cluster. Once the write branch exists, +the parent branch is not followed. **Commit window.** The interval a branch worker batches one author's writes to one target over before it commits. Only a write opens one; nothing else does. It closes on its `idleTimeout` of silence or diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index bea118ec..c85be642 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -2309,6 +2309,9 @@ func (w *BranchWorker) remoteMovedDuringPush( } remoteHash, known := advertisedRootHash(pushErr, rootBranch) + if !known && rootBranch != plumbing.NewBranchReferenceName(w.Branch) { + return w.newBranchMovedDuringPush(rootBranch, rootHash, auth) + } if !known { var fetchErr error w.recordFetch(fetchReasonPushFailureProbe) @@ -2321,6 +2324,29 @@ func (w *BranchWorker) remoteMovedDuringPush( return remoteHash != rootHash } +// newBranchMovedDuringPush is the fallback for a push that was creating the write branch and failed +// without an advertisement to read: a dropped connection, or the server refusing an upload whose +// Old was zero because somebody created the branch after our advertisement. Probing the root alone +// would find the parent unmoved and conclude nothing happened, and the work would never replay. So +// it reads both refs from one advertisement, and a write branch that is now there is contention, +// exactly like the typed rejection naming it. It runs only on this exceptional path. +func (w *BranchWorker) newBranchMovedDuringPush( + rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, auth []gitclient.Option, +) bool { + advertisement, err := advertiseRemoteBranchFn(w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), + rootBranch.Short(), auth) + if err != nil { + w.invalidateBase("remote-state probe failed") + return false + } + if !advertisement.branch.IsZero() { + w.Log.Info("The write branch was created by somebody else during our push; replaying onto it", + "branch", w.Branch) + return true + } + return advertisement.parent != rootHash +} + // advertisedRootHash reads the remote's own answer out of a failed push, when the push got one. // // A compare-and-swap rejection is the remote telling us where the branch is: validatePushState diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go index c169f035..36bc17fd 100644 --- a/internal/git/branch_worker_empty_repo_test.go +++ b/internal/git/branch_worker_empty_repo_test.go @@ -264,3 +264,84 @@ func TestBranchWorker_ADetachedRemoteHeadFollowsGoGitsResolution(t *testing.T) { }) } } + +// noOpWrite sends a live write that changes nothing: the delete of an object Git never had. +func noOpWrite(loop *branchWorkerEventLoop) { + deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) + deleted.Operation = "DELETE" + deleted.Object = nil + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) +} + +// §4.7 R5. With spec.parentBranch omitted the parent is the default branch as last discovered. A +// refresh that sees HEAD switch from main to trunk at the same commit follows it with one fetch, so +// the next publication validates trunk and pushes report the same parent the refresh did. Without +// a refresher the worker keeps main until its next discovery, which the push advertisement cannot +// be: it carries no HEAD. +func TestBranchWorker_ADefaultBranchSwitchIsFollowedAtTheNextDiscovery(t *testing.T) { + seed := func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "trunk", "main") + } + + t.Run("a refresh follows it", func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newRealServerNewBranch(t, "default-switch-refresh", seed) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + require.Equal(t, "main", f.observed().ParentBranch) + + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/trunk") + refreshes := fetchCount(t, reader, f.worker, fetchReasonRefresh) + f.loop.handleRefreshRequest(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + assert.Equal( + t, + refreshes+1, + fetchCount(t, reader, f.worker, fetchReasonRefresh), + "one fetch follows the switch", + ) + assert.Equal(t, "trunk", f.observed().ParentBranch) + + noOpWrite(f.loop) + assert.Equal(t, "trunk", f.observed().ParentBranch, "the next no-op push agrees with the refresh") + assert.Equal(t, ObservedByPush, f.observed().By) + }) + + t.Run("without a refresher it waits for the next discovery", func(t *testing.T) { + f := newRealServerNewBranch(t, "default-switch-no-refresh", seed) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/trunk") + noOpWrite(f.loop) + assert.Equal(t, "main", f.observed().ParentBranch, "publication cannot see HEAD") + }) +} + +// §4.7 R6. The write branch is created by somebody else after our advertisement, with the parent +// unmoved. The server refuses our upload (Old was zero) as an untyped error, and probing the parent +// alone would find it unmoved and conclude nothing happened. The fallback reads both refs, sees the +// write branch, and replays onto their commit. +func TestBranchWorker_AWriteBranchCreatedDuringOurUploadIsReplayedOnto(t *testing.T) { + f := newRealServerNewBranch(t, "refused-upload", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + }) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + + var theirs plumbing.Hash + afterPushValidation = func() { + afterPushValidation = nil + theirs = simulateClientCommitOnDisk(t, f.repoDir, "feature", "THEIRS.md", "theirs\n") + } + t.Cleanup(func() { afterPushValidation = nil }) + roots := pushRoots(t, nil) + + liveWrite(f.loop, "cm1") + + require.False(t, theirs.IsZero(), "the branch was created between the advertisement and the upload") + require.Len(t, *roots, 2, "the refused upload, then the replay") + tip := f.ref("feature") + require.NotEqual(t, theirs, tip, "our commit was published") + gitIn(t, f.repoDir, "merge-base", "--is-ancestor", theirs.String(), tip.String()) + assert.Contains(t, gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "feature"), cmFile("cm1")) + assert.Empty(t, f.loop.pendingWrites) +} diff --git a/internal/git/git_atomic_push.go b/internal/git/git_atomic_push.go index 711c5906..a921eb30 100644 --- a/internal/git/git_atomic_push.go +++ b/internal/git/git_atomic_push.go @@ -372,6 +372,12 @@ func performPush( // // The returned PushOutcome is an observation of the remote, made on the connection the push was // opening anyway. An error means nothing was observed; see PushOutcome for what each kind proves. +// afterPushValidation runs between the advertisement check and the upload. Nil in production; a test +// changes the remote here to make the server, not our client, refuse the upload. +// +//nolint:gochecknoglobals // a test seam, like pushAtomicFn +var afterPushValidation func() + func PushAtomic( ctx context.Context, repo *git.Repository, @@ -404,6 +410,9 @@ func PushAtomic( return PushOutcome{}, fmt.Errorf("failed to get current branch: %w", err) } + if afterPushValidation != nil { + afterPushValidation() + } if err := performPush(ctx, session, repo, rootHash, plan.new, plan.old, branch, logger); err != nil { return PushOutcome{}, err } diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 4b22953d..e8a419b8 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -428,6 +428,25 @@ func ledgerOperations() []ledgerOp { f.loop.runParentProbe() }, }, + { + // R6: somebody creates the write branch between our advertisement and our upload, with + // the parent unmoved. The server refuses the upload; one advertisement reads both refs + // and finds the branch; the replay fetches it, rebuilds, and pushes onto it. It costs + // more than row 6, which is refused at the advertisement, before any upload. + name: "16. standby, the write branch created during our upload", + slug: "refused-upload", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) }, + run: func(f *ledgerFixture) { + afterPushValidation = func() { + afterPushValidation = nil + simulateClientCommitOnDisk(f.t, f.repoDir, "feature", "THEIRS.md", "theirs\n") + } + defer func() { afterPushValidation = nil }() + f.publish("first") + }, + }, } } diff --git a/internal/git/refresh.go b/internal/git/refresh.go index c4f71806..4780a9d7 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -224,7 +224,7 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr // the compare-and-swap on the next push. if advertised.IsZero() { w.Log.V(1).Info("Refresh found no such branch on the remote", "branch", w.Branch) - return l.followParent(advertisement.parent) + return l.followParent(advertisement) } // Everything below compares the advertisement against the local checkout, so a missing one is @@ -285,8 +285,15 @@ func (w *BranchWorker) recordAdvertisement(advertisement remoteAdvertisement, pa // followParent brings the checkout of an absent write branch onto its parent's advertised tip. // It costs nothing beyond the advertisement already read when the parent has not moved, and it // writes nothing either way. -func (l *branchWorkerEventLoop) followParent(parent plumbing.Hash) error { +// +// "Moved" includes a parent that is now a different branch: with spec.parentBranch omitted the +// parent is the remote's default branch as last discovered, and a remote whose HEAD switched from +// main to trunk at the same commit has moved as surely as one whose main got a new commit. One +// fetch follows it, so the next publication validates trunk, and pushes and refreshes stop +// reporting different parents. +func (l *branchWorkerEventLoop) followParent(advertisement remoteAdvertisement) error { w := l.w + parent := advertisement.parent // An empty repository has no parent to follow. if parent.IsZero() { return nil @@ -298,7 +305,7 @@ func (l *branchWorkerEventLoop) followParent(parent plumbing.Hash) error { if err != nil { return fmt.Errorf("open repository: %w", err) } - if w.baseTrusted() && parent == localHead(repo) { + if w.baseTrusted() && parent == localHead(repo) && advertisement.parentBranch == w.checkoutParentBranch(repo) { w.Log.V(1).Info("Refresh confirmed the parent branch has not moved", "branch", w.Branch) return nil } @@ -307,6 +314,21 @@ func (l *branchWorkerEventLoop) followParent(parent plumbing.Hash) error { return w.syncWithRemote(w.ctx, fetchReasonRefresh) } +// checkoutParentBranch is the short name of the branch the checkout of an absent write branch is +// rooted on: the parent the local write branch was created from, or the branch HEAD is on. +func (w *BranchWorker) checkoutParentBranch(repo *gogit.Repository) string { + w.repoMu.Lock() + defer w.repoMu.Unlock() + if w.newBranchParent != "" { + return w.newBranchParent.Short() + } + branch, _, err := GetCurrentBranch(repo) + if err != nil { + return "" + } + return branch.Short() +} + // rescanLayoutForTarget re-reads the target's folder, publishes what its shape resolves to, and // publishes whether the folder can be written at all. // diff --git a/internal/git/testdata/git-roundtrip-ledger.golden b/internal/git/testdata/git-roundtrip-ledger.golden index 0319eb6e..1084d44f 100644 --- a/internal/git/testdata/git-roundtrip-ledger.golden +++ b/internal/git/testdata/git-roundtrip-ledger.golden @@ -33,3 +33,4 @@ operation conn fetch-open 13. standby, a live write that changes nothing 1 0 0 1 -/<1K 0 14. standby, the first write after the parent moved 6 2 -/<1K 1 <1K/<1K 2 -/<1K 1 <4K/<1K 15. parent probe while the parent is missing 1 1 -/<1K 0 0 0 +16. standby, the write branch created during our upload 8 3 -/<4K 1 <1K/<1K 2 -/<1K 2 <4K/<1K From 8dd4d06b2e758cd6e63662e14c2815d4fa61729b Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:57:56 +0000 Subject: [PATCH 16/18] docs: fold the hardening pass into the parent-branch contract, and pin a held request through a long recovery The contract page now carries what shipped: the omitted-parent refusal for a repository that is not empty, recovery as a worker obligation, the push admission boundary, go-git's default-branch resolution, and the new ledger rows. The hardening plan is marked done and records where the build departed from it. TestParentRecovery_AHeldCommitRequestIsCommittedAfterALongRecovery runs the real event loop: a CommitRequest riding a write held back by a missing parent stays WaitingForPush through a recovery far longer than the controller's old bound, and resolves Committed on the parent's tip. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 12 +++-- docs/design/gittarget-parent-branch.md | 35 ++++++++++---- docs/design/gittarget-parent-hardening.md | 18 ++++++- internal/git/parent_recovery_test.go | 57 +++++++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 023a4fa2..d76e8ee3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1253,9 +1253,15 @@ parent. Periodic refresh, ten minutes by default, follows the parent from the same advertisement and updates the idle view independently of the publication check. The push records the parent and the commit it started from as its root, and treats a moved parent like a moved branch: fetch, reset, replay, check again. A branch somebody else created in the -meantime is a moved remote too, so the replay builds on their commit instead of overwriting it. -Proved end to end by the e2e spec "New write branch starts from its parent", whose recovery is -attributed to the publication path rather than the refresher. +meantime is a moved remote too, so the replay builds on their commit instead of overwriting it; that +holds when the server refuses our upload too, because the fallback reads both refs from one +advertisement. A cycle planned on an empty repository that has since gained refs is refused at the +advertisement (`RepositoryNotEmptyError`) and replayed, never pushed as an orphan. A parent that is +missing, or a default branch that does not resolve, is a recovery obligation on the worker: it +probes on its own backoff and publishes what it held back once the parent exists (see +`docs/design/gittarget-parent-branch.md`). Proved end to end by the e2e spec "New write branch +starts from its parent"; that publication alone finds a moved parent is proved by worker tests +with no refresher. **When the cycle fetches at all.** Only the first commit of a cycle may fetch, and only when the base is untrusted or the worktree is dirty (see the ground rule above); a healthy publishing target plans diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md index 18fa4ed3..e2bcf30e 100644 --- a/docs/design/gittarget-parent-branch.md +++ b/docs/design/gittarget-parent-branch.md @@ -1,10 +1,10 @@ # GitTarget parent branch: keep a target on standby until a write needs a branch -> **design**: built in #407, with #408's `spec.parentBranch` folded in. This page records the -> contract as built and the reasoning behind it. Open work: -> [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md) (before #407 merges), then +> **design**: built in #407, with #408's `spec.parentBranch` folded in and the hardening pass +> ([`gittarget-parent-hardening.md`](gittarget-parent-hardening.md), done) applied. This page +> records the contract as built and the reasoning behind it. Deferred: > [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and -> [`gittarget-parent-observation.md`](gittarget-parent-observation.md) (deferred). +> [`gittarget-parent-observation.md`](gittarget-parent-observation.md). > Related: [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), > [`../definitions.md`](../definitions.md), and `architecture.md` (Ground rules; "A branch that does > not exist yet"). @@ -56,7 +56,27 @@ check is what protects the first write. | yes | any | work on the write branch; the parent is not followed | | no | yes | compare against the parent; create the write branch from its tip, checked at publication | | no | no, **set** | refuse: `Ready=False`, `Stalled=True`, reason `ParentBranchNotFound`; nothing written, no orphan branch | -| no | no, omitted, empty repository | unborn `HEAD`; the first commit is a root commit on the write branch | +| no | no, omitted, not empty (`HEAD` resolves to no branch, or only tags) | refuse the same way: `ParentBranchNotFound`, with a message to fix `HEAD` or set the field | +| no | no, omitted, empty repository (no refs at all) | unborn `HEAD`; the first commit is a root commit on the write branch | + +- **Recovery from a missing parent** is an obligation on the branch worker, not an observation. It + latches when a cycle fails on the parent and clears only when the work is published: retained + writes when they are pushed, and dropped writes per `(GitTarget, collection)` when a resync of + that scope is pushed or settles as a no-op. The worker probes with one advertisement after 10s, + doubling to 5m, shared by every target on it; before a deadline nothing spends a connection on + the parent, and reconciles never probe. Once found, it retries the retained writes and raises + a per-target snapshot request, which the controller turns into one recheck. Meanwhile the target + reports `RecoveringParentBranch`. +- **A parent change on a live worker** takes effect at a push admission boundary: every operation + reads one `{name, generation}` snapshot, trust is scoped to the generation, and a push is admitted + only when its root was chosen under the current one. A push already admitted is not recalled. +- **Default-branch resolution follows go-git**, which rewrites a hash-only `HEAD` before + `Remote.List` returns (to `master` when it matches, else the alphabetically first branch at that + hash). With the parent omitted, the parent is the default branch as last discovered; a refresh + that sees `HEAD` switch follows it with one fetch, and a push, which cannot see `HEAD`, does not. +- **Cost.** Unchanged remotes keep the ledger's rows 1–12. The new rows: a standby no-op live write + is 1 connection, the first write after the parent moved is 6 (row 6), the parent probe is 1 with + no fetch, and a write branch created during our upload is 8. - **Immutable**, including adding or removing it, like `branch` and `path`. It decides what history a new write branch gets. Delete and recreate the `GitTarget` to change it. @@ -153,6 +173,5 @@ meantime is [`gittarget-parent-observation.md`](gittarget-parent-observation.md) ## Open questions 1. Should each `GitProvider.status.branches` entry show its write branch's parent? -2. With the parent omitted, when is the default branch re-resolved after the remote's `HEAD` - changes? The push session does not advertise `HEAD`. The hardening pass settles this as "the - default branch resolved at the last discovery" (its §4.7, R5). +2. ~~With the parent omitted, when is the default branch re-resolved after the remote's `HEAD` + changes?~~ Settled: at the next discovery (a refresh, or any fetch). See "The contract as built". diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md index 3214c85b..6c1a953c 100644 --- a/docs/design/gittarget-parent-hardening.md +++ b/docs/design/gittarget-parent-hardening.md @@ -1,7 +1,21 @@ # GitTarget parent branch: hardening pass before #407 merges -> **Plan**: open, nothing built. Dated 2026-10-01. It comes from a three-way review of #407 and two -> rounds of second opinions; §7 records the design choices those rounds settled. +> **Plan**: done, 2026-10-01; the shipped contract is in +> [`gittarget-parent-branch.md`](gittarget-parent-branch.md). It came from a three-way review of +> #407 and two rounds of second opinions; §7 records the design choices those rounds settled. +> +> **Where the build departed from the plan.** +> +> - `SetParentBranch` no longer calls `invalidateBase` (§4.1 step 1 kept it as a belt). The +> generation already makes the old checkout untrusted, and the belt cost an existing write +> branch a fetch per change, which §4.1 test 6 forbids. +> - The replay a refused admission triggers is counted as a `recovery` fetch. +> - Canonical git advertises neither a `HEAD` that resolves to nothing nor its symref, so against +> it the dangling name (§4.3 test 1's `master`) never reaches the client: the observation's +> branch is empty and the message says the remote has no default branch. +> - Once the probe has found the parent it stops reading advertisements: later deadlines retry the +> retained writes and re-request the snapshots still owed. A failure caused by the parent arms +> the latch again. > > **The merge boundary.** #407 merges when this pass is done: the existing parent-branch feature > behaves correctly under races and recovery, and stays as cheap as it is today. New policy diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go index 9c32f481..e25575a9 100644 --- a/internal/git/parent_recovery_test.go +++ b/internal/git/parent_recovery_test.go @@ -286,3 +286,60 @@ func TestParentRecovery_AParentChangeMakesTheProbeDue(t *testing.T) { assert.True(t, loop.probeDue()) assert.False(t, f.worker.awaitingParentProbe(), "a new parent is looked for at once") } + +// A CommitRequest that rides a write held back by a missing parent stays held through the whole +// recovery, however long it takes, and resolves Committed when the work lands. This runs the real +// event loop on its own goroutine; the clock stands in for a recovery far longer than the bound the +// controller used to fail requests on (attachTimeout + maxDuration + 120s). +func TestParentRecovery_AHeldCommitRequestIsCommittedAfterALongRecovery(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-held-commit-request", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "release", "main") + }) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + vanished := false + beforePushAdmission = func() { + if !vanished { // the parent disappears right before the first push + vanished = true + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") + } + } + t.Cleanup(func() { beforePushAdmission = nil }) + require.NoError(t, f.worker.Start(context.Background())) + t.Cleanup(f.worker.Stop) + + require.True(t, f.worker.Enqueue(configMapTargetEvent("cm1", "alice", newBranchTarget))) + req := standbyRequest() + f.worker.EnqueueAttach(req) + phase := func() CommitRequestPhase { + return f.worker.LookupCommitRequestPhase(req.Namespace, req.Name, req.UID) + } + require.Eventually(t, func() bool { + open, _ := f.worker.ParentRecovery() + return open && phase() == PhaseWaitingForPush + }, 10*time.Second, 20*time.Millisecond, "the write is held while the parent is missing") + + clock.advance(10 * time.Minute) // far past attachTimeout + maxDuration + 120s + f.worker.EnqueueRefresh(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + require.Never(t, func() bool { + _, resolved := outcome(t, f.worker) + return resolved + }, 300*time.Millisecond, 20*time.Millisecond, "a held request is never failed for taking long") + assert.True(t, phase().Held()) + + release := simulateClientCommitOnDisk(t, f.repoDir, "release", "RELEASE.md", "back\n") + clock.advance(parentProbeMaxBackoff) + f.worker.EnqueueRefresh(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + + require.Eventually(t, func() bool { + _, resolved := outcome(t, f.worker) + return resolved + }, 10*time.Second, 20*time.Millisecond) + res, _ := outcome(t, f.worker) + require.NoError(t, res.Err) + assert.Equal(t, FinalizeCommitted, res.Outcome) + tip := f.ref("feature") + assert.Equal(t, res.Commit, tip.String()) + assert.Equal(t, []plumbing.Hash{release}, f.parentOf(tip)) +} From 93120caf5c62ee9d3156ddce7d3e2c00daa27177 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 12:59:53 +0000 Subject: [PATCH 17/18] docs: describe the safety window as a withdrawal request, and fix the review nits - api-first-publication.md and commit-timing-surface.md: past the safety window the controller asks the worker to withdraw the request and reports the worker's answer, which can be the real outcome when the worker resolved it first. - configuration.md: the status.remote example shows an absent write branch, so the parent stanza it shows is one the contract can produce. - gittarget-parent-branch.md: drop the settled default-branch question from the open questions. - INDEX.md headings and TODO.md wording follow the style guide. Co-Authored-By: Claude Opus 5.5 --- docs/INDEX.md | 4 ++-- docs/TODO.md | 13 +++++++------ docs/api-first-publication.md | 8 +++++--- docs/configuration.md | 2 +- docs/design/commit-timing-surface.md | 5 +++-- docs/design/gittarget-parent-branch.md | 2 -- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/INDEX.md b/docs/INDEX.md index ad70e018..fedb4eb4 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -124,7 +124,7 @@ decided), **design, decided** (decision made, not built), **partly built**, **bu **deferred** (parked, kept as a decision record). The label is the first thing in the page, so you never have to read a proposal to find out it already shipped. -### Open — 27 pages +### Open (27 pages) | Doc | Open question | |---|---| @@ -156,7 +156,7 @@ so you never have to read a proposal to find out it already shipped. | [`sensitive-resource-diagnostics-follow-up.md`](design/sensitive-resource-diagnostics-follow-up.md) | deferred diagnostics | | [`e2e-git-server-choice.md`](design/e2e-git-server-choice.md) | stay on Gitea or move to Forgejo — the `_csrf` pin is fixable in place on both, so the migration is now a preference call, not a fix; also why we adopt no SDK either way | -### Built, and kept here anyway — 6 pages +### Built, and kept here anyway (6 pages) These have shipped. They stay in `design/` under the exception above, because Go source cites them by path as the rationale for what the code does, and `finished/` declares diff --git a/docs/TODO.md b/docs/TODO.md index 0413dc2b..82d64d97 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -14,19 +14,20 @@ finding is fixed in #407: the controller no longer fails a request the worker ho (`docs/design/commit-timing-surface.md`) says a suspended target is a failure. An attached window that changed nothing resolves `AlreadyPresent` the same way. Decide: fail the save, or change the table. -- [ ] **A `CommitEmpty` save is recorded while the render-fidelity gate is closed.** Found by reading; - reproduce first. `recordCommitRequest` never calls `normalWritesAllowed`, so a save whose events the +- [ ] **The worker records a `CommitEmpty` save while the render-fidelity gate is closed.** Found by + reading; reproduce first. `recordCommitRequest` never calls `normalWritesAllowed`, so a save whose events the gate dropped records "no writes were seen". The attached-window path fails the request instead. **Before the next release** (docs only): - [ ] `docs/UPGRADING.md`, the v1alpha3 window entry: - - step 1: GitOps-managed GitTargets need their Flux or Argo source suspended (or the field removed - there) until step 4, or the string `commit.window` is re-applied and breaks LIST; + - step 1: tell readers to suspend the Flux or Argo source of a GitOps-managed GitTarget (or remove + the field there) until step 4; otherwise that tool re-applies the string `commit.window` and + breaks LIST; - the Was/Is table needs a row for the chart value `quickstart.gitTarget.commit.window`, now an object; - - "a removed `closeDelay` is not a hazard" is too strong: it is pruned silently for programmatic - clients (see `duration_fields_admission_test.go`). + - "a removed `closeDelay` is not a hazard" is too strong: the API server prunes it silently for + programmatic clients (see `duration_fields_admission_test.go`). - [ ] Descriptions that still say a save "closes the window now": the CommitRequest godoc and CRD description, `docs/configuration.md` (lines 14, 32, 123), and the GitTarget `Commit` godoc, which also omits the 1m `maxDuration` default. `docs/spec/commitrequest-design.md:126` still says diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index 2d41bfa2..856aeba4 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -311,9 +311,11 @@ Use `Pushed=True` plus `status.commit` when a particular commit must exist in Gi covers successful outcomes with no commit. Status visibility follows the controller's polling cadence and Kubernetes status writes. A request the worker holds (attached to a window, or committed and waiting for the push) has no controller-side timeout: it ends when the worker says so, however -long a down remote takes. A request the worker has not acted on within -`attachTimeout + maxDuration + 120s` of creation is withdrawn from the worker and then fails, so a -failed request is never committed later. Resolved worker outcomes are retained for `15m`. +long a down remote takes. For a request the worker has not acted on within +`attachTimeout + maxDuration + 120s` of creation, the controller asks the worker to withdraw it, and +reports whatever the worker answers: failed when the worker cancelled it, or the real outcome when +the worker resolved it first. A request reported failed is therefore never committed later. The +worker keeps resolved outcomes for `15m`. See the [request contract](spec/commitrequest-design.md). The two-second `attachTimeout` is not a guarantee that an earlier API edit has reached the diff --git a/docs/configuration.md b/docs/configuration.md index e9288831..19a6e614 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -909,7 +909,7 @@ target has written anything: ```yaml status: remote: - commit: 4f2c1ab9e0... # empty = the branch is not on the remote + commit: "" # empty = the branch is not on the remote (yet) lastVerifiedAt: "2026-09-23T10:14:02Z" verifiedBy: Push # Push | Fetch parent: # only while the branch is not on the remote diff --git a/docs/design/commit-timing-surface.md b/docs/design/commit-timing-surface.md index 2f572cff..1a674016 100644 --- a/docs/design/commit-timing-surface.md +++ b/docs/design/commit-timing-surface.md @@ -216,8 +216,9 @@ Every one of these comes from the worker. Sending an attach does not prove an at controller reports a phase only when the worker says the save reached it. The controller's safety timeout applies only to a save the worker has not acted on: -`attachTimeout + maxDuration + 120s` from creation, after which the save is withdrawn from the worker -and fails. Once the worker holds a save (attached, or committed and waiting for the push), only the +`attachTimeout + maxDuration + 120s` from creation. Then the controller asks the worker to withdraw +the save and reports the worker's answer: failed when it cancelled the save, or the real outcome when +it resolved the save first. Once the worker holds a save (attached, or committed and waiting for the push), only the worker ends it, so the controller never reports a failure the worker later contradicts. ## Naming diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md index e2bcf30e..1b0fc2ba 100644 --- a/docs/design/gittarget-parent-branch.md +++ b/docs/design/gittarget-parent-branch.md @@ -173,5 +173,3 @@ meantime is [`gittarget-parent-observation.md`](gittarget-parent-observation.md) ## Open questions 1. Should each `GitProvider.status.branches` entry show its write branch's parent? -2. ~~With the parent omitted, when is the default branch re-resolved after the remote's `HEAD` - changes?~~ Settled: at the next discovery (a refresh, or any fetch). See "The contract as built". From 504ab7c6a7a4cf9d31350cdece2f0524c5782e9c Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 1 Oct 2026 13:22:55 +0000 Subject: [PATCH 18/18] fix(git): keep a recovery obligation for work dropped in a second outage, and hold retained replay to the probe deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of 93120caf reproduced two §4.4 defects: - P1. A recovery snapshot that had committed but not yet been pushed kept its publication credit when the parent vanished again and another write from the same scope was dropped. Publishing the older snapshot then cleared the newer obligation: cm1 landed, cm2 never did, and no new snapshot was requested. A dropped write now removes its scope's credit, so the scope stays open and the worker asks again. - P2. With retained writes waiting for a rebuild, every incoming write went through recoverRetainedWrites, which fetched without checking the probe hold: one connection per write before the deadline. It now returns the hold's error before any I/O, and the scheduled probe still drives the recovery. The reviewer's reproductions are the regression tests (both red before the fix); the probe-budget test gains a retained-writes case. The docs record the review, its resolution, and the follow-up order after #407 merges. Co-Authored-By: Claude Opus 5.5 --- docs/INDEX.md | 2 +- docs/TODO.md | 8 ++ docs/design/gittarget-parent-hardening.md | 55 ++++++++- internal/git/branch_worker.go | 6 + internal/git/parent_recovery.go | 3 + internal/git/parent_recovery_test.go | 139 ++++++++++++++++------ 6 files changed, 173 insertions(+), 40 deletions(-) diff --git a/docs/INDEX.md b/docs/INDEX.md index fedb4eb4..543ea379 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -129,7 +129,7 @@ so you never have to read a proposal to find out it already shipped. | Doc | Open question | |---|---| | [`gittarget-parent-branch.md`](design/gittarget-parent-branch.md) | **built in #407, not merged yet.** `spec.parentBranch`: a target stays on standby until a write needs a branch, then creates its write branch from the parent's tip as checked at publication. The contract as built, the defect it started from, prior art and naming. Open: parent per `GitProvider.status.branches` entry. | -| [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **plan, before #407 merges.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, all without adding a pull to the fast path. | +| [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **done in #407.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, without adding a pull to the fast path; plus the two recovery fixes from the review at `93120caf`. Records the merge boundary and the next separate PRs. | | [`gittarget-parent-empty-repository.md`](design/gittarget-parent-empty-repository.md) | **deferred.** An empty repository's default branch: the server reports it over protocol v2 but go-git discards it (upstream fix first); then a separate policy decision on which branch the first commit may create. Assuming `main` was considered and rejected. | | [`gittarget-parent-observation.md`](design/gittarget-parent-observation.md) | **deferred.** Keep the parent visible after the write branch exists: availability first, ancestry only after measuring how often a shallow checkout can answer it. Informational; authorizes no branch mutation. | | [`gittarget-red-status-plan.md`](design/gittarget-red-status-plan.md) | **design, partly built.** The smaller alternative to Git write preflight: make a `GitTarget` turn red and explain why no commit was made, without moving Git checks into the admission webhook and without adding a second state source. Keeps the existing kstatus trio and domain conditions, ranks the most likely operator-facing failures, and has now shipped the first status-transition slice: refusal messages republish when details change, scoped refusals cannot be cleared by unrelated scope success, `BlockMessage()` carries stable issue kind/path/actor detail, and the unsupported-folder e2e covers Git-side break and recovery. | diff --git a/docs/TODO.md b/docs/TODO.md index 82d64d97..bce948af 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -46,6 +46,14 @@ finding is fixed in #407: the controller no longer fails a request the worker ho ## Current backlog +- [ ] After #407 and the #404 fixes above, take the first + [parent-observation step](design/gittarget-parent-observation.md#implementation-and-validation): + retain parent availability and its timestamp after the write branch exists. Use existing + advertisements, with no extra connections or changes to readiness or publication. Keep ancestry + classification and empty-repository bootstrap policy in separate changes. The + [merge review](design/gittarget-parent-hardening.md#merge-review-at-93120caf) records the two + recovery fixes made before #407 merged. + - [ ] Finish making `GitTarget` turn red quickly and explain why no commit was made. The plan is in [gittarget-red-status-plan.md](design/gittarget-red-status-plan.md). This is the replacement direction for Git write preflight: no admission webhook Git work, no extra state diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md index 6c1a953c..124ef7be 100644 --- a/docs/design/gittarget-parent-hardening.md +++ b/docs/design/gittarget-parent-hardening.md @@ -1,8 +1,8 @@ # GitTarget parent branch: hardening pass before #407 merges -> **Plan**: done, 2026-10-01; the shipped contract is in -> [`gittarget-parent-branch.md`](gittarget-parent-branch.md). It came from a three-way review of -> #407 and two rounds of second opinions; §7 records the design choices those rounds settled. +> **Plan**: done, 2026-10-01, including the two recovery fixes the review of `93120caf` found +> (below). The implemented contract is in +> [`gittarget-parent-branch.md`](gittarget-parent-branch.md). §7 records the settled design choices. > > **Where the build departed from the plan.** > @@ -25,6 +25,55 @@ > [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and > [`gittarget-parent-observation.md`](gittarget-parent-observation.md), deferred follow-ups. +## Merge review at `93120caf` + +Both are fixed, each with the reviewer's reproduction as its regression test (red before the fix). +Both failures were reproduced through the worker event handlers against canonical `git http-backend`. + +1. **P1: a second parent outage loses the obligation for newer dropped writes.** A recovery + snapshot commits locally and enters `awaitingPush`. Before it is pushed, the parent disappears + again and another event from the same scope is dropped. `noteParentUnavailable` keeps the + scope's old `awaitingPush` entry. On recovery, `noteRecoveryPublished` clears that scope after + publishing the older snapshot, and no new snapshot is requested. The reproduction publishes + `cm1`, leaves `cm2` absent, closes recovery, and leaves `SnapshotRequestSeq` at 1. Invalidate + the old publication credit when new work is dropped, or track the obligation's generation. + Acceptance: publishing the old snapshot keeps the newer obligation open, requests a fresh + snapshot, and eventually publishes `cm2` without another cluster edit. + **Fixed:** a dropped write removes its scope's publication credit; pinned by + `TestParentRecovery_ASecondOutageKeepsTheWorkDroppedInIt`. +2. **P2: retained replay bypasses the parent probe backoff.** After a failed push loses its parent, + retained writes have `replayRequired` set. Every incoming live write calls + `recoverRetainedWrites`, which fetches through `invalidateAndRefresh` without checking + `awaitingParentProbe`. Three writes before the deadline produced three remote connections. + The existing probe-budget test starts with dropped work and no retained batch, so it misses + this path. Apply the hold before replay I/O and extend the budget test to retained work. + Acceptance: incoming writes before the deadline spend no remote connection; the scheduled + probe still drives recovery. + **Fixed:** `recoverRetainedWrites` checks the hold before its rebuild; pinned by the + `retained writes` case of `TestParentRecovery_ProbeBudgetIsPerWorker`. + +The fixes preserve the existing API and require no new policy. The required lint, unit, and e2e +gates have to pass on the corrected code; a green run on `93120caf` does not cover these +reproductions. + +At review, CI lint and unit tests passed. The local run passed all three parent-branch e2e specs, +including recovery without another cluster edit; the full local run and CI e2e were still running. +The targeted worker, controller, and router tests passed under `-race`. + +### After merge + +The immediate next fix PR remains the two `CommitEmpty` findings from #404 in +[`../TODO.md`](../TODO.md#follow-ups-from-404-commit-window-surface). +For the next GitTarget feature PR, take only step 1 of the parent-observation plan: retain parent +availability and its timestamp after the write branch exists, using existing advertisements. +Keep readiness and publication behavior unchanged, and prove no extra remote connections. + +Ancestry classification follows only after measuring the pinned go-git shallow-history behavior. +The empty-repository plan starts with the upstream unborn-HEAD fix; choosing which branch the first +commit creates remains a separate compatibility decision. The configuration-freshness record stays +deferred, and the remaining red-status contract work stays on its own branch. Do not reopen the +older API-wave plan as part of any of these changes. + ## 0. Handoff: read this first ### Where the code is diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index c85be642..cd69863a 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -1523,6 +1523,12 @@ func (l *branchWorkerEventLoop) recoverRetainedWrites() error { if !dirty && !needsReplay && !parentChanged { return nil } + // The rebuild fetches, and a parent known to be missing is not fetched again before its probe + // is due: every commit path calls this, so without the check each write arriving in the + // meantime would cost a connection. + if l.w.awaitingParentProbe() { + return errAwaitingParentProbe + } l.w.Log.Info("Rebuilding retained writes onto the remote tip", "pendingWrites", len(l.pendingWrites), diff --git a/internal/git/parent_recovery.go b/internal/git/parent_recovery.go index a22a51e7..b91c5b27 100644 --- a/internal/git/parent_recovery.go +++ b/internal/git/parent_recovery.go @@ -113,6 +113,9 @@ func (l *branchWorkerEventLoop) noteParentUnavailable(err error, scopes ...recov r.scopes = map[recoveryScope]struct{}{} } r.scopes[scope] = struct{}{} + // A snapshot of this scope that is still waiting for its push was taken before this write + // was dropped, so publishing it proves nothing about this write. The scope needs a new one. + delete(r.awaitingPush, scope) } l.armRecoveryTimer() l.publishRecovery() diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go index e25575a9..3c164920 100644 --- a/internal/git/parent_recovery_test.go +++ b/internal/git/parent_recovery_test.go @@ -204,49 +204,116 @@ func TestParentRecovery_DroppedWritesClearPerScopeWhenPublished(t *testing.T) { // Test 4. However many targets share the worker, and however often refreshes tick and live writes // arrive, a parent that stays missing is asked about at most once per deadline of the shared -// schedule, and never fetched. +// schedule, and never fetched. That holds with work retained too: there, every write first wants to +// rebuild the retained writes, which is a fetch the deadline must hold back as well. func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { - reader, err := telemetry.InitTestExporter() - require.NoError(t, err) - f := newRealServerNewBranch(t, "recovery-probe-budget", func(repoDir string) { - simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + for name, retained := range map[string]bool{"dropped writes": false, "retained writes": true} { + t.Run(name, func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + slug := map[bool]string{false: "recovery-probe-budget", true: "recovery-probe-budget-retained"}[retained] + f := newRealServerNewBranch(t, slug, func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "release", "main") + }) + clock := useTestClock(f.worker) + createPlainGitTarget(t, f.worker, "second", "second") + createPlainGitTarget(t, f.worker, "third", "third") + targets := []string{newBranchTarget, "second", "third"} + + if retained { + f.worker.SetParentBranch("release") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + f.loop.lastPushAt = time.Now() + liveWrite(f.loop, "first") + require.Len(t, f.loop.pendingWrites, 1) + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") + f.loop.pushPending() // the push that finds the parent gone + require.True(t, f.worker.replayRequired(), "the retained writes wait for a rebuild") + } else { + f.worker.SetParentBranch("missing") + liveWrite(f.loop, "first") // the one fetch that finds the parent missing + } + open, _ := f.worker.ParentRecovery() + require.True(t, open) + + fetches := totalFetches(t, reader, f.worker) + before := f.mark() + probes := 0 + const window = 10 * time.Minute + for elapsed := time.Duration(0); elapsed < window; elapsed += 2 * time.Second { + clock.advance(2 * time.Second) + if f.loop.probeDue() { + probes++ + f.loop.runParentProbe() // the timer + } + for _, target := range targets { + f.loop.handleRefreshRequest(&RefreshRequest{ + Target: itypes.NewResourceReference(target, "default"), MaxAge: time.Nanosecond, + }) + event := configMapTargetEvent("cm-"+target, "alice", target) + f.loop.handleQueueItem( + WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}, + ) + } + } + + // 10s, then doubling to the 5m cap: deadlines at 10s, 30s, 70s, 150s and 310s. + assert.Equal(t, 5, probes) + assert.LessOrEqual(t, f.mark().since(before).connections(), int64(probes), + "one advertisement per deadline, whatever else happens") + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "and no fetch at all") + }) + } +} + +// A second outage while a recovery snapshot waits for its push. The write dropped during it was +// made after that snapshot was taken, so publishing the snapshot must not settle its scope: the +// worker asks for another one, and the obligation stays open until that is published. +func TestParentRecovery_ASecondOutageKeepsTheWorkDroppedInIt(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-second-outage", func(dir string) { + simulateClientCommitOnDisk(t, dir, "main", "README.md", "main\n") }) clock := useTestClock(f.worker) - createPlainGitTarget(t, f.worker, "second", "second") - createPlainGitTarget(t, f.worker, "third", "third") f.worker.SetParentBranch("release") - targets := []string{newBranchTarget, "second", "third"} + collection := itypes.CollectionKey{Resource: "configmaps", Namespace: "default"} + write := func(name string) { + event := configMapTargetEvent(name, "alice", newBranchTarget) + event.SourceCollection = collection + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + } - liveWrite(f.loop, "first") // the one fetch that finds the parent missing - open, _ := f.worker.ParentRecovery() - require.True(t, open) - - fetches := totalFetches(t, reader, f.worker) - before := f.mark() - probes := 0 - const window = 10 * time.Minute - for elapsed := time.Duration(0); elapsed < window; elapsed += 2 * time.Second { - clock.advance(2 * time.Second) - if f.loop.probeDue() { - probes++ - f.loop.runParentProbe() // the timer - } - for _, target := range targets { - f.loop.handleRefreshRequest(&RefreshRequest{ - Target: itypes.NewResourceReference(target, "default"), MaxAge: time.Nanosecond, - }) - event := configMapTargetEvent("cm-"+target, "alice", target) - f.loop.handleQueueItem( - WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}, - ) - } + write("cm1") // dropped: release does not exist + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(parentProbeMaxBackoff) + f.loop.runParentProbe() + seq := f.worker.SnapshotRequestSeq(refreshTarget()) + require.Equal(t, uint64(1), seq) + + f.loop.lastPushAt = time.Now() // the snapshot commits, and its push waits for the cooldown + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCM("cm1", "blue")}, + ResourceVersion: "1", GitTargetName: newBranchTarget, GitTargetNamespace: "default", + Scope: &ResyncScope{Collection: collection}, + Result: make(chan ResyncResult, 1), } + f.loop.applyResync(req) + require.NoError(t, (<-req.Result).Err) + require.Len(t, f.loop.recovery.awaitingPush, 1) - // 10s, then doubling to the 5m cap: deadlines at 10s, 30s, 70s, 150s and 310s. - assert.Equal(t, 5, probes) - assert.LessOrEqual(t, f.mark().since(before).connections(), int64(probes), - "one advertisement per deadline, whatever else happens") - assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "and no fetch at all") + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") // the second outage + f.loop.pushPending() + write("cm2") // dropped during it + require.Len(t, f.loop.pendingWrites, 1, "cm2 was dropped; only the snapshot is retained") + + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(parentProbeMaxBackoff) + f.loop.runParentProbe() + + require.False(t, f.ref("feature").IsZero(), "the snapshot was published") + open, _ := f.worker.ParentRecovery() + assert.True(t, open, "cm2 still needs a snapshot") + assert.Greater(t, f.worker.SnapshotRequestSeq(refreshTarget()), seq, "and the worker asks for one") } // Test 5. An unresolved default branch, once repaired, recovers through the same path: the probe