diff --git a/.coverage-baseline b/.coverage-baseline index 7a60a910..feb00d3b 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -79.0 +79.4 diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index bfa6c811..e54b6ac4 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -28,6 +28,7 @@ import ( // +kubebuilder:validation:XValidation:rule="!has(oldSelf.gitProviderRef) || self.gitProviderRef == oldSelf.gitProviderRef",message="spec.gitProviderRef is immutable; delete and recreate the GitTarget to change its destination" // +kubebuilder:validation:XValidation:rule="self.branch == oldSelf.branch",message="spec.branch is immutable; delete and recreate the GitTarget to change its destination" // +kubebuilder:validation:XValidation:rule="self.path == oldSelf.path",message="spec.path is immutable; delete and recreate the GitTarget to change its destination" +// +kubebuilder:validation:XValidation:rule="has(self.parentBranch) == has(oldSelf.parentBranch) && (!has(self.parentBranch) || self.parentBranch == oldSelf.parentBranch)",message="spec.parentBranch is immutable; delete and recreate the GitTarget to change it" // // spec.clusterProviderRef names the SOURCE cluster a GitTarget mirrors FROM (see its field doc). It // is immutable — a folder's source cluster is part of what the folder means, like @@ -49,6 +50,25 @@ type GitTargetSpec struct { // +kubebuilder:validation:MinLength=1 Branch string `json:"branch"` + // Why "parent" and not base/source/revision: docs/definitions.md reserves those words, and the + // field is flat because it is one fact. It is immutable like branch and path: it decides what + // history a new write branch gets, and a parent changed under a worker holding writes is the + // one case that needed its own replay. Delete and recreate is cheap, because the destination is + // unchanged. It can never hold a commit or tag: a pinned starting point would be a sibling field. + + // ParentBranch is the branch the write branch (spec.branch) is created from, and the branch the + // target compares against while the write branch does not exist on the remote. Omitted: the + // branch the remote's HEAD points at (its default branch), or a branch with no history when + // the repository is empty. When set, it must exist on the remote for the write branch to be + // created; otherwise the GitTarget reports ParentBranchNotFound and writes nothing. Once the + // write branch exists, it is used as it is and the parent is not followed. It is only read, so + // it does not need to be in the GitProvider's allowedBranches. All GitTargets on one GitProvider + // and branch must name the same parent branch (omitted counts as its own value). + // Immutable, including adding or removing it: delete and recreate the GitTarget to change it. + // +optional + // +kubebuilder:validation:MinLength=1 + ParentBranch string `json:"parentBranch,omitempty"` + // Path within the repository to write resources to, relative to the repository // root. Required and must be non-empty — there is no default, so a GitTarget can // never silently write to the repository root. To deliberately target the @@ -457,6 +477,48 @@ type GitTargetRemoteStatus struct { // +optional // +kubebuilder:validation:Enum=Push;Fetch VerifiedBy string `json:"verifiedBy,omitempty"` + + // Parent is what the write branch would be created from, observed with it: present only + // while the remote does not carry the write branch. Once the branch exists its parent is not + // followed, so this is omitted. + // +optional + Parent *GitTargetParentStatus `json:"parent,omitempty"` +} + +// GitTargetParentState is what a write branch that does not exist yet would be created from. +// +kubebuilder:validation:Enum=Found;Missing;Unborn +type GitTargetParentState string + +const ( + // GitTargetParentFound means the parent branch exists; the first commit builds on its tip. + GitTargetParentFound GitTargetParentState = "Found" + // GitTargetParentMissing means the parent is not on the remote either, so the write branch + // cannot be created; the GitTarget reports ParentBranchNotFound. The parent is spec.parentBranch, + // or, when that is omitted, the default branch of a repository that is not empty but whose HEAD + // names no branch it carries. + GitTargetParentMissing GitTargetParentState = "Missing" + // GitTargetParentUnborn means the repository is empty, with no refs at all (tags included), and + // no parent is configured, so the first commit starts the write branch with no history. + GitTargetParentUnborn GitTargetParentState = "Unborn" +) + +// GitTargetParentStatus is the parent branch of a write branch that does not exist yet. +type GitTargetParentStatus struct { + // State is what the write branch would be created from: Found (the parent's tip), Missing + // (the parent is not on the remote; nothing is written) or Unborn (an empty repository, with no + // refs at all, and no parent configured; the first commit has no history). + // +required + State GitTargetParentState `json:"state"` + + // Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when + // that is omitted. Empty when State is Unborn, and when the remote names no default branch. + // +optional + Branch string `json:"branch,omitempty"` + + // Commit is the commit hash the parent branch is at on the remote: the commit the write + // branch's first commit would build on. Set only when State is Found. + // +optional + Commit string `json:"commit,omitempty"` } // Two rules for anything added here. A field earns its place only if a reader cannot get it from diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index bc96bda8..890184c4 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -831,6 +831,21 @@ func (in *GitTargetList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitTargetParentStatus) DeepCopyInto(out *GitTargetParentStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetParentStatus. +func (in *GitTargetParentStatus) DeepCopy() *GitTargetParentStatus { + if in == nil { + return nil + } + out := new(GitTargetParentStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitTargetPlacementSpec) DeepCopyInto(out *GitTargetPlacementSpec) { *out = *in @@ -884,6 +899,11 @@ func (in *GitTargetRemoteStatus) DeepCopyInto(out *GitTargetRemoteStatus) { in, out := &in.LastVerifiedAt, &out.LastVerifiedAt *out = (*in).DeepCopy() } + if in.Parent != nil { + in, out := &in.Parent, &out.Parent + *out = new(GitTargetParentStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetRemoteStatus. diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index dfccbdf4..bafbcae5 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -348,6 +348,19 @@ spec: - Ignore - PushEmptyCommit type: string + parentBranch: + description: |- + ParentBranch is the branch the write branch (spec.branch) is created from, and the branch the + target compares against while the write branch does not exist on the remote. Omitted: the + branch the remote's HEAD points at (its default branch), or a branch with no history when + the repository is empty. When set, it must exist on the remote for the write branch to be + created; otherwise the GitTarget reports ParentBranchNotFound and writes nothing. Once the + write branch exists, it is used as it is and the parent is not followed. It is only read, so + it does not need to be in the GitProvider's allowedBranches. All GitTargets on one GitProvider + and branch must name the same parent branch (omitted counts as its own value). + Immutable, including adding or removing it: delete and recreate the GitTarget to change it. + minLength: 1 + type: string path: description: |- Path within the repository to write resources to, relative to the repository @@ -521,6 +534,10 @@ spec: - message: spec.path is immutable; delete and recreate the GitTarget to change its destination rule: self.path == oldSelf.path + - message: spec.parentBranch is immutable; delete and recreate the GitTarget + to change it + rule: has(self.parentBranch) == has(oldSelf.parentBranch) && (!has(self.parentBranch) + || self.parentBranch == oldSelf.parentBranch) - message: spec.clusterProviderRef is immutable; delete and recreate the GitTarget to change the cluster it mirrors rule: self.clusterProviderRef == oldSelf.clusterProviderRef @@ -682,6 +699,35 @@ spec: an old value there means the layout has not changed rather than that nothing has looked. format: date-time type: string + parent: + description: |- + Parent is what the write branch would be created from, observed with it: present only + while the remote does not carry the write branch. Once the branch exists its parent is not + followed, so this is omitted. + properties: + branch: + description: |- + Branch is the parent branch: spec.parentBranch, or the branch the remote's HEAD names when + that is omitted. Empty when State is Unborn, and when the remote names no default branch. + type: string + commit: + description: |- + Commit is the commit hash the parent branch is at on the remote: the commit the write + branch's first commit would build on. Set only when State is Found. + type: string + state: + description: |- + State is what the write branch would be created from: Found (the parent's tip), Missing + (the parent is not on the remote; nothing is written) or Unborn (an empty repository, with no + refs at all, and no parent configured; the first commit has no history). + enum: + - Found + - Missing + - Unborn + type: string + required: + - state + type: object verifiedBy: description: |- VerifiedBy is what proved it: `Push` means the server accepted a ref update of ours, so diff --git a/docs/INDEX.md b/docs/INDEX.md index e39d606a..543ea379 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -124,10 +124,14 @@ decided), **design, decided** (decision made, not built), **partly built**, **bu **deferred** (parked, kept as a decision record). The label is the first thing in the page, so you never have to read a proposal to find out it already shipped. -### Open — 23 pages +### Open (27 pages) | Doc | Open question | |---|---| +| [`gittarget-parent-branch.md`](design/gittarget-parent-branch.md) | **built in #407, not merged yet.** `spec.parentBranch`: a target stays on standby until a write needs a branch, then creates its write branch from the parent's tip as checked at publication. The contract as built, the defect it started from, prior art and naming. Open: parent per `GitProvider.status.branches` entry. | +| [`gittarget-parent-hardening.md`](design/gittarget-parent-hardening.md) | **done in #407.** Parent changes on a live worker, an empty repository gaining a ref, an unresolved default branch, recovery that cannot strand work, and the refused upload, without adding a pull to the fast path; plus the two recovery fixes from the review at `93120caf`. Records the merge boundary and the next separate PRs. | +| [`gittarget-parent-empty-repository.md`](design/gittarget-parent-empty-repository.md) | **deferred.** An empty repository's default branch: the server reports it over protocol v2 but go-git discards it (upstream fix first); then a separate policy decision on which branch the first commit may create. Assuming `main` was considered and rejected. | +| [`gittarget-parent-observation.md`](design/gittarget-parent-observation.md) | **deferred.** Keep the parent visible after the write branch exists: availability first, ancestry only after measuring how often a shallow checkout can answer it. Informational; authorizes no branch mutation. | | [`gittarget-red-status-plan.md`](design/gittarget-red-status-plan.md) | **design, partly built.** The smaller alternative to Git write preflight: make a `GitTarget` turn red and explain why no commit was made, without moving Git checks into the admission webhook and without adding a second state source. Keeps the existing kstatus trio and domain conditions, ranks the most likely operator-facing failures, and has now shipped the first status-transition slice: refusal messages republish when details change, scoped refusals cannot be cleared by unrelated scope success, `BlockMessage()` carries stable issue kind/path/actor detail, and the unsupported-folder e2e covers Git-side break and recovery. | | [`open-asks-priority.md`](design/open-asks-priority.md) | **the work queue.** Merges three overlapping backlogs — the downstream consumer asks, the API-surface block left unbuilt by the status and configuration-model review, and the config-surface proposal (B1–B6) — into one ordered queue under four stated tests, and says where we deliberately do **not** do what was asked. The standing caveat narrowed once the layout model reversed: a Tier 2 entry belongs to postponed [#294](https://github.com/ConfigButler/gitops-reverser/issues/294) only if it breaks a `GitTarget` field, and everything else is independently schedulable. Makes one design call against what was asked: **delete Option C sibling inference** rather than ship an off-switch for it, because it let a human's edit to the repository change operator behavior with nothing in status recording the move. That deletion has shipped, and "what the deletion taught" records what building it found. **F9 is Tier 1**: the only item whose answer is unknown rather than whose work is unscheduled, and it gates planning the enum work | | [`proving-the-metric-surface.md`](design/proving-the-metric-surface.md) | **design, partly built.** What identity an operator needs to locate a failure, and what an e2e check has to do to be worth trusting. Written after a metric value that was declared, switched on, surfaced as a condition reason and counted turned out to have no producer at all, and the check added to catch that class then skipped whenever the counter read zero — so deleting every recording call left the suite green. Two rules follow: an assertion must not take its trigger from the metric under test, and no enum value may be reachable only through an injected fake. Carries a cardinality policy that separates bounded configuration identity from per-request identity, a table of which instrument families should own `source_cluster` directly rather than by join, and the PromQL reasoning for `increase()` versus `max_over_time` (neither applies mechanically: one cannot see an event before the first sample, the other cannot reconstruct totals across a reset). Steps 1–4 are built; step 5, diagnostics export, the `e2e_run` scrape label and a pinned-seed regression are not | @@ -152,7 +156,7 @@ so you never have to read a proposal to find out it already shipped. | [`sensitive-resource-diagnostics-follow-up.md`](design/sensitive-resource-diagnostics-follow-up.md) | deferred diagnostics | | [`e2e-git-server-choice.md`](design/e2e-git-server-choice.md) | stay on Gitea or move to Forgejo — the `_csrf` pin is fixable in place on both, so the migration is now a preference call, not a fix; also why we adopt no SDK either way | -### Built, and kept here anyway — 6 pages +### Built, and kept here anyway (6 pages) These have shipped. They stay in `design/` under the exception above, because Go source cites them by path as the rationale for what the code does, and `finished/` declares diff --git a/docs/TODO.md b/docs/TODO.md index 2af5e7bf..bce948af 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -2,8 +2,58 @@ This file is meant to track the smaller current backlog, not historical notes. +## Follow-ups from #404 (commit window surface) + +PR #404 is merged; these need fix PRs against main, in this order. (Its CommitRequest timeout +finding is fixed in #407: the controller no longer fails a request the worker holds.) + +**Next, in one fix PR:** + +- [ ] **A suspended target makes a `CommitEmpty` save report Ready=True with no commit.** + `recordCommitRequest` returns `false, nil` on `errTargetSuspended`; the design table + (`docs/design/commit-timing-surface.md`) says a suspended target is a failure. An attached window + that changed nothing resolves `AlreadyPresent` the same way. Decide: fail the save, or change the + table. +- [ ] **The worker records a `CommitEmpty` save while the render-fidelity gate is closed.** Found by + reading; reproduce first. `recordCommitRequest` never calls `normalWritesAllowed`, so a save whose events the + gate dropped records "no writes were seen". The attached-window path fails the request instead. + +**Before the next release** (docs only): + +- [ ] `docs/UPGRADING.md`, the v1alpha3 window entry: + - step 1: tell readers to suspend the Flux or Argo source of a GitOps-managed GitTarget (or remove + the field there) until step 4; otherwise that tool re-applies the string `commit.window` and + breaks LIST; + - the Was/Is table needs a row for the chart value `quickstart.gitTarget.commit.window`, now an + object; + - "a removed `closeDelay` is not a hazard" is too strong: the API server prunes it silently for + programmatic clients (see `duration_fields_admission_test.go`). +- [ ] Descriptions that still say a save "closes the window now": the CommitRequest godoc and CRD + description, `docs/configuration.md` (lines 14, 32, 123), and the GitTarget `Commit` godoc, which + also omits the 1m `maxDuration` default. `docs/spec/commitrequest-design.md:126` still says + `NoOpenWindow` and "close deadline". + +**Later, non-blocking:** + +- [ ] `expireWaitingCommitRequests` ranges over a map, so empty commits for saves that time out in + the same pass land in random order; sort by `seq`. +- [ ] `buildRequestRecordWrite` keeps only the username; carry the full `UserInfo` so the record has + the same author identity as a window commit. +- [ ] A record commit skips `requestTemplate`; fix the code or the doc comment that says it is phrased + like every other commit. +- [ ] Nits: a stale doc comment stacked on `commitWindowFor`; kstatus test names that still say + "close-delay". + ## Current backlog +- [ ] After #407 and the #404 fixes above, take the first + [parent-observation step](design/gittarget-parent-observation.md#implementation-and-validation): + retain parent availability and its timestamp after the write branch exists. Use existing + advertisements, with no extra connections or changes to readiness or publication. Keep ancestry + classification and empty-repository bootstrap policy in separate changes. The + [merge review](design/gittarget-parent-hardening.md#merge-review-at-93120caf) records the two + recovery fixes made before #407 merged. + - [ ] Finish making `GitTarget` turn red quickly and explain why no commit was made. The plan is in [gittarget-red-status-plan.md](design/gittarget-red-status-plan.md). This is the replacement direction for Git write preflight: no admission webhook Git work, no extra state diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index 877ab549..856aeba4 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -309,8 +309,13 @@ leaving it to time out, so a shutdown mid-publication is reported as a failure a Use `Pushed=True` plus `status.commit` when a particular commit must exist in Git. `Ready=True` also covers successful outcomes with no commit. Status visibility follows the controller's polling -cadence and Kubernetes status writes. The controller's safety timeout is `420s` from object -creation; resolved worker outcomes are retained for `15m` with cleanup on subsequent resolutions. +cadence and Kubernetes status writes. A request the worker holds (attached to a window, or committed +and waiting for the push) has no controller-side timeout: it ends when the worker says so, however +long a down remote takes. For a request the worker has not acted on within +`attachTimeout + maxDuration + 120s` of creation, the controller asks the worker to withdraw it, and +reports whatever the worker answers: failed when the worker cancelled it, or the real outcome when +the worker resolved it first. A request reported failed is therefore never committed later. The +worker keeps resolved outcomes for `15m`. See the [request contract](spec/commitrequest-design.md). The two-second `attachTimeout` is not a guarantee that an earlier API edit has reached the diff --git a/docs/architecture.md b/docs/architecture.md index 6f1b8c4f..d76e8ee3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,7 +71,11 @@ that a write failed part-way and is cleared **only by a reset** (a third, `repla retained writes whose local commits a reset discarded before the replay could rebuild them). Collapsing them would let one kind of doubt clear another. Either flag makes the next cycle fetch. The failure direction is safe by construction: a stale "untrusted" costs one fetch, while a stale "trusted" -is caught by the compare-and-swap on the next push. **The corollary is the rule to keep in your head +is caught by the compare-and-swap on the next push. For a new write branch, the worker checks the +parent's advertised tip before sending the push. A moved parent triggers fetch, reset, and replay. +The server's compare-and-swap protects creation of the write branch with `Old = zero`; it does not +lock the parent. The parent can still move after the advertisement, so what is guaranteed is the +parent's tip as that advertisement showed it. **The corollary is the rule to keep in your head when adding code here:** anything that resolves, commits, or concludes without reaching a push advertisement must either fetch, or refuse to conclude. See [inbound push notification](design/push-notification-and-reconcile-trigger.md) §3. @@ -355,6 +359,10 @@ Key fields: - `spec.gitProviderRef`: a `GitProvider` in the same namespace, by name. - `spec.clusterProviderRef`: a cluster-scoped source `ClusterProvider`; it defaults to `{name: default}`. - `spec.branch`: immutable branch, validated against `GitProvider.spec.allowedBranches`. +- `spec.parentBranch`: optional, immutable branch the write branch is created from and compared + against while it does not exist. Omitted means the remote's default branch. An explicit parent + must exist when creating the write branch. It does not need to be in `allowedBranches` because + it is only read. - `spec.path`: immutable, required path under the repo (`MinLength=1`; `.` means repo root and must be chosen explicitly). - `spec.encryption`: optional SOPS/age encryption settings for sensitive resources. @@ -367,8 +375,10 @@ may batch and phrase their commits differently. A branch worker serves a `(provi resolves the window per open window, since a window is bound to exactly one target. `gitProviderRef`, `clusterProviderRef`, `branch`, and `path` are immutable so a target cannot silently -orphan an old materialization or change its source cluster. The controller also rejects path overlaps -between GitTargets sharing a provider and branch. +orphan an old materialization or change its source cluster. `parentBranch` is immutable too, +including adding or removing it, because it decides what history a new write branch gets. The +controller also rejects path overlaps between GitTargets sharing a provider and branch, and rejects +two GitTargets on one branch that name different parent branches. Status has a kstatus-compatible summary layer plus domain conditions: @@ -561,14 +571,15 @@ instead. ### Remote moved while we were writing -If someone else pushes to the same remote branch before GitOps Reverser's push lands, the operator does -not treat its local clone as authoritative. It keeps the finalized pending writes, fetches the new remote +If someone else pushes to the same remote branch before GitOps Reverser's push lands, or, for a branch +not created yet, if its parent branch moved, the operator does not treat its local clone as +authoritative. It keeps the finalized pending writes, fetches the new remote tip, resets the local clone, replays those writes, and pushes again. ```mermaid flowchart TD LOCAL[Window finalized
pending write retained] --> CHECK[PushAtomic checks remote ref] - CHECK --> MATCH{Remote still at expected SHA?} + CHECK --> MATCH{Write branch, or while it is absent its parent,
still at the expected commit?} MATCH -->|yes| ACCEPT[Push accepted] ACCEPT --> CLEAR[Clear retained pending writes] @@ -1235,6 +1246,23 @@ A branch whose remote has been **deleted** takes the same path. The advertisemen branch at all, which is reported as a moved remote with a zero hash, so the replay re-roots on the remote's default branch and the retry re-creates the branch with the retained writes on top. +**A branch that does not exist yet** is created from its parent (`spec.parentBranch`, or the +remote's default branch when that is omitted) only when a write has something to commit, so an +idle or in-sync target never creates it. Until then the worker compares the folder against the +parent. Periodic refresh, ten minutes by default, follows the +parent from the same advertisement and updates the idle view independently of the publication +check. The push records the parent and the commit it started from as its root, and treats a moved +parent like a moved branch: fetch, reset, replay, check again. A branch somebody else created in the +meantime is a moved remote too, so the replay builds on their commit instead of overwriting it; that +holds when the server refuses our upload too, because the fallback reads both refs from one +advertisement. A cycle planned on an empty repository that has since gained refs is refused at the +advertisement (`RepositoryNotEmptyError`) and replayed, never pushed as an orphan. A parent that is +missing, or a default branch that does not resolve, is a recovery obligation on the worker: it +probes on its own backoff and publishes what it held back once the parent exists (see +`docs/design/gittarget-parent-branch.md`). Proved end to end by the e2e spec "New write branch +starts from its parent"; that publication alone finds a moved parent is proved by worker tests +with no refresher. + **When the cycle fetches at all.** Only the first commit of a cycle may fetch, and only when the base is untrusted or the worktree is dirty (see the ground rule above); a healthy publishing target plans straight onto its own last push. Once writes are retained the guard flips off entirely, because a diff --git a/docs/configuration.md b/docs/configuration.md index 31a71c4c..19a6e614 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -79,6 +79,7 @@ The destination fields are immutable: to move a target, delete it and create a n |---|---|---| | `gitProviderRef` | **required** | The `GitProvider` backing this target, in the same namespace | | `branch` | **required** | Branch to write. Must be in the provider's `allowedBranches` | +| `parentBranch` | the remote's default branch | Branch a `branch` the remote does not have yet is created from. Immutable. Only read, so it need not be allowed. See [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) | | `path` | **required** | Folder within the repository. `.` targets the root; empty is rejected | | `clusterProviderRef` | `{"name":"default"}` | Source cluster to mirror from. The default names a `ClusterProvider` called `default` | | `commit.window.idleTimeout` | `5s` | Close a commit window after this much silence. `0s` commits every write on its own. See [the commit window](#the-commit-window-speccommitwindow) | @@ -551,6 +552,8 @@ The important fields are: - `spec.clusterProviderRef`: which `ClusterProvider` supplies resources; omit it to reference the user-created `default` provider - `spec.branch`: which allowed branch to write to +- `spec.parentBranch`: which branch `spec.branch` is created from while it does not exist yet; omit it + for the remote's default branch - `spec.path`: required relative path inside the repository; use `.` only when you deliberately want the repository root - `spec.encryption`: how `Secret` resources should be encrypted before commit @@ -747,6 +750,53 @@ two are always separate objects, and deleting the preview cannot disturb the rea For inspecting a repository without a cluster at all, use the `manifest-analyzer` CLI. +### Starting the write branch from another branch (`spec.parentBranch`) + +A target whose `spec.branch` does not exist on the remote waits on standby: it compares the cluster +against the **parent branch** and creates nothing while they agree. The first write that has +something to commit creates the branch from the parent's current tip, checked when it pushes, so a +target that waited a week does not publish onto a week-old checkout. A write that finds nothing to +change creates no branch at all. + +The parent is the remote's default branch unless you name one: + +```yaml +spec: + gitProviderRef: {name: homelab} + branch: reverser/prod-edits # the write branch: created on the first commit + parentBranch: env/prod # what it is created from + path: apps/checkout +``` + +- An explicit parent must exist when the write branch is created. Otherwise the target reports + `Ready=False` with reason `ParentBranchNotFound` and writes nothing. +- It recovers on its own once the parent is pushed, with no edit to the cluster and with periodic + refresh on or off. The branch worker looks for the parent with one ref advertisement after 10s, + then after twice as long each time, up to every 5 minutes, shared by every target on the branch. + When the parent is back it publishes the writes it held and asks for a fresh snapshot of the ones + it had to drop; until those are published the target reports `Ready=False` with reason + `RecoveringParentBranch`. +- An omitted parent is the remote's default branch, as last discovered. In an empty repository (no + refs at all, tags included) the first commit starts a branch with no history. A repository that + is not empty but whose `HEAD` names no branch it carries also reports `ParentBranchNotFound`, and + writes nothing until `HEAD` is fixed or `parentBranch` is set; that never starts an orphan. +- A detached `HEAD`, or a server that does not say what `HEAD` points at, resolves the way go-git + resolves it: to `master` when it is at `HEAD`'s commit, otherwise to the alphabetically first + branch that is. Set `parentBranch` when you need certainty rather than that guess. +- When the remote's `HEAD` switches to another branch, the next discovery (a refresh, or any fetch) + follows it. A push cannot see `HEAD`, so with periodic refresh off a write branch can still start + from the previous default branch until then. +- Once the write branch exists, it is used as it is: new commits on the parent do not reach it. + Delete the write branch after its changes are merged, and the next edit starts a fresh one. +- `parentBranch` is immutable, including adding or removing it: delete and recreate the target to + change it, which loses nothing because the destination is unchanged. It is only read, so it does + not need to be in `allowedBranches`. +- While the write branch is absent, `status.remote.parent` shows the parent, whether it was + `Found`, `Missing` or `Unborn`, and the commit the first commit would build on. With + `parentBranch` omitted, that is how you see which default branch the remote resolved. +- All `GitTarget` objects on one `GitProvider` and branch must name the same parent branch; omitted + counts as its own value. The later-created one reports `TargetConflict`. + `spec.suspend` is **not** this. It is the next section. ### Stopping a target from writing (`spec.suspend`) @@ -859,9 +909,13 @@ target has written anything: ```yaml status: remote: - commit: 4f2c1ab9e0... # empty = the branch is not on the remote + commit: "" # empty = the branch is not on the remote (yet) lastVerifiedAt: "2026-09-23T10:14:02Z" verifiedBy: Push # Push | Fetch + parent: # only while the branch is not on the remote + state: Found # Found | Missing | Unborn + branch: main # spec.parentBranch, or the remote's default branch + commit: 9a1e07c3d2... # set only when Found ``` - `commit` is where the branch is. Empty means the branch is not on the remote at all, which is @@ -873,6 +927,12 @@ status: commit is Reverser's own work. `Fetch` means it went and looked, and this is what was there. A `Fetch` beside a commit none of your publications produced is how a **foreign push** to the branch is read off `kubectl`. +- `parent` appears only while the branch is not on the remote: it says what the branch would be + created from, observed together with the branch's absence. `state` is `Found` (the first commit + builds on `commit`, the parent's tip), `Missing` (the parent is not on the remote either; the + target reports `ParentBranchNotFound` and writes nothing) or `Unborn` (an empty repository, with + no refs at all, and no parent configured; the first commit starts the branch with no history). See + [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch). `kubectl get gittarget -o wide` shows `lastVerifiedAt` as an age, in the `Verified` column. diff --git a/docs/definitions.md b/docs/definitions.md index 0196be68..789611c5 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -124,6 +124,17 @@ Observation failure supplies no evidence of absence. The code calls this write-s branch)` tuple. Every write to that branch goes through it. Two words, lowercase in prose, `BranchWorker` as the Go type. +**Write branch.** The branch a `GitTarget` writes to: `spec.branch`. It must be in the +`GitProvider`'s `allowedBranches`. + +**Parent branch.** The branch a write branch is created from, and compared against while the write +branch does not exist on the remote: `spec.parentBranch`, or the remote's default branch when that +is omitted. Say "parent branch"; a bare "parent" is fine where the subject is already a branch, such +as the `status.remote.parent` stanza or a sentence that has just named the parent branch. Use the +full term wherever a parent *resource* could be meant. Not "base", which names a kustomize read-only +base and base trust, and not "source", which names the source cluster. Once the write branch exists, +the parent branch is not followed. + **Commit window.** The interval a branch worker batches one author's writes to one target over before it commits. Only a write opens one; nothing else does. It closes on its `idleTimeout` of silence or its `maxDuration` after opening, whichever comes first. Windows do not merge across authors, which @@ -138,8 +149,8 @@ one. One word for the act, in the API (`window.attach`, `attachTimeout`), the co `status.remote`. **Commit.** A bare Git commit hash, wherever one appears in the API: `status.remote.commit`, -`CommitRequest.status.commit`, `status.placement.resolvedAtCommit`. Not "sha", which names an -algorithm Git is in the middle of changing, and not "revision", which Flux uses for a polymorphic +`status.remote.parent.commit`, `CommitRequest.status.commit`, `status.placement.resolvedAtCommit`. +Not "sha", which names an algorithm Git is in the middle of changing, and not "revision", which Flux uses for a polymorphic identifier that may be a tag, a chart version or a composite like `main@sha1:`. If this project ever needs that composite, `revision` is the word waiting for it. diff --git a/docs/design/commit-timing-surface.md b/docs/design/commit-timing-surface.md index 66ad035f..1a674016 100644 --- a/docs/design/commit-timing-surface.md +++ b/docs/design/commit-timing-surface.md @@ -215,8 +215,11 @@ after the remote confirmed the push. Every one of these comes from the worker. Sending an attach does not prove an attach, so the controller reports a phase only when the worker says the save reached it. -The controller's safety timeout covers the largest `attachTimeout`, the largest `maxDuration`, and -the push cooldown and retries. +The controller's safety timeout applies only to a save the worker has not acted on: +`attachTimeout + maxDuration + 120s` from creation. Then the controller asks the worker to withdraw +the save and reports the worker's answer: failed when it cancelled the save, or the real outcome when +it resolved the save first. Once the worker holds a save (attached, or committed and waiting for the push), only the +worker ends it, so the controller never reports a failure the worker later contradicts. ## Naming diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md new file mode 100644 index 00000000..1b0fc2ba --- /dev/null +++ b/docs/design/gittarget-parent-branch.md @@ -0,0 +1,175 @@ +# GitTarget parent branch: keep a target on standby until a write needs a branch + +> **design**: built in #407, with #408's `spec.parentBranch` folded in and the hardening pass +> ([`gittarget-parent-hardening.md`](gittarget-parent-hardening.md), done) applied. This page +> records the contract as built and the reasoning behind it. Deferred: +> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and +> [`gittarget-parent-observation.md`](gittarget-parent-observation.md). +> Related: [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), +> [`../definitions.md`](../definitions.md), and `architecture.md` (Ground rules; "A branch that does +> not exist yet"). + +**The use case.** A `GitTarget` stays on standby while the cluster matches its parent branch. Its +watches run, its write branch does not exist on the remote, and normal Git changes keep flowing +through the cluster's GitOps reconciler. When a captured cluster edit needs a commit, the target +creates its write branch from the parent's current tip. A target that waited a week must not +publish that edit on a week-old checkout. + +```yaml +spec: + branch: reverser/prod-edits # the write branch (immutable) + parentBranch: env/prod # optional, immutable; omitted = the remote's default branch +``` + +**The constraint.** An idle, converged target creates no remote branch and no commit. Publication +checks the parent against the push session's advertisement and replays the captured writes if it +moved. If the replay leaves no difference, the target stays on standby. Commit windows and push +scheduling still apply. "No branch" means none on the remote; local working refs may exist. + +**Independent of polling.** The contract holds with `--git-refresh-interval=0`, before the first +refresh tick and between ticks. Periodic refresh improves the idle view of Git; the publication +check is what protects the first write. + +## Use cases + +- **Capture an operational edit for review.** The cluster follows `main` and `reverser/prod-edits` + is absent. An operator changes a Deployment during an incident. The edit is captured, the parent + is checked when publishing, and the write branch is created for review. Commits that landed on + `main` in the meantime are part of its starting history, and the replay re-plans the edit + against that tree, including a changed file layout. Keeping the edit active in the cluster still + depends on the GitOps reconciler's own policy. +- **Keep normal deployments quiet.** A deployment advances `main`, and the reconciler applies the + same manifests. The watch events can first look like edits against the cached parent; after the + check and replay they produce no difference, so no empty review branch and no catch-up commit + is created. An idle refresh only reads and reports. +- **A release or environment branch.** A cluster following `release-1.x` or `env/prod` needs its + proposed changes based on that branch. `parentBranch` says so. A set parent that is missing + blocks branch creation. +- **Return to standby after review.** When the PR is merged and the write branch deleted, the next + edit recreates it from the parent's current tip. While the write branch exists, new commits build + on it; it is never rebased onto the parent. + +## The contract as built + +| Write branch on remote | Parent branch on remote | Behavior | +|---|---|---| +| yes | any | work on the write branch; the parent is not followed | +| no | yes | compare against the parent; create the write branch from its tip, checked at publication | +| no | no, **set** | refuse: `Ready=False`, `Stalled=True`, reason `ParentBranchNotFound`; nothing written, no orphan branch | +| no | no, omitted, not empty (`HEAD` resolves to no branch, or only tags) | refuse the same way: `ParentBranchNotFound`, with a message to fix `HEAD` or set the field | +| no | no, omitted, empty repository (no refs at all) | unborn `HEAD`; the first commit is a root commit on the write branch | + +- **Recovery from a missing parent** is an obligation on the branch worker, not an observation. It + latches when a cycle fails on the parent and clears only when the work is published: retained + writes when they are pushed, and dropped writes per `(GitTarget, collection)` when a resync of + that scope is pushed or settles as a no-op. The worker probes with one advertisement after 10s, + doubling to 5m, shared by every target on it; before a deadline nothing spends a connection on + the parent, and reconciles never probe. Once found, it retries the retained writes and raises + a per-target snapshot request, which the controller turns into one recheck. Meanwhile the target + reports `RecoveringParentBranch`. +- **A parent change on a live worker** takes effect at a push admission boundary: every operation + reads one `{name, generation}` snapshot, trust is scoped to the generation, and a push is admitted + only when its root was chosen under the current one. A push already admitted is not recalled. +- **Default-branch resolution follows go-git**, which rewrites a hash-only `HEAD` before + `Remote.List` returns (to `master` when it matches, else the alphabetically first branch at that + hash). With the parent omitted, the parent is the default branch as last discovered; a refresh + that sees `HEAD` switch follows it with one fetch, and a push, which cannot see `HEAD`, does not. +- **Cost.** Unchanged remotes keep the ledger's rows 1–12. The new rows: a standby no-op live write + is 1 connection, the first write after the parent moved is 6 (row 6), the parent probe is 1 with + no fetch, and a write branch created during our upload is 8. + +- **Immutable**, including adding or removing it, like `branch` and `path`. It decides what history + a new write branch gets. Delete and recreate the `GitTarget` to change it. +- **Not in `allowedBranches`.** It is only read. +- **`parentBranch == branch`** is legal when that branch exists. When it is absent, creation is + refused, including in an empty repository. Only an omitted parent bootstraps. +- **One parent per write branch.** One worker owns one checkout, so two targets on the same + provider and write branch that name different parents are a `TargetConflict`; the later one + loses. Values are compared as written: omitted and `main` conflict even when the remote's `HEAD` + is `main`. The answer needs no round trip and does not flip when the default branch changes. +- **`status.remote.parent`** shows, while the write branch is absent, what it would be created + from: `state` (`Found`, `Missing` or `Unborn`), `branch`, and `commit` when found. It disappears + once the write branch exists. + +## The defect this started from + +Before #407 the push already recorded the parent as the cycle's root, so that it could refuse a +moved parent. But `validatePushState` compared the root with its advertised tip only when the write +branch existed: the one case the root was recorded for was the one it skipped. That left three +holes, each now fixed and tested: + +1. **Stale parent.** A warm worker that trusted its checkout of `main` at A created the write + branch on A after `main` moved to B. +2. **A concurrently created branch.** If somebody created the write branch between our fetch and + our push, our non-descendant commit could overwrite theirs. +3. **A no-op created the branch.** A window with no difference is still pushed, because only the + remote can confirm "already present", and it created the write branch at the parent's commit. + +The fix compares the root whether or not the pushed branch exists, rejects a write branch that +appeared, and settles a new branch whose tip is still the parent's as "no branch". Both +rejections take the existing replay path, and no connection was added: the advertisement is read +on the push session anyway. + +**Rejected: distrust the base while the write branch is absent.** An earlier version did that and +was reverted for its cost, and it would still leave the gap between a fetch and the push. + +**What is and is not guaranteed.** The parent is checked against the advertisement of the +successful push attempt. Creation is protected by the server's compare-and-swap with `Old = zero`. +Neither locks the parent, which can still move after the advertisement. + +## Prior art + +- **Flux `ImageUpdateAutomation`.** `spec.git.push.branch` "is created using + `.spec.checkout.branch` as the starting point, if it doesn't already exist". The starting point + matters only at creation; there is no rebase, and the usual remedy is to delete the branch after + the PR merges. +- **Argo CD source hydrator.** Free-form revisions appear only where Argo reads; everything it + writes to or creates a branch from is a typed `targetBranch`. `CheckoutOrNew(target, base)` uses + the base at creation only. With no difference there is no commit (a git note instead). +- **git-town and Graphite** use "parent branch" for a relationship that includes synchronization. + Ours is narrower: follow the parent only while the write branch is absent. + +We took a branch-typed field at a write position (Argo), and the parent as a creation-time input +(both). + +## Naming + +[`definitions.md`](../definitions.md) binds here. + +| Candidate | Problem | +|---|---| +| `sourceRevision` | "source" is the source *cluster* (`clusterProviderRef`, `rules[].sourceNamespace`) | +| `baseRevision`, `baseRef`, `baseBranch` | "base" already means a kustomize read-only base and base trust; `revision` is reserved for a polymorphic identifier; a `...Ref` field must reference an object | +| `checkout.ref.branch` | verbatim Flux, but a one-field block nested twice, and "checkout" names an action | +| `startPoint` | Git's own word, but it means creation only, and the field also governs what is followed | + +The cost of the name: it can never hold a commit or a tag. + +## Implicit when omitted, strict when set + +Omitted is lenient: the default branch, and a root commit when the repository is empty, because +that is the first-install case. Set is strict: it must exist, or the target refuses. Which branch +an empty repository's first commit may create is the open question in +[`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md). + +**Deferred, additive when someone asks:** + +- **A deliberately empty write branch in a non-empty repository** (the `gh-pages` pattern; Argo's + `CheckoutOrOrphan`). It would get its own field. +- **A pinned commit or tag.** Small value: the starting point only decides where the write branch is + born, and a pin leaves nothing to follow. Fetching one commit by hash also needs server support. + It would be a sibling field, never `parentBranch`. + +## After a merge + +Deleting the write branch after its PR merges returns the target to standby. A write branch left +on the remote is a separate policy question: its next change builds on that branch, which may have +fallen behind the parent, and after a squash merge a later PR can show old history. A candidate +rule is to reset the write branch to the parent's tip when it adds nothing to it, but resetting, +force-pushing or deleting is a mutation that needs its own authorization. It belongs with the +PR-creation design, not with status. What the operator can see about the relationship in the +meantime is [`gittarget-parent-observation.md`](gittarget-parent-observation.md). + +## Open questions + +1. Should each `GitProvider.status.branches` entry show its write branch's parent? diff --git a/docs/design/gittarget-parent-empty-repository.md b/docs/design/gittarget-parent-empty-repository.md new file mode 100644 index 00000000..58aa02df --- /dev/null +++ b/docs/design/gittarget-parent-empty-repository.md @@ -0,0 +1,198 @@ +# GitTarget parent branch: an empty repository's default branch + +> **design**: deferred; not part of #407. The experiment in §2 is done, nothing else is built. +> Dated 2026-10-01. +> +> Related: [`gittarget-parent-branch.md`](gittarget-parent-branch.md), the feature as built; +> [`gittarget-parent-hardening.md`](gittarget-parent-hardening.md), which keeps today's +> empty-repository behavior and defines "empty" (§4.2) and `Unborn` (§4.3). + +## 1. The problem + +Take an empty repository whose default branch is `main`, but which has no commits yet. Add a +`GitTarget` with `branch: reverser/edits` and no `parentBranch`. + +Today the first write creates **`reverser/edits` as the repository's first branch**: + +- The commit has no parent, and no `main` exists afterwards. +- What we pushed does not match the branch the repository says is its default. +- On GitHub, the first branch pushed to an empty repository probably becomes the default branch + (unverified, §5). If so, the operator's review branch silently becomes the repository's default + branch. Even if not, the eventual PR `reverser/edits` → `main` cannot be made: the two branches + have no common history. + +This is more than a status detail: for a review-branch workflow it is the wrong branch on the first +commit we make. It is also what the bootstrap contract says today ("only an omitted parent +bootstraps", and the first commit is a root on the write branch), so changing it is a policy and +compatibility decision, not a bug fix. + +`branch: main`, which is what the quickstart uses, is unaffected: the first commit creates `main`, +which is correct. + +## 2. What we know (experiment, 2026-10-01) + +**The server already tells us the default branch.** Over protocol v2, `ls-refs` with the `unborn` +argument reports the target of an unborn HEAD: + +```text +git -c protocol.version=2 ls-remote --symref https://github.com/ConfigButler/empty.git + (GIT_TRACE_PACKET) ls-remote< unborn HEAD symref-target:refs/heads/main +``` + +- Canonical git 2.39 does the same: a bare repository made with `git init --bare -b trunk` + answers `unborn HEAD symref-target:refs/heads/trunk`. +- Protocol v0 reports nothing. + +**go-git asks the question, then throws the answer away.** At v6.0.0-alpha.5, and unchanged on +upstream main at `fc18716c`: + +- It speaks protocol v2 by default, and sends `unborn` when the server advertises + `ls-refs=unborn` (`internal/transport/v2.go`, `lsRefsSupportsUnborn`). +- But when there are no hash refs, `GetRemoteRefs` returns plain `ErrEmptyRemoteRepository` for an + upload-pack session, and the reply is discarded: + - `plumbing/transport/http/handshake.go` ~252; + - `plumbing/transport/pack_stream.go` ~119. +- `RemoteRefs.Unborn` exists, but it is unreachable for an empty repository. +- So our `CheckRepo` has to answer "no way to know" (`internal/git/git.go:59`). + +**Why our live test passes.** `TestCheckRepo_PublicConnectivityEmpty` +(`internal/git/git_operations_test.go:175`, against `ConfigButler/empty.git`) runs, and is not +disabled. It skips only without network. It asserts that `DefaultBranch` is **empty** and that +the local HEAD is unborn on the *write* branch (`cool-test`). It pins today's behavior rather than +the server's knowledge. + +**A small, backward-compatible go-git patch recovers the answer.** It was tested through a +temporary `-modfile` `replace`; nothing in the repository changed. The patch: + +- adds `transport.EmptyRemoteRepositoryError{Unborn plumbing.ReferenceName}`, whose `Is(ErrEmptyRemoteRepository)` + is true; +- returns it from both empty-advertisement paths above. + +Results: + +- `Remote.List` against `ConfigButler/empty.git` returns an error that still matches + `errors.Is(err, ErrEmptyRemoteRepository)`, and `errors.As` yields `Unborn = refs/heads/main`. +- The existing live test still passes unchanged. +- The diff is in §7. + +**Our hermetic harness cannot show this today.** `startGitHTTPServer` +(`internal/git/ado_multiack_test.go` ~125) deliberately strips `Git-Protocol` to force protocol +v0, so that it simulates Azure DevOps. Against it, the patched go-git returns the plain error with +no `Unborn`. + +**Not reported:** any server or path that negotiates v0/v1. Whether Azure DevOps supports v2 is +unknown; the harness assumes v0. SSH to GitHub needs `GIT_PROTOCOL` sent over the session; that is +not verified. + +## 3. Direction + +Two things, in this order: + +1. **Get the evidence.** Recover the server's unborn `HEAD` through go-git (§4). With it, an empty + repository can report what it actually is: `status.remote.parent = {state: Unborn, branch: X}` + when the server said X, and an empty `branch` when it did not. Behavior does not change. +2. **Then decide the policy**, separately, with an explicit compatibility note. The question: when + the server says the default branch is X and the write branch is not X, may the first commit + still create the write branch as the repository's first branch? + + | Option | Effect | + |---|---| + | keep today's bootstrap | the write branch becomes the first branch (as now) | + | refuse until X exists | `Ready=False`, `Stalled=True`, a reason naming X: "push a first commit to 'X', or set `spec.branch: X`"; recovery is the normal standby flow | + | opt-in bootstrap of X | one atomic push: an empty-tree commit on X and the content on the write branch; X must be in `allowedBranches`. This writes to a branch the parent contract says is only read, so it needs its own field | + + Whichever is chosen applies only when the server reported X. Without that evidence, today's + behavior stays. + +### Considered and rejected: assume `main` + +Assuming the default branch is `main` when the server does not say (never `master`) was considered +as a stage that needs no go-git change. It is rejected: + +- It turns working configurations into refusals. A repository whose default is `trunk`, with + `branch: trunk`, bootstraps correctly today and would be refused. +- The default is configurable in practice: `git init` takes `init.defaultBranch`, and GitHub + organizations set the default name for new repositories. A guess is wrong exactly for those + users. +- It would publish a guessed name inside a remote observation. Making the guess visible does not + make it evidence; `status.remote` reports what the remote said. + +Also not introduced here: reading `parentBranch == branch` on an empty repository as "start this +branch with no history". That would reverse a rule of the built contract as a side effect; if an +explicit bootstrap form is wanted, it gets its own design. + +## 4. Getting the go-git change + +1. **Upstream first.** Open a go-git issue and a PR with §7: the error type, both paths, and tests + against a v2 `git http-backend` and over `file://`. It is additive and backward-compatible for + every `errors.Is` caller. An upstream PR may prefer to expose the unborn target on the + `Remote.List` result rather than only through the error; settle the shape with the + maintainers. +2. **Our side waits for a go-git release that carries it.** Until then nothing changes. +3. **Fallback, if upstream stalls:** a temporary `replace` to a fork tag, only with the user's + agreement. Do not hand-roll a v2 `ls-refs` client in this repository. + +## 5. Steps once go-git carries it + +1. **Verify the GitHub claim** in §1 on a *throwaway* repository, never on `ConfigButler/empty.git`: + does pushing a non-default branch first make it the default? Record the answer in §1. +2. **Harness.** Add a v2 variant of `startRealGitServer` that keeps `Git-Protocol`. Leave the + ADO/v0 server and the ledger rows untouched. +3. **Discovery.** `CheckRepo`, `listRemoteRefs` and `SmartFetchFrom` read the unborn target with + `errors.As`. `CheckRepo` returns `DefaultBranch{ShortName: X, Unborn: true}`; the observation + carries `ParentBranch=X` for `Unborn`. Update the doc comments that say "empty when Unborn" + (`remote_observation.go`, `api/v1alpha3/gittarget_types.go`, `configuration.md`). +4. **Tests.** On the v2 harness with a bare repository made by `init -b trunk`, status reports + `trunk`; on the v0 harness it reports an empty branch. Behavior is unchanged in both. The + ledger is unchanged: the target rides on the `ls-refs` call discovery already makes. +5. **Live test.** `TestCheckRepo_PublicConnectivityEmpty` expects `main` / `Unborn`. +6. **Then the policy decision in §3**, as its own change with an UPGRADING note. + +## 6. Out of scope + +- The policy decision in §3, until the evidence exists. +- Repositories that are non-empty but whose default branch is unresolved. That is hardening §4.3. + +## 7. The go-git patch used in the experiment (against v6.0.0-alpha.5) + +```text +--- a/plumbing/transport/errors.go ++++ b/plumbing/transport/errors.go +@@ import ( + "errors" + ++ "github.com/go-git/go-git/v6/plumbing" ++ + internal "github.com/go-git/go-git/v6/internal/transport" + ) +@@ ++// EmptyRemoteRepositoryError is ErrEmptyRemoteRepository with what the server ++// said about its unborn HEAD (protocol v2 ls-refs "unborn"), when it said it. ++type EmptyRemoteRepositoryError struct { ++ // Unborn is the branch HEAD points at, e.g. refs/heads/main; empty when ++ // the server did not report it (v0/v1, or no ls-refs=unborn support). ++ Unborn plumbing.ReferenceName ++} ++ ++func (e *EmptyRemoteRepositoryError) Error() string { return ErrEmptyRemoteRepository.Error() } ++ ++func (e *EmptyRemoteRepositoryError) Is(target error) bool { return target == ErrEmptyRemoteRepository } ++ ++func NewEmptyRemoteRepositoryError(refs []*plumbing.Reference) error { ++ return &EmptyRemoteRepositoryError{Unborn: NewRemoteRefs(refs).Unborn} ++} +--- a/plumbing/transport/http/handshake.go (smartPackSession.GetRemoteRefs, v2 branch) +- return nil, transport.ErrEmptyRemoteRepository ++ return nil, transport.NewEmptyRemoteRepositoryError(refs) +--- a/plumbing/transport/pack_stream.go (StreamSession.GetRemoteRefs, v2 branch) +- return nil, ErrEmptyRemoteRepository ++ return nil, NewEmptyRemoteRepositoryError(refs) +``` + +To reproduce the experiment: + +1. Copy the module from `/go/pkg/mod/github.com/go-git/go-git/v6@v6.0.0-alpha.5` into a scratch + directory and apply the patch. +2. Copy `go.mod` and `go.sum` to `exp.mod` and `exp.sum`, and append + `replace github.com/go-git/go-git/v6 => /go-git`. +3. Run `go test -modfile=exp.mod ./internal/git -run `. diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md new file mode 100644 index 00000000..124ef7be --- /dev/null +++ b/docs/design/gittarget-parent-hardening.md @@ -0,0 +1,721 @@ +# GitTarget parent branch: hardening pass before #407 merges + +> **Plan**: done, 2026-10-01, including the two recovery fixes the review of `93120caf` found +> (below). The implemented contract is in +> [`gittarget-parent-branch.md`](gittarget-parent-branch.md). §7 records the settled design choices. +> +> **Where the build departed from the plan.** +> +> - `SetParentBranch` no longer calls `invalidateBase` (§4.1 step 1 kept it as a belt). The +> generation already makes the old checkout untrusted, and the belt cost an existing write +> branch a fetch per change, which §4.1 test 6 forbids. +> - The replay a refused admission triggers is counted as a `recovery` fetch. +> - Canonical git advertises neither a `HEAD` that resolves to nothing nor its symref, so against +> it the dangling name (§4.3 test 1's `master`) never reaches the client: the observation's +> branch is empty and the message says the remote has no default branch. +> - Once the probe has found the parent it stops reading advertisements: later deadlines retry the +> retained writes and re-request the snapshots still owed. A failure caused by the parent arms +> the latch again. +> +> **The merge boundary.** #407 merges when this pass is done: the existing parent-branch feature +> behaves correctly under races and recovery, and stays as cheap as it is today. New policy +> (empty-repository bootstrap rules, ancestry status) is not part of it. +> +> Related: [`gittarget-parent-branch.md`](gittarget-parent-branch.md), the feature as built; +> [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and +> [`gittarget-parent-observation.md`](gittarget-parent-observation.md), deferred follow-ups. + +## Merge review at `93120caf` + +Both are fixed, each with the reviewer's reproduction as its regression test (red before the fix). +Both failures were reproduced through the worker event handlers against canonical `git http-backend`. + +1. **P1: a second parent outage loses the obligation for newer dropped writes.** A recovery + snapshot commits locally and enters `awaitingPush`. Before it is pushed, the parent disappears + again and another event from the same scope is dropped. `noteParentUnavailable` keeps the + scope's old `awaitingPush` entry. On recovery, `noteRecoveryPublished` clears that scope after + publishing the older snapshot, and no new snapshot is requested. The reproduction publishes + `cm1`, leaves `cm2` absent, closes recovery, and leaves `SnapshotRequestSeq` at 1. Invalidate + the old publication credit when new work is dropped, or track the obligation's generation. + Acceptance: publishing the old snapshot keeps the newer obligation open, requests a fresh + snapshot, and eventually publishes `cm2` without another cluster edit. + **Fixed:** a dropped write removes its scope's publication credit; pinned by + `TestParentRecovery_ASecondOutageKeepsTheWorkDroppedInIt`. +2. **P2: retained replay bypasses the parent probe backoff.** After a failed push loses its parent, + retained writes have `replayRequired` set. Every incoming live write calls + `recoverRetainedWrites`, which fetches through `invalidateAndRefresh` without checking + `awaitingParentProbe`. Three writes before the deadline produced three remote connections. + The existing probe-budget test starts with dropped work and no retained batch, so it misses + this path. Apply the hold before replay I/O and extend the budget test to retained work. + Acceptance: incoming writes before the deadline spend no remote connection; the scheduled + probe still drives recovery. + **Fixed:** `recoverRetainedWrites` checks the hold before its rebuild; pinned by the + `retained writes` case of `TestParentRecovery_ProbeBudgetIsPerWorker`. + +The fixes preserve the existing API and require no new policy. The required lint, unit, and e2e +gates have to pass on the corrected code; a green run on `93120caf` does not cover these +reproductions. + +At review, CI lint and unit tests passed. The local run passed all three parent-branch e2e specs, +including recovery without another cluster edit; the full local run and CI e2e were still running. +The targeted worker, controller, and router tests passed under `-race`. + +### After merge + +The immediate next fix PR remains the two `CommitEmpty` findings from #404 in +[`../TODO.md`](../TODO.md#follow-ups-from-404-commit-window-surface). +For the next GitTarget feature PR, take only step 1 of the parent-observation plan: retain parent +availability and its timestamp after the write branch exists, using existing advertisements. +Keep readiness and publication behavior unchanged, and prove no extra remote connections. + +Ancestry classification follows only after measuring the pinned go-git shallow-history behavior. +The empty-repository plan starts with the upstream unborn-HEAD fix; choosing which branch the first +commit creates remains a separate compatibility decision. The configuration-freshness record stays +deferred, and the remaining red-status contract work stays on its own branch. Do not reopen the +older API-wave plan as part of any of these changes. + +## 0. Handoff: read this first + +### Where the code is + +- Branch `fix/new-branch-checks-parent` = PR #407. #408 (the `spec.parentBranch` field) is already + merged into it. +- Run `git fetch` and `git log origin/main..HEAD` before starting. + +### Done first: CommitRequest liveness + +Recovery can hold work for minutes, so a request must not be failed on a controller clock while the +worker still holds it. That is already fixed on this branch: the controller waits for any request +the worker holds (`CollectingWindow`, `WaitingForPush`), and fails an unheld one only after +withdrawing it from the worker (`docs/spec/commitrequest-design.md`). The §4.4 tests must still +include a request retained through a recovery longer than `attachTimeout + maxDuration + 120s`, +resolving `Committed`. + +### Out of scope for this pass + +- [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md): which branch an + empty repository's first commit may create. Keep today's bootstrap behavior here. +- [`gittarget-parent-observation.md`](gittarget-parent-observation.md): the parent's state after + the write branch exists. + +### Process + +- Follow `AGENTS.md`. Its validation sequence is `task fmt`, `generate`, `manifests`, `vet`, + `lint`, `test`, `test-e2e`. Run the e2e commands sequentially, and check `docker info` first. +- **When to push: the user's standing instruction overrides AGENTS.md's "When to push".** AGENTS.md + says Git-write-path changes wait for a local e2e pass. The user has said, more than once: + "Don't wait for passing of e2e, as soon as linting is green you push", and "never ever keep + these commits unpushed: now we have to wait another 30 minutes for CI". + - So commit and push in one step as soon as `task lint` and `task test` pass. + - Keep local e2e running in the background, or read the CI legs, and fix forward. + - The PR is not ready until e2e is green somewhere. +- Use one commit per work item. Each item's red tests come first: either in their own commit, or + in the same commit with the red-then-green recorded in the PR body. +- At the end, update the #407 PR body. Add one entry per item to its "Review fixes" section. + After each push, read the CodeRabbit inline comments on `pulls/407/comments`; they are not the + same as `gh pr view --comments`. + +### Test harness facts + +- Any claim about server-side compare-and-swap, or about a refused upload, needs + `startRealGitServer` (canonical `git http-backend` over HTTP). The go-git v6 `file://` + transport never compares `cmd.Old`, so such a test passes vacuously over `file://`. Tests that + only assert our client-side rejection may use `file://`. +- `pushAtomicFn` is a package variable, so a test can wrap it. `fetchRemoteBranchHashFn` is one + too. Where §4.1 needs a hook at a specific point, add a nil-by-default package variable and set + it only in tests. +- `TestGitRoundTripLedger` runs against its golden file `internal/git/testdata/git-roundtrip-ledger.golden`. + The golden file counts connections to the Git host for each operation. It is the cost gate + (§1). +- Run controller envtest alone. It binds `:8080`, so a parallel run on the same host fails 21 + unrelated specs. + +**Do not re-raise.** These were fixed in earlier review rounds: + +- the concurrently created write branch, at the advertisement; +- the stale repository identity behind `ParentBranchNotFound`; +- retained writes not rebuilt on a parent change; +- path validity gating parent conflicts; +- the mutability of `parentBranch`, which is now immutable; +- the parent-change flag lost on a failed rebuild (`dcbc8b82`). §4.1 replaces that mechanism, but + its regression test `TestBranchWorker_AParentChangeSurvivesAFailedRebuild` must stay green. + +## 1. The contract this pass must not break + +**API first, not API only.** The design is optimized for a remote that rarely moves under us. The +normal path is: + +> use the trusted checkout → validate it through the push advertisement (a connection the push +> opens anyway) → fetch, reset and replay **only** when that advertisement says the base is +> stale. + +There is **no unconditional preflight fetch on a trusted checkout**. A pull before a push cannot +close the race anyway: the remote can move between any fetch and any push. What closes it is the +compare-and-swap plus the bounded replay, and both already exist. The legitimate fetches are: + +- bootstrap; +- a base that was invalidated, including by a parent-configuration change (§4.1); +- recovery after a detected change, a failed write, or a refused push. + +The guarantee we offer, and the only one: + +- The parent matches the push advertisement. +- The server's compare-and-swap protects the write-branch update. `Old` is the advertised tip, or + zero for creation. +- The parent can still move after the advertisement, and nothing pretends otherwise. + +### Acceptance criteria for every item + +1. **Cost.** Unchanged remotes keep today's request counts. Ledger rows **1–12** must not change. + §2 names the new rows. +2. **Bounded recovery.** A detected change triggers recovery that is bounded in attempts, in + connections, and in how long it waits. +3. **Progress.** Pending work eventually lands **without another cluster edit**, with + `--git-refresh-interval` on or `0`. That includes a recovery attempt that fails transiently + (§4.4). +4. **Loud failure.** A situation we cannot resolve correctly surfaces as a condition reason with + an actionable message. It never produces an orphan branch, or a branch on the wrong parent. + +## 2. Use cases, ranked by how often they happen + +The ranking decides what must stay fast: + +- **Hot paths (H)** have a connection budget that the ledger enforces. +- **Rare paths (R)** may spend a fetch, but each one needs a deterministic test and either a + correct outcome or a clear failure. + +| # | Situation | Expected | Budget (connections) | +|---|---|---|---| +| H1 | Write branch exists, remote unchanged, steady edits | commit and push | 2 per publication, 1 when there is nothing to commit (rows 3–5, unchanged) | +| H2 | Write branch absent (standby), idle | nothing written; the refresher only reads | 1 advertisement per refresh tick, 0 fetches (row 12, plus `TestRefresh_AnAbsentBranchWithAnUnmovedParentCostsOneConnection`) | +| H3 | Standby; a no-op live write (a deployment echo) | the branch stays absent; the request resolves `AlreadyPresent` | 1, the push advertisement; 0 fetches. **New row** | +| H4 | Standby; first real edit; parent unmoved since the checkout | branch created on the parent's tip | 2, 0 fetches (row 11, unchanged) | +| H5 | Standby; parent moved since the checkout | client-side refusal → contention fetch → replay → push | **New row**, pinned for the new-branch case (expected to equal row 6) | +| H6 | Write branch deleted after a merge; next edit | back to standby, recreated from the parent | one replay (existing lifecycle test) | +| H7 | Explicit `spec.parentBranch`, and the parent exists | as H2–H6, against that branch | same budgets | +| R1 | Parent configuration changes on a live worker (all targets replaced before the worker sweep) | the next push not yet admitted uses the new parent (§4.1) | at most one fetch per change | +| R2 | Empty repository gains its first ref while writes are pending | writes rebuilt onto the new default branch and published, or refused per R3 | one replay | +| R3 | Non-empty repository with no resolvable default branch, parent omitted | refuse with an actionable reason; no orphan | probe budget (§4.4) | +| R4 | Configured parent missing, later pushed | recover and publish without another cluster edit | probe budget (§4.4) | +| R5 | Remote default branch switched (parent omitted) | followed at the next discovery; documented | one fetch at that discovery | +| R6 | Write branch created by someone else after our advertisement, with the parent unmoved | the server refuses the upload → replay onto their commit | **its own new row**, measured (§4.7). It is not row 6: row 6 is refused at the advertisement, before any upload | + +**Probe budget.** While a parent is missing or unresolved, the remote is asked at most once per +backoff deadline per worker. That holds however many targets share the worker, and however many +reconciles and refresh ticks occur. §4.4 defines it and tests it. + +## 3. Decisions already taken (do not reopen) + +- **Keep `status.remote.parent`.** Revise the vocabulary rule instead (§4.5). +- **An omitted parent means the default branch resolved at the last discovery** (a fetch, a + refresh, or a probe). + - Publication validates that branch's tip in the push advertisement. The receive-pack + advertisement carries no `HEAD`, so publication cannot see a switch, and we do not add a + lookup before pushes to rediscover it. + - Stable intent is what `spec.parentBranch` is for. +- **The push admission boundary** (§4.1) decides when a configuration change takes effect. + - A change before admission makes that attempt replay. + - A change after admission does not recall the push. If that push created the write branch from + the old parent, the branch now exists, and an existing branch does not follow its parent. + That is consistent with the contract; document it. +- **The e2e contract.** + - e2e asserts ancestry and content: the first commit's parent is B, the files are present, and + the branch stays absent on standby. + - Deterministic worker tests with no refresher prove that publication alone finds a moved + parent. + - e2e does not assert which fetch reason found B (§4.6). +- **Default-branch resolution follows go-git, not a stricter rule of our own** (§4.3). +- **Recovery comes before cost.** §4.4 writes the recovery tests first. If the worker-side probe + grows larger than expected, ship the recovery obligation and its tests with a simpler + controller-side backoff (§4.4, fallback). Never weaken recovery to save fetches. + +## 4. Work items, in order + +### 4.1 Parent changes on a live worker (review items 1–2) → R1 + +#### Problem + +`SetParentBranch` runs on the controller goroutine. It drops base trust once and raises +`parentChangedState`. The event loop can then undo or bypass that: + +- **During a push or fetch.** + - `runPushCycle` calls `setBaseTrusted(true)` after the push returns (`branch_worker.go` + ~2021). `updateBranchMetadataFromPullReport` does the same after a fetch (~2953). Either can + run *after* the drop. + - `recoverRetainedWrites` with nothing retained then swaps the flag to false (~1416), relying on + a distrust that is gone. + - The `newBranchParent` shortcut in `ensureWriteBranch` (~2346) then roots the cycle on + `(main, A)`, and the push is accepted. + - Reproduced in review. +- **Between recovery and publication.** + - `pushPending` (~1656) runs `recoverRetainedWrites`, then `pushPendingCommits`. A change that + lands between the two is pushed on the old root. + - Reproduced. + +#### Design: one configuration snapshot, generation-scoped trust, and one admission boundary + +1. **Configuration snapshot.** + - Replace `parentBranchName` + `parentChangedState` with one atomic pointer to an immutable + `parentConfig{name string; gen uint64}`. + - `SetParentBranch` installs a new value, with `gen+1`, only when the name changes. It keeps + its `invalidateBase` as a belt. + - Every operation that consults the parent (fetch/sync, refresher advertisement, the §4.4 + probe, the rebuild) loads the snapshot **once, at its start**, and uses that `{name, gen}` + throughout. + - The snapshot also goes into the observation that operation records: `ParentRequested` (see + §4.3) and the generation. + - Nothing re-reads the current name after the network call returns. That way an old lookup can + never report a result about a newly configured parent. +2. **Generation-scoped trust.** + - The worker keeps `baseParentGen` (under `repoMu`): the generation of the snapshot whose fetch + produced the checkout. + - Trust means `baseTrustedState && baseParentGen == current gen`, so `setBaseTrusted(true)` + after an old-generation push or fetch can never vouch for the current configuration. + - `newBranchParent` is honored only when `baseParentGen` is current. + - The root's generation, `pushCycleRootGen`, is stamped wherever `pushCycleRootBranch` is set: + in `ensureWriteBranch` and in the rebuild. +3. **Retained writes.** + - In `recoverRetainedWrites`, "parent changed" becomes `pushCycleRootGen != current gen`. + - A failed rebuild leaves the generations mismatched, so the next attempt rebuilds again. This + replaces the `dcbc8b82` hand-back. +4. **The admission boundary.** One local point in `runPushCycle`, under `repoMu`, immediately + before each `pushAtomicFn` call, including the retries: + - load the snapshot and compare its generation with `pushCycleRootGen`; + - **on mismatch, do not push.** Take the existing replay branch: `markReplayRequired`, sync + with the new snapshot, rebuild, re-stamp, loop. This adds no extra connection; + - on match, the attempt is **admitted**. Any change from here on is in flight, per §3. + + Add a test-only hook, `beforePushAdmission`, called just *before* the comparison. A change + injected there is a "before" case. A change injected inside the wrapped `pushAtomicFn` is an + "after" case. +5. **An existing write branch is unaffected.** + - When the cycle root is the write branch itself, and it was proven present, a parent change + cannot alter the plan. + - Re-stamp the generation without fetching. The push's compare-and-swap still guards the + branch. + +#### Tests (red first; run with `-race` too) + +1. **Change during a fetch.** + - Inject `SetParentBranch("release")` inside the sync of an initial fetch and of a contention + fetch. + - Assert that `feature`'s first commit sits on `release`'s tip. + - Assert that the observation recorded for that fetch names the parent it actually used + (`main`), not `release`. +2. **Change during a no-op push** that settles as `PushNoBranch`. This is the review's + reproduction, an "after" case. Assert that the next real write lands on `release`. +3. **Change before admission** (the `beforePushAdmission` hook) in a cycle with retained writes. + - Assert that no push leaves on the old root. + - Assert that the replay builds on `release`. +4. **Change after admission** (inside the wrapped `pushAtomicFn`), in a cycle that creates the + branch. + - Assert that the branch is created from `main`, which is the documented in-flight rule. + - Assert that later writes go to the existing branch, with no attempt to re-root it. +5. **The new parent is missing**, in cases 1 and 3. + - Assert that `feature` is never created. + - Assert that the work stays retained. + - Assert that §4.4's recovery picks it up. +6. **An existing write branch, then a parent change.** + - Assert zero extra connections. + - Assert that the next push goes to the write branch. +7. `TestBranchWorker_AParentChangeSurvivesAFailedRebuild` and + `TestBranchWorker_AParentChangeRebuildsRetainedWrites` stay green. + +**Cost.** Everything added here is local. The ledger is unchanged. + +### 4.2 An empty repository gains a ref (review item 3) → R2 + +#### Problem + +Cloning an empty repository leaves `HEAD` unborn on the write branch, with the cycle root +`(feature, zero)`. If somebody then pushes `main`, neither check fires: + +- `validatePushState`'s missing-root check needs a non-zero `rootHash`. +- `remoteMovedSinceBase` compares `refs[feature]`, which is absent and therefore zero, with the + zero root. + +So the push creates `feature` as an orphan (`git_atomic_push.go` ~192–259). Reproduced on the real +server. + +**Defining "empty"**, once, and using it in discovery, publication and the observation: + +- A repository is empty when its advertisement carries **no hash refs at all**: no branches, no + tags, nothing else. +- A symbolic `HEAD` pointing at an unborn branch does not count. go-git reports that as + `RemoteRefs.Unborn`. +- A repository with only tags is **not empty**. + +#### Design + +- **A new typed error, `RepositoryNotEmptyError{Refs int}`, in `git_atomic_push.go`.** Do not + reuse `RemoteMovedError`: its `Branch` and `Advertised` fields mean "the root, compared", and + filling them with another branch's hash would make them lie. + - `validatePushState` returns it when `rootHash.IsZero()`, the write branch is not advertised, + and the advertisement is not empty by the definition above. + - The check reads the advertisement the push already holds, so it costs zero extra connections. +- **`remoteMovedDuringPush` classifies it directly as contention**, without the fallback probe. The + existing replay then runs: + 1. fetch, which resolves the default branch through §4.3, or refuses; + 2. rebuild, which **re-plans** from the retained writes rather than reusing the orphan commits; + 3. push. +- **Tags only:** discovery finds a non-empty repository with no resolvable default branch. It + refuses per §4.3, keeps the work retained, and §4.4 takes over. + +#### Tests (red first; real git server) + +1. **Empty remote with retained work, then `main` is pushed externally.** Start from an empty + remote holding a retained live write and a `CommitRequest`. Push `main` from outside before + publication. Assert: + - `feature`'s first commit has `main`'s tip as its parent; + - the content is present; + - `git merge-base feature main` is non-empty; + - exactly one contention fetch happened; + - the request resolves `Committed`. +2. **The same with a no-op write.** The branch stays absent, and nothing is orphaned. +3. **Empty remote, then only a tag is pushed.** Assert: + - no orphan is created; + - the work stays retained; + - the observation is `Missing` per §4.3; + - recovery happens once a branch with a resolvable `HEAD` appears. +4. **Empty remote throughout.** Ledger row 1 is unchanged, and the first push still creates the + root commit. + +### 4.3 Empty versus unresolved default branch (review item 4) → R3 + +#### Problem + +`SmartFetchFrom` returns `""`, the "empty repository" result, in two situations: + +- when the advertisement is empty (`git_smart_fetch.go` ~62); +- when refs exist but no default branch resolves and no parent is configured (~77–83, via + `analyzeRemoteRefs`, which only logs "Remote HEAD is broken"). + +The caller then runs `makeHeadUnborn`, and the result is an orphan. The surrounding status is +wrong in the same way: + +- the observation says `Unborn`; +- `recordAdvertisement` in `refresh.go` (~260) says the same; +- `parentBranchReadiness` treats every omitted parent as fine (`gittarget_controller.go` ~823). + +#### What the evidence looks like after go-git (verified at v6.0.0-alpha.5) + +`Remote.List` does not hand over the raw advertisement. A hash-only `HEAD`, from a server without +the `symref` capability or from a detached `HEAD`, is rewritten **before** `List` returns. +`packp.ResolveHeadFromHashHeuristic` turns it into a symbolic `HEAD`: + +- pointing to `master` if `master` has that hash; +- otherwise to the alphabetically first branch at that hash. + +That is git's own spirit too: `guess_remote_head` in `remote.c` prefers fallbacks over requiring a +unique match. So a stricter "exactly one branch matches" rule cannot be built on `Remote.List` +without dropping beneath it. + +**Decision: adopt go-git's resolution.** This is a deliberate choice, not an oversight. + +- Bypassing `Remote.List` to keep the raw evidence would mean a bespoke advertisement path for a + rare server configuration. +- The resolved branch is visible in `status.remote.parent.branch`. +- Anyone who needs certainty sets `spec.parentBranch`. + +The rules are then: + +- **`Unborn`**: the repository is empty by the §4.2 definition, and only then. +- **Resolved**: after `List`, `HEAD` is symbolic and its target is an advertised branch. This + covers go-git's heuristic. +- **Unresolved**: the repository is not empty, but `HEAD` is absent, symbolic to a missing branch, + or still a bare hash with no matching branch. + - Return a new sentinel, `ErrDefaultBranchUnresolved`. + - Handle it exactly like `ErrParentBranchNotFound` on every path: fetch, publication, refresh + and the §4.4 probe. + - The observation is `ParentState=Missing`. `ParentBranch` is the dangling `HEAD` target (for + example `master`), or empty when there is none. + - No write is planned. The work is retained, or the dropped writes are remembered (§4.4). + +#### Make an omitted parent able to stall + +- Add `ParentRequested` to `RemoteObservation`: the spec value, `""` when omitted, taken from the + §4.1 snapshot. +- `parentBranchReadiness` compares `ParentRequested` with `spec.parentBranch`, instead of the + resolved name, so an omitted parent can stall too. +- Use the same reason, `ParentBranchNotFound`, so that recovery stays one mechanism. Only the + message differs: + - "the remote's default branch (`HEAD` → `master`) does not exist; fix the remote's HEAD or set + `spec.parentBranch`"; + - or, with no `HEAD` at all: "the remote has no default branch". + +Update the documentation in three places: + +- the doc comment of `ParentUnborn` in `remote_observation.go`; +- the API types; +- the parent-branch contract table in `configuration.md`. That table should also record the + go-git resolution decision. + +**Tests**, all through the transport, against the real git server, not on synthetic ref lists: + +1. **`HEAD` symbolic to a missing branch.** Seed a bare repository with + `git symbolic-ref HEAD refs/heads/master` and only a `main` branch. Assert: + - the init observation and the refresh observation are both `Missing` with branch `master`; + - a write produces no orphan and stays retained; + - readiness is `ParentBranchNotFound` with the `HEAD` message. +2. **Detached `HEAD`.** Use `git update-ref --no-deref HEAD ` with two branches at that + hash. Assert that the resolution is the one go-git chooses (`master` if present, otherwise + the first alphabetically), and that the status names it. This pins the decision above. +3. **Recovery.** Repair the remote's `HEAD`, then recover through §4.4 with no new cluster edit. +4. **A truly empty remote** is still `Unborn`, and ledger row 1 is unchanged. + +### 4.4 Recovery from a missing or unresolved parent (review item 6) → R3, R4 + +#### Problem today + +- While `ParentBranchNotFound` holds, every reconcile sets `forceRecheck` + (`gittarget_controller.go` ~189, ~275). A target that is not Ready requeues every 10s. +- Each forced recheck restarts all of the target's streams. That means: + - a re-LIST of every watched type, which is the real cost; + - a replay that spends one advertisement connection. +- A typo such as `parentBranch: relase` keeps this up indefinitely. + +#### Why "force only on the transition, let the refresher probe" strands work + +- The refresher skips a branch that is mid-cycle (`refresh.go` ~101): retained writes, an open + window, a dirty worktree, or a required replay. +- A failed `pushPending` stops its push timer (~1673), and nothing re-arms it until the next + event. +- With `--git-refresh-interval=0` there is no refresher at all. +- An observation flip is not an obligation: + - If the first recovery attempt after `Found` fails transiently, the observation stays `Found`, + the timer is stopped, a probe gated on `Missing` no longer runs, and the work is stranded. + - A controller that keys off "the stored Ready reason was `ParentBranchNotFound`" can miss a + whole missing→found episode that happens between two of its reconciles. + +#### Design + +1. **A recovery obligation on the worker.** Its lifetime is decoupled from observations. + - The worker latches `parentRecovery` the moment a cycle fails with `ErrParentBranchNotFound` + or `ErrDefaultBranchUnresolved`. + - The latch has two independent halves: + - **`retained`**: set when writes were kept. It clears only when `pushPending` actually + publishes them, or settles them as `PushNoBranch`. + - **`snapshotNeeded`, per scope**: the set of `(target, collection)` scopes whose writes the + worker **dropped**, or whose resync it refused, because of the missing parent. One worker + serves several targets, and resyncs run per collection, so a successful ConfigMap + snapshot for target A proves nothing about a Deployment of A, or about target B. (The + refusal recovery in `resync_flush.go` is scoped the same way, for the same reason.) + Before implementing, verify which paths drop and which retain: `noteMissingParent` at + ~2425 and its callers. + - A scope clears only when a resync of that scope, started *after* the parent was found, is + **published**: its commit pushed, or settled as a no-op. A resync reply is not enough, + because a resync can succeed while its write still waits for the push. + - Neither half clears on an observation, on a probe result, or on a failed attempt. A failed + attempt re-arms the probe deadline (step 3). +2. **The worker drives recovery.** It runs on the event loop, while the latch is set, whatever the + refresh configuration and whether or not writes are retained. + - When the probe deadline fires, read one advertisement through `advertiseRemoteBranch`, with + the §4.1 snapshot. This is a read, not a fetch. + - If the parent is still missing or unresolved, record that and back off. + - If the parent (or the write branch) is now present, record it. Then: + - with `retained` set, call `pushPending`. It recovers through the normal replay: one fetch, + a rebuild, a push. A failure leaves the latch set and re-arms the deadline; + - for every target with an open scope, raise a **snapshot request**: a monotonically + increasing `snapshotRequestSeq` per target, exposed alongside the observation. A scope + that fails raises its target's sequence again on the next deadline. +3. **One probe deadline per worker, shared by every background probe.** + - The worker keeps `nextRemoteProbeAt`, with backoff 10s, doubling, capped at 5m. It resets on + a generation change (§4.1) and on success. + - While the latch is set, a **refresher tick** that arrives before the deadline is a no-op: no + connection. A tick after it counts as the probe and advances the deadline. + - **Reconciles never probe.** That holds however many targets share the worker. + - Live events, while the latch is set and before the deadline, are retained or dropped as + today, **without** a fetch. The deadline governs them too. +4. **The controller acts on the request sequence, not on the Ready reason.** + - It remembers, in memory per target (like `reconcileRequests.take`), the last + `snapshotRequestSeq` it acted on. + - When the worker's sequence is higher, it forces **one** recheck and records the sequence. A + missing→found episode between two reconciles is therefore never lost. + - After a controller restart the memory is empty. That is harmless: every stream starts fresh + with an initial list, which is itself the snapshot. + - Drop the per-reconcile `parentWasMissing` force. + - Ready reporting: + - `ParentBranchNotFound` stays Ready=False/Stalled while the parent is missing. + - While the obligation is open but the parent has been found, report Ready=False with + `Reconciling=True`, reason `RecoveringParentBranch`. This keeps the not-Ready requeue + cadence, so the controller sees the sequence within one requeue, and the state stays + visible. Verify that the not-Ready requeue applies to that reason. + - The 10s requeues read local observations only, so they cost zero connections. + +#### Tests + +Write the recovery tests first, against current code. They are the safety net for the design. + +1. **Worker, retained writes, real server.** The parent is missing while writes are retained. + Then: + - the parent is pushed externally; + - the **first recovery attempt fails** (inject a GitProvider read failure, as the reproduction + for `dcbc8b82` did); + - the next deadline recovers and publishes, **without another cluster edit**. + + Run it with refresh enabled and with refresh at `0`. +2. **Worker, dropped writes, two scopes.** Two targets share the worker; writes for both are + dropped while the parent is missing. When it appears, one scope's resync succeeds and the + other's fails once. Assert that only the published scope clears, that the failing target's + sequence rises again, and that it clears after a later success. A resync that succeeded but + has not been pushed yet must not clear its scope. +3. **Controller unit.** A sequence bump forces exactly one recheck. A bump that happens entirely + between two reconciles is still acted on. Ten reconciles at an unchanged sequence force + nothing. +4. **Probe budget.** Take three targets sharing one worker, a parent that stays missing, refresh + ticks every few seconds, and repeated reconciles, over a simulated window of N backoff + periods. Assert that the advertisement connections stay ≤ the backoff schedule's count, and + that there are zero fetches. + + This needs an injectable clock on the probe deadline. Add a small `now func() time.Time` field + rather than sleeping. +5. **The §4.3 cases.** An unresolved `HEAD`, once repaired, recovers through the same path. +6. **e2e: dropped-write integration.** There is no controller+watch+worker harness outside e2e, + and building one is out of scope; e2e is that integration test. Add a third context to + `new_branch_parent_e2e_test.go`: + - set `parentBranch: release` while `release` is absent; + - assert `ParentBranchNotFound` and that no write branch exists; + - push `release` from outside; + - assert Ready, and the content on a new branch whose first commit's parent is `release`'s + tip; + - make no cluster edit after the push. + + It also covers recovery from `ParentBranchNotFound`, which no test exercises today. +7. **Ledger rows.** Add "parent probe while missing" = 1 connection, 0 fetches. Row-level counts + do not prove frequency; test 4 does. + +#### Fallback, if step 2 or 3 grows too large + +- Keep step 1 (the obligation) and step 4 (the request sequence). The obligation is the + correctness part and is non-negotiable. +- Replace the worker probe with a controller-side forced recheck under exponential backoff, still + gated on the latch rather than on the Ready reason. +- **The schedule is per worker, not per target**, so the per-worker budget in §2 still holds: key + `lastForcedAt` by the worker's `(provider, branch)` and force all of that worker's affected + targets in one round. Per-target backoff would give three targets on one worker three + independent retry streams. +- Tests 1–3 and 6 still apply. Test 4 then asserts the controller-side schedule across three + targets on one worker. +- Record the worker probe as deferred work in the PR body. + +### 4.5 Vocabulary (review item 5) + +Keep `status.remote.parent`. Reword the new rule in `docs/definitions.md` (~132): + +> "parent branch"; a bare "parent" is fine where the subject is already a branch, such as the +> `status.remote.parent` stanza or a sentence that has just named the parent branch. Use the full +> term wherever a parent *resource* could be meant. + +Then sweep `configuration.md`, `architecture.md` and the Go doc comments, but only for genuinely +ambiguous uses. + +### 4.6 e2e: no dependence on refresh timing (review item 7; CodeRabbit comment on `:238`) + +The last step, "the publication check, not the refresher, is what found B", asserts both that +`contention` rose and that `refresh` did not change. + +- With `--git-refresh-interval=30s`, a refresh can find B first. `contention` then legitimately + never rises. +- Relaxing only the `refresh` equality does not fix this. + +#### The fix + +1. Delete that step, and keep the ancestry and content assertions. +2. Add a comment pointing at the deterministic worker tests that prove publication alone finds B, + with no refresher: + - `TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip`; + - `TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent`. +3. Reply on the CodeRabbit thread with that rationale. + +### 4.7 Default-branch switches, and the refused upload (review item 8) → R5, R6 + +#### R5: make the decision in §3 true, and stop the status flapping + +- Today `followParent` (`refresh.go` ~276) compares hashes only. Make it also compare + `advertisement.parentBranch` with the branch the checkout was rooted on: `newBranchParent`, or + the branch the last sync resolved. +- On a mismatch, call `syncWithRemote(fetchReasonRefresh)`. That is one fetch, on a rare event. +- This also ends the flapping of `status.remote.parent.branch`, where no-op pushes record `main` + while refreshes record `trunk`. +- **Tests.** + - Switch `HEAD` from `main` to `trunk`, both at the same commit. Assert that a refresh resyncs + once, and that the next no-op push records `trunk`. + - With refresh at `0`, assert that the worker keeps `main` until the next discovery. That is the + documented behavior. +- **Docs.** Document the omitted-parent semantics in `configuration.md`. + +#### R6 needs a production fix, not only a test + +How it fails today: + +- `feature` is created by somebody else after our advertisement, and the parent `main` is + unmoved. +- Our upload with `Old=zero` is refused by the server, as an untyped error. +- `remoteMovedDuringPush` (~2084) finds no typed rejection. It falls back to + `fetchRemoteBranchHashFn` for the **root** branch, `main` (~2100). +- That probe sees `main` unmoved and concludes "not moved". There is no replay, and the work is + stuck. + +The fix: + +- Only on this exceptional path: when the cycle creates the branch (`rootBranch` is not the write + branch) and the failure is untyped, make the fallback discovery read **both** refs from one + advertisement. +- If the write branch is now advertised, treat it as contention, as the typed write-branch case + already does. +- Do not add any discovery to the normal path. + +**Tests** (real server; add a test-only seam between `validatePushState` and `performPush`): + +- Create `feature` from outside inside that seam. +- Assert that the server refuses the upload. +- Assert that the worker replays onto their commit, and that their commit is an ancestor of ours. + +**Ledger.** Add R6 as its own measured row. It includes a refused upload plus the fallback +discovery, so it will not equal row 6. Record whatever it measures, check that the number is +explained by the steps above, and do not tune toward row 6. + +## 5. Finish + +1. Regenerate the ledger golden file (`-update`) and **diff it by hand**. + - Rows 1–12 must be byte-identical. + - Only the new rows may appear: H3, H5, the parent probe, and R6. +2. Run `task lint` and `task test`, then commit and push per item (§0). +3. Update the #407 PR body: + - one "Review fixes" entry per item; + - the cost table from §2; + - the e2e contract and the go-git resolution decision from §3 and §4.3. +4. Read the CodeRabbit inline comments again after each push. +5. Run `task test-e2e`, and the `full-manager` leg in CI. §4.4's new e2e context must pass at least + once before calling the PR ready. +6. Fold what shipped into [`gittarget-parent-branch.md`](gittarget-parent-branch.md) (the + built contract) and `architecture.md`, and mark this plan done. + +## 6. Out of scope + +- The two deferred follow-ups named in §0. +- Resetting, force-pushing or deleting a surviving write branch, and PR management. Ancestry + status or folder equality is not enough authorization for those mutations. +- Any fetch before publication on a trusted checkout. + +## 7. Design choices the review rounds settled + +Kept here so they are not re-litigated. Each was checked against the code. + +- **No pull before push.** A pull cannot close the race; the compare-and-swap plus bounded replay + does. Every fix in this pass reads evidence the push or discovery already holds. +- **Recovery is an obligation, not an observation.** It stays open until the work is published, + so a transient failure after the parent reappears cannot strand it (§4.4). It is tracked per + `(target, collection)` scope. +- **A configuration change is applied at a push admission boundary**, with one `{name, gen}` + snapshot per operation (§4.1). A push already admitted is not recalled. +- **One probe budget per worker**, whichever variant of §4.4 ships. +- **An empty repository has no hash refs**, tags included, and gaining one is reported with its own + typed error, not a `RemoteMovedError` that would misstate its fields (§4.2). +- **Default-branch resolution follows go-git**, which rewrites a hash-only `HEAD` before `List` + returns; a stricter rule would need a bespoke advertisement path for a rare server setup (§4.3). +- **The refused upload (R6) needed a production fix**: the fallback probe looked at the root only + (§4.7). It gets its own ledger row, because a refused upload costs more than a refusal at the + advertisement. +- **The dropped-write integration test is e2e.** No controller, watch and worker harness exists + outside it, and building one would be larger than the bug. diff --git a/docs/design/gittarget-parent-observation.md b/docs/design/gittarget-parent-observation.md new file mode 100644 index 00000000..5bb369db --- /dev/null +++ b/docs/design/gittarget-parent-observation.md @@ -0,0 +1,260 @@ +# GitTarget parent observations with shallow history + +> **design**: deferred follow-up to PR #407; nothing in this document is implemented. +> Date: 2026-10-01. + +Keep the parent branch visible after the write branch exists. Reuse remote ref advertisements to +report its current tip, then use bounded local history checks to describe the relationship where +the evidence allows it. An incomplete history produces an explicit `Unknown` result. + +The cost boundary is part of the design: no extra remote connection, history fetch, or deepening +for this status feature. Parent observations are informational. Existing publication checks still +decide whether a write can proceed. + +## Why keep observing the parent + +An operator needs to see whether an existing write branch still includes the parent's current +tip. The parent can advance, rewind, disappear, or acquire unrelated history while the write +branch remains active. Dropping the parent from status removes that context precisely when the +branches can start to differ. + +PR #407's [API types](../../api/v1alpha3/gittarget_types.go) describe `status.remote.parent` only +while the write branch is absent. Its `Found`, `Missing`, and `Unborn` states explain where a new +branch would start. This proposal retains that observation after creation and adds a separate +ancestry result. + +The parent remains the branch selected by `spec.parentBranch`, or the advertised remote default +branch when that field is omitted. This describes the current comparison target. Git does not +record a branch's historical creation parent, and this status makes no such claim. A change to +the remote default branch changes the comparison target, even if its tip hash stays the same. + +## What the existing operations can tell us + +The [remote refresher](../../internal/git/refresh.go) already reads the write branch and parent +from one ref advertisement. Keeping both observations adds no object transfer. The advertisement +still has network and ref-listing costs, so reuse the existing refresh cadence and write-related +observations rather than adding another poll. + +Two advertised hashes prove equality or inequality. They provide no history explaining the +difference. A changed parent hash can mean an advance, a rewind, or a force push. Calling all three +`ParentAhead` would give operators the wrong information. + +[`SmartFetchFrom`](../../internal/git/git_smart_fetch.go) fetches with `Depth: 1`. That limits +downloaded history, but local commits can extend the fetched tip: + +```text +older history unavailable <- A <- W1 <- W2 + ^ ^ + parent tip write tip +``` + +If the objects from `W2` back to `A` remain available, their parent links prove that the write tip +includes `A`. There is no need to fetch the history before `A` for that answer. + +There is a second boundary: Git's normal revision walk treats commits listed in `.git/shallow` +as roots, even though their commit objects retain parent links. See +[Git's shallow repository documentation](https://git-scm.com/docs/shallow). An exploratory check +with system Git 2.39.5 found that fetching the pushed write tip again with `--depth=1` shortened +the visible log to that tip, while an earlier base object remained readable with `git cat-file`. + +This suggests inspecting locally stored commit objects directly for positive ancestry evidence. +The experiment does not establish the behavior of the repository's `go-git/v6` dependency; that +needs a focused test before implementation. Object retention is also optional: a new checkout, +repository replacement, or cleanup can remove evidence that a long-lived worker once had. + +## Proposed status contract + +Parent availability and ancestry answer different questions and use separate fields. + +### Parent availability + +Retain `status.remote.parent` after the write branch exists. Its existing fields continue to name +the selected parent and its last observed tip, with a parent-specific observation timestamp. + +| State | Meaning | +|---|---| +| `Found` | The selected parent branch was advertised with a commit. | +| `Missing` | The selected parent branch was absent from a successful advertisement. | +| `Unborn` | The repository was empty and no explicit parent was configured. | + +`Missing` blocks creation only while the write branch is absent, preserving PR #407's behavior +for an explicit parent. An existing write branch stays writable when its parent disappears. +`Unborn` must not become a catch-all for a nonempty repository with an unresolved default branch. +That case needs an explicit observation reason and no ancestry comparison. + +A failed advertisement does not prove that a branch is missing. Keep the last successful +observation and its timestamp; existing remote-error reporting describes the failed attempt. +Before the first successful observation, leave the parent observation absent. + +### Ancestry relationship + +Compare the advertised remote write tip with the advertised parent tip. Local `HEAD` can include +unpublished commits and must not substitute for the remote write tip. + +The recommended first implementation has four outcomes: + +| Relationship | Required evidence | +|---|---| +| `SameTip` | Both advertised hashes are equal. | +| `WriteAhead` | Distinct tips; a parent-link path leads from the write tip to the parent tip. | +| `ParentAhead` | Distinct tips; a parent-link path leads from the parent tip to the write tip. | +| `Unknown` | Available evidence and the work budget do not establish one of the above. | + +These values describe ancestry, not elapsed time or the number of commits. A parent rewind can +produce `WriteAhead`. Following a merge's second parent can also establish ancestry. Commit author +names and timestamps provide no substitute for parent links. + +`Unknown` carries a reason, such as `HistoryIncomplete`, `BudgetExceeded`, or +`RelationshipUnclassified`. The last reason covers a completed local check whose result falls +outside the initial enum. An object read failure also leaves the relationship unknown and must +retain a diagnostic; it cannot establish a negative ancestry result. + +When either branch has no observed tip, omit the comparison. The parent availability and write +branch observation already explain why comparing two commits is inapplicable. + +### Bind the result to the evidence + +Store the exact compared hashes and the time of their joint remote observation with the result. +An illustrative layout is below; field names remain subject to API review. Hashes are shortened +for readability, but status stores complete object IDs. + +```yaml +status: + remote: + commit: "b4e29d1" + lastVerifiedAt: "2026-10-01T12:00:00Z" + verifiedBy: Fetch + parent: + state: Found + branch: main + commit: "a73c012" + lastVerifiedAt: "2026-10-01T12:00:00Z" + comparison: + relationship: WriteAhead + writeCommit: "b4e29d1" + parentCommit: "a73c012" + observedAt: "2026-10-01T12:00:00Z" +``` + +A successful push verifies the new write tip. It does not by itself verify the parent's tip at +that same time. Preserve the parent's own observation time and clear the comparison when its +inputs no longer match. The next suitable advertisement can establish a new comparison. Never +refresh the parent timestamp solely because `status.remote.lastVerifiedAt` changed. + +## Bounded local comparison + +Run comparison as observation work, with finite limits on visited commits, queued parent edges, +and elapsed time. Support cancellation and serialize object reads with repository replacement or +cleanup. Choose internal limits using measurements before implementation; this proposal adds no +operator configuration for graph-walk budgets. + +1. Resolve both branch names and hashes from the same successful advertisement. If either tip + is absent, publish availability and omit the comparison. +2. Return `SameTip` immediately for equal hashes. No commit objects are needed. +3. Look for a cached result for this repository identity, resolved branch pair, and exact tip pair. +4. Walk locally available parent links from the write tip, looking for the parent hash. A found + path proves `WriteAhead`. Check the reverse direction within the same total budget for + `ParentAhead` when needed. +5. Stop on proof or budget exhaustion. A missing object leaves that path incomplete; other queued + paths may still prove ancestry. If neither direction supplies proof, publish `Unknown` and + the reason for the limit on the result. + +Bound the cache as well as the walk. Successful proofs for an unchanged pair can be reused after +a fresh advertisement. Cache unknown results only while the local object store has unchanged +evidence: a normal fetch may make the same pair answerable. Budget-limited results should not +start an immediate retry loop. Discard work whose repository or observed tips changed before the +result was published, and invalidate the cache when the repository identity changes. + +The comparison must not extend the write critical path without a bound or block readiness. +Continue existing status-write throttling, while publishing changed facts promptly. With +`--git-refresh-interval=0`, observations age until another existing operation reads the refs; +this feature does not introduce a replacement timer. + +## Why a missing overlap remains unknown + +A bounded search that finds no shared commit cannot distinguish missing history from histories +that never intersected. Even finding a shared commit does not by itself prove divergence: one tip +could still be an ancestor of the other along an unexplored merge parent. + +Two additional classifications are useful but deferred from the first implementation: + +| Relationship | Proof required before reporting it | +|---|---| +| `Diverged` | A common ancestor exists, and neither tip is an ancestor of the other. | +| `Unrelated` | Both complete ancestor graphs have been explored with no common commit. | + +Negative proofs require exploring every relevant parent edge or an equivalent sound proof. An +unresolved shallow boundary, missing object, or exhausted budget prevents that conclusion. Neither +outcome justifies downloading more history solely to fill status. + +There is also no requirement to publish the first overlapping commit. Git's graph can contain +multiple best common ancestors, so the first intersection found by a walk is not a reliable +merge-base contract. See [Git's merge-base documentation](https://git-scm.com/docs/git-merge-base). +The initial status needs only the relationship evidence above. + +## A cheaper alternative: compare with a remembered starting tip + +A worker could remember the verified parent hash used when it creates or rebuilds an absent write +branch, then compare later parent advertisements with that hash. This needs no graph walk: +`Unchanged`, `Changed`, or `Unknown` would describe movement relative to that recorded starting tip. + +This is a different question from ancestry. `Changed` cannot say whether the parent advanced, +rewound, or was replaced, nor whether the write branch later incorporated its new tip. The saved +hash also needs provenance tied to the repository and the particular branch creation. External +branch replacement, deletion and recreation, or lost worker state can invalidate that provenance. + +Keep this as an alternative if bounded local ancestry proves too costly or rarely informative. +Do not add both models in the first implementation. An in-memory starting tip is lost on restart; +durable origin tracking would need a separate design and must not be reconstructed by guessing. + +## Operational boundaries + +Parent comparison is status only. It does not merge, rebase, reset, delete, or recreate an existing +write branch. `Unknown` and a missing parent after branch creation do not change `Ready`. +The publication checks and replay behavior from PR #407 continue to protect creation of an absent +write branch independently of these observations. + +Exact ahead/behind counts, pull request management, and content-equivalence checks are outside +this proposal. A squash merge can integrate the same changes while preserving distinct ancestry; +this status must not claim that a branch is merged or that its changes remain unapplied. + +Extending the lifetime of `status.remote.parent` changes its documented meaning. Implementation +must update API comments, generated CRD descriptions, and user documentation together. Consumers +must use the remote write commit to determine branch presence; parent visibility no longer +implies write-branch absence. + +## Implementation and validation + +Ship it in two steps, on a follow-up branch after PR #407: + +1. **Parent availability and timestamps**: retain the parent observation through the existing + refresh path after the write branch exists, and project it into status. This needs no history. +2. **Ancestry classification**, only after measuring how often the shallow checkout can answer + it. If most results come out `Unknown`, the graph walk is worth less than it looks, and the + remembered starting tip above may be enough. + +Before step 2, test the shallow-object behavior with the pinned `go-git/v6` version. + +Status is informational only. Nothing here authorizes resetting, force-pushing or deleting a write +branch; that stays with the PR-creation design. + +The implementation needs these cases: + +- Equal advertised tips with no local objects; an absent write branch; an empty repository. +- Parent disappearance with and without an existing write branch; failed advertisements that + preserve the last successful observation; an unresolved default branch in a nonempty repository. +- A shallow base followed by local commits, including another depth-one fetch after publication. +- Restart with only the two tips available, yielding `Unknown` when intermediate objects are absent. +- Positive ancestry through a merge's second parent; no false divergence at a missing object. +- Both ancestry directions, a parent rewind, a force push, and a parent branch rename or default + branch change, including different names that resolve to the same hash. +- Count, memory, time, and cancellation limits; an unchanged tip pair becoming answerable after a + normal fetch; cache invalidation after repository replacement. +- A local unpublished commit and a later successful push, proving that comparisons never silently + switch from remote evidence to local `HEAD` or acquire an unearned freshness timestamp. +- Remote call and fetch counts showing no additional network work, unchanged readiness, and no + status-driven Git mutations with periodic refresh enabled or disabled. + +Before implementation, settle the exact field names, the representation of an unresolved default +branch, and measured internal work limits. Those choices must preserve the cost boundary and the +distinction between an unavailable answer and a proved relationship. diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index c3f0b914..3df4ebb8 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -103,9 +103,21 @@ object is visible. | Finalize or push error | `Ready=False`, `Pushed=False`, `Stalled=True`, reason `FinalizeFailed` | `Reconciling=True` is the normal in-progress state, with the phase the worker reports as its reason: -`Progressing`, `WaitingForWorker`, `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. The controller fails -with `FinalizeFailed` only if the worker does not resolve the request within its bounded safety window; it -never polls indefinitely. +`Progressing`, `WaitingForWorker`, `WaitingForWindow`, `CollectingWindow`, `WaitingForPush`. Once the +request is `CollectingWindow` or `WaitingForPush` the worker holds it, and only the worker ends it. Before +that, a request still unresolved `attachTimeout + maxDuration + 120s` after creation is withdrawn from the +worker, which cancels it for good, and fails with `FinalizeFailed`; a GitTarget that never started a worker +is named in the message. The withdraw rides the worker's queue behind every attach, and is a no-op for a +request the worker holds, so the controller's failure and the worker's answer cannot disagree. + +The withdraw goes to the worker that accepted the request's attach, not to whichever worker the GitTarget +names now: a worker shared by several GitTargets outlives a deleted one, and a retired worker finishes its +last push after the manager stopped listing it. Only a request no worker holds falls back to the GitTarget, +and only then does "no GitTarget" or "no worker" mean that nothing can commit it. Once the controller starts +withdrawing it sends no further attach, and the worker keeps a request's outcome while any attach for it is +still queued, so a late attach can never register a withdrawn request again. A worker whose loop has exited, +including one that never started because its GitProvider could not be read, gives back every request it +never acted on and answers a withdraw at once. **`Committed` and `AlreadyPresent` are decided by the push, including the no-commit one.** (`WindowMismatch` is decided locally, at the deadline: no window was attached, so there is nothing @@ -127,8 +139,8 @@ the re-send is recognized as the same request: it cannot register afresh and exp same-author window and stamp its message on somebody else's commit. Its close deadline is spent and no longer arms anything. -**A worker that stops while still holding the write fails the request**, rather than leaving the -controller to poll until its own safety window expires: a timeout says nothing about what happened, -and "the worker stopped before the commit reached the remote" does. +**A worker that stops while still holding the write fails the request**, because the controller +does not time out a request the worker holds, and "the worker stopped before the commit reached the +remote" says what happened. The complete status vocabulary is in the [status conditions guide](status-conditions-guide.md). diff --git a/hack/crdfields/fields.yaml b/hack/crdfields/fields.yaml index 67ed1952..cdad4efd 100644 --- a/hack/crdfields/fields.yaml +++ b/hack/crdfields/fields.yaml @@ -88,6 +88,12 @@ kinds: doc: The `GitProvider` backing this target, in the same namespace - path: branch doc: Branch to write. Must be in the provider's `allowedBranches` + - path: parentBranch + default: the remote's default branch + doc: >- + Branch a `branch` the remote does not have yet is created from. Immutable. Only read, so + it need not be allowed. See + [starting the write branch from another branch](#starting-the-write-branch-from-another-branch-specparentbranch) - path: path doc: Folder within the repository. `.` targets the root; empty is rejected - path: clusterProviderRef diff --git a/internal/controller/commitrequest_controller.go b/internal/controller/commitrequest_controller.go index 154cefba..434743e0 100644 --- a/internal/controller/commitrequest_controller.go +++ b/internal/controller/commitrequest_controller.go @@ -33,6 +33,10 @@ import ( // registration and runs the window's timers, so the controller never holds the finalize itself. type CommitRequestFinalizer interface { ServiceCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) + // WithdrawCommitRequest cancels a request the worker has not acted on and reports the worker's + // answer: resolved with git.ErrCommitRequestWithdrawn when it is cancelled, the real outcome + // when the worker got there first, or resolved=false with the worker's phase while it decides. + WithdrawCommitRequest(ctx context.Context, attach git.AttachCommitRequest) (git.FinalizeResult, bool, error) } // CommandAuthorLookup resolves the author of a CommitRequest from the submitter @@ -50,11 +54,14 @@ type CommandAuthorLookup interface { const windowMismatchMessage = "the open commit window belongs to a different author or GitTarget; " + "nothing was committed for this request" -// resolveTimeoutMessage explains the fail-closed resolve bound: the worker never -// reported an outcome for the attached request within the safety window (e.g. the -// branch worker vanished), so the request fails closed rather than poll forever. +// resolveTimeoutMessage explains the fail-closed resolve bound: the worker never acted on the +// request within the safety window, so it was withdrawn rather than polled forever. const resolveTimeoutMessage = "the CommitRequest finalize did not resolve within the safety window" +// noWorkerTimeoutMessage is resolveTimeoutMessage for a GitTarget that never started a branch +// worker, the one cause the controller can name without asking anybody. +const noWorkerTimeoutMessage = "the CommitRequest's GitTarget did not start a branch worker within the safety window" + const ( // commitRequestPollInterval is the requeue cadence while polling the worker // for the attached request's outcome (attach-then-poll). @@ -108,9 +115,11 @@ func resolveRequestWindow(spec configbutleraiv1alpha3.CommitRequestSpec) request return resolved } -// resolveTimeout bounds the attach-then-poll wait, measured from object creation: the request's -// own attach wait, then its collection, then the push. Past it, a request the worker never resolved -// (e.g. a vanished worker) fails closed instead of polling forever. +// resolveTimeout bounds how long a request may go without the worker acting on it, measured from +// object creation: the request's own attach wait, then its collection, then the push. It does NOT +// bound a request the worker holds (attached to a window, or committed and waiting for the push): +// only the worker can say how that ends, however long a down remote or a recovery takes. Past the +// bound, an unheld request is withdrawn, and fails only once the worker agrees. func (w requestWindow) resolveTimeout() time.Duration { return w.attachTimeout + w.maxDuration + commitRequestPushAllowance } @@ -232,7 +241,7 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques // (no controller-side delay — the worker anchors the wait at its registration) // and poll the outcome. window := resolveRequestWindow(commitRequest.Spec) - result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, git.AttachCommitRequest{ + attach := git.AttachCommitRequest{ Namespace: commitRequest.Namespace, Name: commitRequest.Name, UID: string(commitRequest.UID), @@ -246,9 +255,33 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques IdleTimeout: window.idleTimeout, MaxDuration: window.maxDuration, CommitEmpty: commitRequest.Spec.WhenNothingToCommit == configbutleraiv1alpha3.NothingToCommitCommitEmpty, - }) + } + if time.Since(commitRequest.CreationTimestamp.Time) >= window.resolveTimeout() { + // Past the bound the controller only withdraws: an attach sent now would sit behind the + // withdraw on the worker's FIFO and could register the request after it was cancelled. + return r.withdrawCommitRequest(ctx, log, req, commitRequest, attribution, attach) + } + return r.attachCommitRequest(ctx, log, req, commitRequest, attribution, attach) +} + +// attachCommitRequest sends the attach and reports the outcome once the worker has one. Until then +// it records the worker's phase and polls; a service error is a not-yet-serviceable worker, so it +// polls on the same terms. +func (r *CommitRequestReconciler) attachCommitRequest( + ctx context.Context, + log logr.Logger, + req ctrl.Request, + commitRequest *configbutleraiv1alpha3.CommitRequest, + attribution commitRequestAttribution, + attach git.AttachCommitRequest, +) (ctrl.Result, error) { + result, resolved, serviceErr := r.Finalizer.ServiceCommitRequest(ctx, attach) if serviceErr != nil || !resolved { - return r.awaitAttachOutcome(ctx, log, req, commitRequest, attribution, window, result.Phase, serviceErr) + if serviceErr != nil { + log.V(1).Info("CommitRequest attach not yet serviceable; will retry", + "name", req.NamespacedName, "err", serviceErr.Error()) + } + return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) } if result.Err != nil { @@ -260,45 +293,69 @@ func (r *CommitRequestReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } -// awaitAttachOutcome handles an attach that has not produced an outcome yet: it keeps polling -// while the request is inside its safety window, and fails closed once that window is spent. A -// service error is a not-yet-serviceable worker, so it polls on the same terms. -func (r *CommitRequestReconciler) awaitAttachOutcome( +// withdrawCommitRequest handles a request past its safety window. A request the worker holds keeps +// polling with no bound of its own: only the worker can say how it ends. Any other request is +// withdrawn, and fails only on the worker's answer, so a request reported failed can never be +// committed afterwards. A withdraw that cannot be serviced yet polls on the same terms. +func (r *CommitRequestReconciler) withdrawCommitRequest( ctx context.Context, log logr.Logger, req ctrl.Request, commitRequest *configbutleraiv1alpha3.CommitRequest, attribution commitRequestAttribution, - window requestWindow, - phase git.CommitRequestPhase, - serviceErr error, + attach git.AttachCommitRequest, ) (ctrl.Result, error) { - if serviceErr != nil { - log.V(1).Info("CommitRequest attach not yet serviceable; will retry", - "name", req.NamespacedName, "err", serviceErr.Error()) + result, resolved, withdrawErr := r.Finalizer.WithdrawCommitRequest(ctx, attach) + if withdrawErr != nil || !resolved { + if withdrawErr != nil { + log.V(1).Info("CommitRequest withdraw not yet serviceable; will retry", + "name", req.NamespacedName, "err", withdrawErr.Error()) + } + // Not answered yet, or the worker holds the request: keep polling. + return r.pollCommitRequest(ctx, commitRequest, attribution, result.Phase) } - if time.Since(commitRequest.CreationTimestamp.Time) < window.resolveTimeout() { - if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: commitRequestPollInterval}, nil + if !errors.Is(result.Err, git.ErrCommitRequestWithdrawn) { + // The worker resolved it on its own before the withdraw: report that outcome, as the + // attach path would have. + r.writeTerminalStatus(ctx, log, commitRequest, result, result.Err, attribution, + resolvedCommitRequestTarget(commitRequest)) + return ctrl.Result{}, nil } - log.Info("CommitRequest did not resolve within the safety window; failing closed", - "name", req.NamespacedName) - // A nil serviceErr means the last attempt DID get the GitTarget and simply had no outcome - // yet, so the identity is known; a non-nil one never resolved it and takes the fallback. + // Withdrawn. A worker's answer, or a GitTarget that was read, carries the branch, and only + // then is the request's identity known. target := commitRequestTarget{} - if serviceErr == nil { + if result.Branch != "" { target = resolvedCommitRequestTarget(commitRequest) } - r.writeTerminalStatus(ctx, log, commitRequest, - git.FinalizeResult{}, errors.New(resolveTimeoutMessage), attribution, target) + message := resolveTimeoutMessage + switch { + case errors.Is(result.Err, git.ErrBranchWorkerStopped): + message = git.ErrBranchWorkerStopped.Error() + case result.Phase == git.PhaseWaitingForWorker: + message = noWorkerTimeoutMessage + } + log.Info("CommitRequest was not acted on within the safety window; withdrawn and failed closed", + "name", req.NamespacedName, "reason", message) + r.writeTerminalStatus(ctx, log, commitRequest, git.FinalizeResult{}, errors.New(message), attribution, target) return ctrl.Result{}, nil } +// pollCommitRequest reports where the worker says the request stands and polls again. +func (r *CommitRequestReconciler) pollCommitRequest( + ctx context.Context, + commitRequest *configbutleraiv1alpha3.CommitRequest, + attribution commitRequestAttribution, + phase git.CommitRequestPhase, +) (ctrl.Result, error) { + if err := r.recordWorkerPhase(ctx, commitRequest, attribution, phase); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: commitRequestPollInterval}, nil +} + // stampFirstSightConditions stamps the still-running conditions the first time a request is seen, // so the object reports its progress (kstatus InProgress) and AuthorAttributed is settled // immediately. A disabled controller returns before this, so it never stamps. diff --git a/internal/controller/commitrequest_controller_unit_test.go b/internal/controller/commitrequest_controller_unit_test.go index 0cb87b3f..5d432bc9 100644 --- a/internal/controller/commitrequest_controller_unit_test.go +++ b/internal/controller/commitrequest_controller_unit_test.go @@ -36,6 +36,25 @@ type fakeFinalizer struct { err error calls []git.AttachCommitRequest + + // The withdraw reply. Left unset, the withdraw answers like the router does before it cancels + // anything: with the worker's outcome, or its phase while unresolved. withdrawPending makes an + // unset reply mean "not answered yet" instead. + withdrawResult git.FinalizeResult + withdrawResolved bool + withdrawErr error + withdrawPending bool + withdrawCalls []git.AttachCommitRequest +} + +func (f *fakeFinalizer) WithdrawCommitRequest( + _ context.Context, attach git.AttachCommitRequest, +) (git.FinalizeResult, bool, error) { + f.withdrawCalls = append(f.withdrawCalls, attach) + if f.withdrawPending || f.withdrawResolved || f.withdrawErr != nil || f.withdrawResult.Phase != "" { + return f.withdrawResult, f.withdrawResolved, f.withdrawErr + } + return f.result, f.resolved, f.err } func (f *fakeFinalizer) ServiceCommitRequest( @@ -65,12 +84,15 @@ func attributedAlice() *fakeAuthorLookup { return &fakeAuthorLookup{author: queue.CommandAuthor{Author: "alice"}, found: true} } +// newCommitRequest builds a request created just now, inside its safety window, as the API server +// would stamp it. pastTheBound ages one beyond it. func newCommitRequest(name string) *configv1alpha3.CommitRequest { return &configv1alpha3.CommitRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - UID: types.UID("uid-" + name), + Name: name, + Namespace: "default", + UID: types.UID("uid-" + name), + CreationTimestamp: metav1.Now(), }, Spec: configv1alpha3.CommitRequestSpec{ GitTargetRef: meta.LocalObjectReference{Name: "team-a-config"}, @@ -79,6 +101,12 @@ func newCommitRequest(name string) *configv1alpha3.CommitRequest { } } +// pastTheBound ages a request beyond its safety window, where the controller only withdraws. +func pastTheBound(cr *configv1alpha3.CommitRequest) *configv1alpha3.CommitRequest { + cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * resolveRequestWindow(cr.Spec).resolveTimeout())) + return cr +} + // withReadyCommitted stamps a terminal Committed CommitRequest (Ready=True). func withReadyCommitted(cr *configv1alpha3.CommitRequest) *configv1alpha3.CommitRequest { apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ @@ -424,22 +452,120 @@ func TestCommitRequestReconcile_ServiceErrorPolls(t *testing.T) { assert.False(t, commitRequestIsTerminal(&got)) } -// Past the resolve safety window an attach the worker never resolved fails closed. +// Past the safety window a request the worker never acted on is withdrawn, and fails closed once +// the worker confirms the withdraw. func TestCommitRequestReconcile_ResolveTimeoutFailsClosed(t *testing.T) { - // Zero CreationTimestamp: far past the resolve bound. - cr := newCommitRequest("save-stuck") + cr := pastTheBound(newCommitRequest("save-stuck")) c := newCommitRequestClient(t, nil, cr) - f := &fakeFinalizer{resolved: false} + f := &fakeFinalizer{ + result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, + withdrawResult: git.FinalizeResult{Branch: "main", Err: git.ErrCommitRequestWithdrawn}, + withdrawResolved: true, + } r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} reconcileCommitRequest(t, r, "save-stuck") + require.Len(t, f.withdrawCalls, 1, "the request is withdrawn before it is failed") + assert.Empty(t, f.calls, "past the bound no attach is sent: it would queue behind the withdraw") got := fetchCommitRequest(t, c, "save-stuck") requireCondition(t, got, ConditionTypeReady, metav1.ConditionFalse, crReasonFinalizeFailed) stalled := requireCondition(t, got, ConditionTypeStalled, metav1.ConditionTrue, crReasonFinalizeFailed) assert.Equal(t, resolveTimeoutMessage, stalled.Message) } +// A request the worker holds is never failed on the controller's clock: a down remote or a parent +// branch recovery can keep it in WaitingForPush far past the bound, and the worker still commits it. +func TestCommitRequestReconcile_AHeldRequestOutlivesTheBound(t *testing.T) { + for _, phase := range []git.CommitRequestPhase{git.PhaseCollectingWindow, git.PhaseWaitingForPush} { + t.Run(string(phase), func(t *testing.T) { + cr := pastTheBound(newCommitRequest("save-held")) + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{result: git.FinalizeResult{Phase: phase}} + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + res := reconcileCommitRequest(t, r, "save-held") + + assert.Equal(t, commitRequestPollInterval, res.RequeueAfter) + assert.Empty(t, f.calls, "past the bound no attach is sent") + assert.Len(t, f.withdrawCalls, 1, "the withdraw finds the request held and changes nothing") + got := fetchCommitRequest(t, c, "save-held") + assert.False(t, commitRequestIsTerminal(&got)) + + // The push lands much later: the request resolves Committed, not FinalizeFailed. + f.result = git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"} + f.resolved = true + reconcileCommitRequest(t, r, "save-held") + got = fetchCommitRequest(t, c, "save-held") + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + }) + } +} + +// Until the worker answers the withdraw the request keeps polling; and when the worker turns out to +// hold it after all (it attached between the poll and the withdraw), nothing fails. +func TestCommitRequestReconcile_AnUnansweredWithdrawKeepsPolling(t *testing.T) { + for name, reply := range map[string]git.FinalizeResult{ + "not handled yet": {}, + "the worker now holds it": {Phase: git.PhaseCollectingWindow}, + } { + t.Run(name, func(t *testing.T) { + cr := pastTheBound(newCommitRequest("save-race")) + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Phase: git.PhaseWaitingForWindow}, + withdrawResult: reply, + withdrawPending: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + res := reconcileCommitRequest(t, r, "save-race") + + assert.Equal(t, commitRequestPollInterval, res.RequeueAfter) + got := fetchCommitRequest(t, c, "save-race") + assert.False(t, commitRequestIsTerminal(&got)) + }) + } +} + +// A worker that resolved the request on its own just before the withdraw wins: its real outcome is +// what the request reports. +func TestCommitRequestReconcile_AWithdrawThatLostTheRaceReportsTheRealOutcome(t *testing.T) { + cr := pastTheBound(newCommitRequest("save-late")) + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + withdrawResult: git.FinalizeResult{Outcome: git.FinalizeCommitted, Commit: "abc", Branch: "main"}, + withdrawResolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + reconcileCommitRequest(t, r, "save-late") + + got := fetchCommitRequest(t, c, "save-late") + requireCondition(t, got, ConditionTypeReady, metav1.ConditionTrue, crReasonCommitted) + assert.Equal(t, "abc", got.Status.Commit) +} + +// A GitTarget that never started a branch worker is named as the cause. +func TestCommitRequestReconcile_NoWorkerIsNamedWhenItFailsClosed(t *testing.T) { + cr := pastTheBound(newCommitRequest("save-no-worker")) + c := newCommitRequestClient(t, nil, cr) + f := &fakeFinalizer{ + result: git.FinalizeResult{Phase: git.PhaseWaitingForWorker}, + withdrawResult: git.FinalizeResult{ + Branch: "main", Phase: git.PhaseWaitingForWorker, Err: git.ErrCommitRequestWithdrawn, + }, + withdrawResolved: true, + } + r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} + + reconcileCommitRequest(t, r, "save-no-worker") + + got := fetchCommitRequest(t, c, "save-no-worker") + stalled := requireCondition(t, got, ConditionTypeStalled, metav1.ConditionTrue, crReasonFinalizeFailed) + assert.Equal(t, noWorkerTimeoutMessage, stalled.Message) +} + func TestCommitRequestReconcile_TerminalShortCircuits(t *testing.T) { cr := withReadyCommitted(newCommitRequest("save-done")) c := newCommitRequestClient(t, nil, cr) diff --git a/internal/controller/commitrequest_outcome_metric_test.go b/internal/controller/commitrequest_outcome_metric_test.go index ac3fc9b7..8f2dec59 100644 --- a/internal/controller/commitrequest_outcome_metric_test.go +++ b/internal/controller/commitrequest_outcome_metric_test.go @@ -112,8 +112,13 @@ func TestCommitRequestMetric_UnresolvableTargetTakesTheFallback(t *testing.T) { // Older than the resolve timeout, so the poll gives up on this pass rather than requeueing. cr.CreationTimestamp = metav1.NewTime(time.Now().Add(-2 * resolveRequestWindow(cr.Spec).resolveTimeout())) c := newCommitRequestClient(t, nil, cr) - // A service error is what EventRouter.ServiceCommitRequest returns when the GitTarget Get fails. - f := &fakeFinalizer{err: errors.New("get GitTarget default/no-such-target: not found")} + // A service error is what EventRouter.ServiceCommitRequest returns when the GitTarget Get fails, + // and a missing GitTarget has no worker to hold the request, so the withdraw resolves at once. + f := &fakeFinalizer{ + err: errors.New("get GitTarget default/no-such-target: not found"), + withdrawResult: git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, + withdrawResolved: true, + } r := &CommitRequestReconciler{Client: c, APIReader: c, Finalizer: f, AuthorLookup: attributedAlice()} reconcileCommitRequest(t, r, "save-bogus") @@ -187,7 +192,7 @@ func TestCommitRequestMetric_EveryOutcomeIsReachableThroughReconcile(t *testing. nil, crOutcomeAlreadyPresent, }, - {"failed", git.FinalizeResult{}, errors.New("attach failed"), crOutcomeFailed}, + {"failed", git.FinalizeResult{Branch: "main", Err: errors.New("commit failed")}, nil, crOutcomeFailed}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 945023cb..5b50e503 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -53,9 +53,12 @@ const ( const ( // GitTargetReasonOK is the healthy reason. It is the shared Succeeded vocabulary rather than // a per-kind spelling; the name is kept for call-site stability. - GitTargetReasonOK = ReasonSucceeded - GitTargetReasonProviderNotFound = "ProviderNotFound" - GitTargetReasonBranchNotAllowed = "BranchNotAllowed" + GitTargetReasonOK = ReasonSucceeded + GitTargetReasonProviderNotFound = "ProviderNotFound" + GitTargetReasonBranchNotAllowed = "BranchNotAllowed" + // GitTargetReasonParentBranchNotFound is the Ready reason for a write branch that cannot be + // created: the remote carries neither it nor the spec.parentBranch it would start from. + GitTargetReasonParentBranchNotFound = "ParentBranchNotFound" GitTargetReasonTargetConflict = "TargetConflict" GitTargetReasonNotChecked = "NotChecked" GitTargetReasonBlocked = "Blocked" @@ -141,6 +144,10 @@ type GitTargetReconciler struct { // value is usable. reconcileRequests reconcileRequestTracker + // snapshotRequests remembers which of a branch worker's snapshot requests were acted on, so a + // missing parent's recovery forces one recheck per request rather than one per reconcile. + snapshotRequests snapshotRequestTracker + // remotePublications is what each GitTarget last wrote to status.remote: when, and about which // repository. It is internal by design; see remotePublicationLedger. remotePublications remotePublicationLedger @@ -181,6 +188,10 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( st := beginStatus(r.Client, r.Recorder, &target) gitPathWasRefused := conditionIsFalse(target.Status.Conditions, GitTargetConditionGitPathAccepted) + // Writes are dropped while the parent is missing, so its return is recovered the way a refused + // folder is: by a recheck that re-derives what the cluster holds. The branch worker decides + // when, and asks; this acts once per request. + parentRecheck, recoveringParent := r.parentRecovery(&target, target.Namespace) providerNS := target.Namespace // One read of the GitProvider for everything below it; see getGitProvider. @@ -266,7 +277,7 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // A standing reconcile request forces the same re-check a refused Git path does: the watch // plane re-anchors the target's streams, which is what makes it re-read the folder rather than // wait for the periodic pass. Taken once per distinct annotation value. - forceRecheck := gitPathWasRefused || r.reconcileRequests.take( + forceRecheck := gitPathWasRefused || parentRecheck || r.reconcileRequests.take( types.NewResourceReference(target.Name, target.Namespace), reconcileRequestedAt(&target)) observed := r.observeDataPlane(&target, sourceProvider, forceRecheck, log) st.setValue(GitTargetConditionStreamsRunning, observed.axes.Streams) @@ -277,7 +288,9 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( rd := newGitTargetReadiness() convergeAsSuspended(rd, &target) - gitTargetReadinessGates(rd, observed, refs.gitProvider, refs.clusterProvider, refs.sourceCluster) + gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS, repoIdentityOf(gitProvider)), + refs.gitProvider, refs.clusterProvider, refs.sourceCluster) + parentRecoveryReadiness(rd, recoveringParent) st.applyReadiness(rd) if err := st.commit(ctx); err != nil { @@ -762,12 +775,14 @@ func (r *GitTargetReconciler) publishReferenceReadiness( func gitTargetReadinessGates( rd *readiness, observed dataPlaneObservation, - provider, clusterProvider, sourceReach conditionValue, + parent, provider, clusterProvider, sourceReach conditionValue, ) { // Terminal, most specific first. Each of these needs a human: the folder holds content the - // operator will not manage, a watch is refused, or what was written no longer matches live. + // operator will not manage, the write branch has no parent to be created from, a watch is + // refused, or what was written no longer matches live. rd.stalledIf(observed.axes.GitPath.Status == metav1.ConditionFalse, observed.axes.GitPath.Reason, observed.axes.GitPath.Message) + rd.stalledIf(parent.Status == metav1.ConditionFalse, parent.Reason, parent.Message) rd.stalledIf(observed.streams.Blocked > 0, observed.streams.Reason, observed.streams.Message) rd.stalledIf(observed.axes.Render.Status == metav1.ConditionFalse, observed.axes.Render.Reason, observed.axes.Render.Message) @@ -803,6 +818,55 @@ func gitTargetReadinessGates( "Stream declaration has not landed yet; the data-plane surface is not observable") } +// parentBranchReadiness reports whether the write branch can be created: False when the last +// look at the remote found neither the write branch nor the parent it would be created from. That +// parent is the configured spec.parentBranch, or, when it is omitted, the remote's default branch: +// a repository that is not empty but whose HEAD names no branch it carries refuses as well, because +// starting there would make an orphan. +// +// It is read from the branch worker's remote observation, so it costs no round trip and holds +// with periodic refresh disabled: every fetch on the write path records the same observation. The +// observation counts only when it was made under the parent configured now, and only for the +// repository the GitProvider names now (a repoint proves nothing about the new one). +func (r *GitTargetReconciler) parentBranchReadiness( + target *configbutleraiv1alpha3.GitTarget, + providerNS string, + repo git.RepoIdentity, +) conditionValue { + ok := conditionValue{Status: metav1.ConditionTrue, Reason: GitTargetReasonOK} + observed, seen := r.observeRemote(target, providerNS) + if !seen || observed.ParentState != git.ParentMissing || + observed.ParentRequested != target.Spec.ParentBranch || differentRepository(observed.Repo, repo) { + return ok + } + return conditionValue{ + Status: metav1.ConditionFalse, + Reason: GitTargetReasonParentBranchNotFound, + Message: parentBranchNotFoundMessage(target, observed.ParentBranch), + } +} + +// parentBranchNotFoundMessage says which parent is missing and what fixes it. +func parentBranchNotFoundMessage(target *configbutleraiv1alpha3.GitTarget, missing string) string { + switch { + case target.Spec.ParentBranch != "": + return fmt.Sprintf( + "Branch '%s' does not exist on the remote, and neither does its parent branch '%s' to create "+ + "it from; nothing is written until one of them exists", + target.Spec.Branch, target.Spec.ParentBranch) + case missing != "": + return fmt.Sprintf( + "Branch '%s' does not exist on the remote, and the remote's default branch (HEAD -> '%s') does "+ + "not exist either; fix the remote's HEAD or set spec.parentBranch. Nothing is written until then", + target.Spec.Branch, missing) + default: + return fmt.Sprintf( + "Branch '%s' does not exist on the remote, and the remote has no default branch to create it "+ + "from; set the remote's HEAD or spec.parentBranch. Nothing is written until then", + target.Spec.Branch) + } +} + // requestRemoteRefresh asks this target's branch worker to re-prove where the branch is, if // nothing has proved it within the interval. // @@ -902,6 +966,7 @@ func (r *GitTargetReconciler) ensureEventStream( providerNS, target.Spec.Branch, repo, + target.Spec.ParentBranch, ) if err != nil { return nil, fmt.Errorf( @@ -1089,13 +1154,6 @@ func (r *GitTargetReconciler) checkForConflicts( target *configbutleraiv1alpha3.GitTarget, providerNS string, ) (bool, string, string, error) { - // A path the writer would reject (absolute, backslashes, ".." traversal) owns - // nothing, so it must neither block others nor be blocked here — its own write - // path fails it. Only well-formed paths participate in overlap detection. - if !git.IsValidTargetPath(target.Spec.Path) { - return false, "", "", nil - } - var allTargets configbutleraiv1alpha3.GitTargetList if err := r.List(ctx, &allTargets); err != nil { return false, "", "", fmt.Errorf("list GitTargets for conflict validation: %w", err) @@ -1109,41 +1167,18 @@ func (r *GitTargetReconciler) checkForConflicts( if existing.Namespace != providerNS || existing.Spec.GitProviderRef.Name != target.Spec.GitProviderRef.Name { continue } - if existing.Spec.Branch != target.Spec.Branch || - !git.IsValidTargetPath(existing.Spec.Path) || - !gitTargetPathsOverlap(target.Spec.Path, existing.Spec.Path) { + if existing.Spec.Branch != target.Spec.Branch { continue } - // Two GitTargets on the same provider+branch whose paths are equal or - // nested fight over which documents each one owns. The later-created - // target loses (ties broken deterministically by identity) so every - // materialized folder keeps exactly one owner. - if gitTargetLosesConflict(target, existing) { - var msg string - if normalizeGitTargetPath(target.Spec.Path) == normalizeGitTargetPath(existing.Spec.Path) { - msg = fmt.Sprintf( - "Conflict detected. Another GitTarget '%s/%s' (created at %s) is already using GitProvider '%s/%s', branch '%s', path '%s'. This GitTarget was created later and will not be processed.", - existing.Namespace, - existing.Name, - existing.CreationTimestamp.Format(time.RFC3339), - providerNS, - target.Spec.GitProviderRef.Name, - target.Spec.Branch, - target.Spec.Path, - ) - } else { - msg = fmt.Sprintf( - "Conflict detected. This GitTarget's path '%s' overlaps the path '%s' of GitTarget '%s/%s' (created at %s) on GitProvider '%s/%s', branch '%s' — one path nests inside the other (sibling paths are allowed). This GitTarget was created later and will not be processed.", - target.Spec.Path, - existing.Spec.Path, - existing.Namespace, - existing.Name, - existing.CreationTimestamp.Format(time.RFC3339), - providerNS, - target.Spec.GitProviderRef.Name, - target.Spec.Branch, - ) - } + // Checked whatever either path looks like: a target with a path its writer will reject + // still wires the shared worker, and would set its parent. + if lost, msg := losesParentBranchConflict(target, existing, providerNS); lost { + return true, msg, GitTargetReasonTargetConflict, nil + } + if existing.Spec.ParentBranch != target.Spec.ParentBranch { + continue // the other target loses, on its own reconcile + } + if lost, msg := losesPathConflict(target, existing, providerNS); lost { return true, msg, GitTargetReasonTargetConflict, nil } } @@ -1151,6 +1186,77 @@ func (r *GitTargetReconciler) checkForConflicts( return false, "", "", nil } +// losesPathConflict reports whether target loses to existing, on the same write branch, because +// their paths are equal or nested. Two such targets fight over which documents each one owns. The +// later-created target loses (ties broken deterministically by identity) so every materialized +// folder keeps exactly one owner. +// +// A path the writer would reject (absolute, backslashes, ".." traversal) owns nothing, so it must +// neither block others nor be blocked here — its own write path fails it. Only well-formed paths +// participate in overlap detection. +func losesPathConflict(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) (bool, string) { + if !git.IsValidTargetPath(target.Spec.Path) || + !git.IsValidTargetPath(existing.Spec.Path) || + !gitTargetPathsOverlap(target.Spec.Path, existing.Spec.Path) || + !gitTargetLosesConflict(target, existing) { + return false, "" + } + if normalizeGitTargetPath(target.Spec.Path) == normalizeGitTargetPath(existing.Spec.Path) { + return true, fmt.Sprintf( + "Conflict detected. Another GitTarget '%s/%s' (created at %s) is already using GitProvider '%s/%s', branch '%s', path '%s'. This GitTarget was created later and will not be processed.", + existing.Namespace, + existing.Name, + existing.CreationTimestamp.Format(time.RFC3339), + providerNS, + target.Spec.GitProviderRef.Name, + target.Spec.Branch, + target.Spec.Path, + ) + } + return true, fmt.Sprintf( + "Conflict detected. This GitTarget's path '%s' overlaps the path '%s' of GitTarget '%s/%s' (created at %s) on GitProvider '%s/%s', branch '%s' — one path nests inside the other (sibling paths are allowed). This GitTarget was created later and will not be processed.", + target.Spec.Path, + existing.Spec.Path, + existing.Namespace, + existing.Name, + existing.CreationTimestamp.Format(time.RFC3339), + providerNS, + target.Spec.GitProviderRef.Name, + target.Spec.Branch, + ) +} + +// losesParentBranchConflict reports whether target loses to existing, on the same write branch, +// because the two name different parent branches. One branch worker owns one checkout per write +// branch, so every target on it has to agree on where the branch starts. Compared as written, with +// omitted as its own value: the answer needs no remote round trip and cannot flip when somebody +// changes the remote's default branch. +func losesParentBranchConflict(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) (bool, string) { + if existing.Spec.ParentBranch == target.Spec.ParentBranch || !gitTargetLosesConflict(target, existing) { + return false, "" + } + return true, parentBranchConflictMessage(target, existing, providerNS) +} + +// parentBranchConflictMessage explains a TargetConflict over spec.parentBranch. +func parentBranchConflictMessage(target, existing *configbutleraiv1alpha3.GitTarget, providerNS string) string { + return fmt.Sprintf( + "Conflict detected. GitTarget '%s/%s' (created at %s) writes GitProvider '%s/%s', branch '%s' with parent "+ + "branch %s, and this GitTarget names parent branch %s. All GitTargets on one branch must name the same "+ + "parent branch. This GitTarget was created later and will not be processed.", + existing.Namespace, existing.Name, existing.CreationTimestamp.Format(time.RFC3339), + providerNS, target.Spec.GitProviderRef.Name, target.Spec.Branch, + describeParentBranch(existing.Spec.ParentBranch), describeParentBranch(target.Spec.ParentBranch), + ) +} + +func describeParentBranch(name string) string { + if name == "" { + return "(omitted: the remote's default branch)" + } + return "'" + name + "'" +} + func (r *GitTargetReconciler) ensureEncryptionSecret( ctx context.Context, target *configbutleraiv1alpha3.GitTarget, @@ -1359,6 +1465,7 @@ func (r *GitTargetReconciler) cleanupDeletedGitTarget( // condition gauge is released on the same terms and for a sharper reason: a condition series // that outlives its object reports Ready=False forever and the alert on it never clears. r.reconcileRequests.forget(gitDest) + r.snapshotRequests.forget(gitDest) // Same terms: the publication ledger is this reconciler's memory of what the object's status // said, and an entry that outlives the object is a rate limit held against a name that may be // recreated tomorrow with nothing published. diff --git a/internal/controller/gittarget_immutability_test.go b/internal/controller/gittarget_immutability_test.go index d73d15f1..5a19011b 100644 --- a/internal/controller/gittarget_immutability_test.go +++ b/internal/controller/gittarget_immutability_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" @@ -75,6 +76,60 @@ var _ = Describe("GitTarget Destination Immutability", func() { }, "spec.gitProviderRef is immutable") }) + // spec.parentBranch decides what history a new write branch gets, so it is fixed at creation + // like the destination, including adding or removing it. It can be omitted, but never empty. + It("rejects changing, adding or removing spec.parentBranch, and an empty one", func() { + ctx := context.Background() + withParent := types.NamespacedName{Name: "parent-branch-target", Namespace: "default"} + withoutParent := types.NamespacedName{Name: "no-parent-branch-target", Namespace: "default"} + + for _, tc := range []struct { + key types.NamespacedName + parent string + }{{withParent, "main"}, {withoutParent, ""}} { + gitTarget := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: tc.key.Name, Namespace: tc.key.Namespace}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "prov-a"}, + Branch: "edits", + Path: "apps-" + tc.key.Name, + ParentBranch: tc.parent, + }, + } + Expect(k8sClient.Create(ctx, gitTarget)).Should(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, gitTarget) }) + } + + expectRejected := func(key types.NamespacedName, parent string) { + Eventually(func(g Gomega) { + current := &configbutleraiv1alpha3.GitTarget{} + g.Expect(k8sClient.Get(ctx, key, current)).To(Succeed()) + current.Spec.ParentBranch = parent + err := k8sClient.Update(ctx, current) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("spec.parentBranch is immutable")) + }, timeout, interval).Should(Succeed()) + } + expectRejected(withParent, "release") // changed + expectRejected(withParent, "") // removed + expectRejected(withoutParent, "release") // added + + empty := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "configbutler.ai/v1alpha3", + "kind": "GitTarget", + "metadata": map[string]interface{}{"name": "empty-parent-target", "namespace": "default"}, + "spec": map[string]interface{}{ + "gitProviderRef": map[string]interface{}{"name": "prov-a"}, + "branch": "edits", + "path": "apps", + "parentBranch": "", + }, + }} + err := k8sClient.Create(ctx, empty) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.parentBranch")) + }) + It("requires a non-empty path: rejects an omitted or empty path but allows an explicit \".\" root", func() { ctx := context.Background() key := types.NamespacedName{Name: "root-policy-target", Namespace: "default"} diff --git a/internal/controller/gittarget_layout_test.go b/internal/controller/gittarget_layout_test.go index 391dde11..bab5a610 100644 --- a/internal/controller/gittarget_layout_test.go +++ b/internal/controller/gittarget_layout_test.go @@ -206,7 +206,7 @@ func TestGitTargetReadiness_StalledFollowsGitPathAccepted(t *testing.T) { }, Render: conditionValue{Status: metav1.ConditionTrue}, }, - }, healthyDependency(), healthyDependency(), healthyDependency()) + }, healthyDependency(), healthyDependency(), healthyDependency(), healthyDependency()) trio := rd.trio() assert.Equal(t, wantStalled.Status, string(trio.Stalled.Status)) diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go new file mode 100644 index 00000000..134526c1 --- /dev/null +++ b/internal/controller/gittarget_parent_branch_test.go @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/types" + "github.com/ConfigButler/gitops-reverser/internal/watch" +) + +func parentBranchTarget(name, branch, path, parent string, created time.Time) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "shop", CreationTimestamp: metav1.NewTime(created), + }, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "repo1"}, + Branch: branch, + Path: path, + ParentBranch: parent, + }, + } +} + +func parentBranchScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configbutleraiv1alpha3.AddToScheme(scheme)) + return scheme +} + +// TestCheckForConflicts_ParentBranch: one write branch, one checkout, one parent. Two targets on the +// same branch that name different parents conflict even on sibling paths, and the later one loses. +// Omitted is its own value, so it conflicts with an explicit "main". +func TestCheckForConflicts_ParentBranch(t *testing.T) { + earlier := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + later := earlier.Add(time.Hour) + + cases := []struct { + name string + existing *configbutleraiv1alpha3.GitTarget + target *configbutleraiv1alpha3.GitTarget + wantConflict bool + }{ + { + name: "omitted versus main conflicts", + existing: parentBranchTarget("first", "edits", "apps/a", "", earlier), + target: parentBranchTarget("second", "edits", "apps/b", "main", later), + wantConflict: true, + }, + { + name: "the earlier target wins", + existing: parentBranchTarget("second", "edits", "apps/b", "main", later), + target: parentBranchTarget("first", "edits", "apps/a", "", earlier), + }, + { + name: "the same parent on sibling paths is fine", + existing: parentBranchTarget("first", "edits", "apps/a", "release", earlier), + target: parentBranchTarget("second", "edits", "apps/b", "release", later), + }, + { + name: "another write branch is another checkout", + existing: parentBranchTarget("first", "edits", "apps/a", "", earlier), + target: parentBranchTarget("second", "other", "apps/b", "release", later), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + k8sClient := fake.NewClientBuilder().WithScheme(parentBranchScheme(t)).WithObjects(tc.existing).Build() + r := &GitTargetReconciler{Client: k8sClient} + + conflict, msg, reason, err := r.checkForConflicts(context.Background(), tc.target, "shop") + + require.NoError(t, err) + assert.Equal(t, tc.wantConflict, conflict, msg) + if tc.wantConflict { + assert.Equal(t, GitTargetReasonTargetConflict, reason) + assert.Contains(t, msg, "(omitted: the remote's default branch)") + assert.Contains(t, msg, "'main'") + } + }) + } +} + +func startParentBranchWorkers(t *testing.T) (*GitTargetReconciler, *git.WorkerManager) { + t.Helper() + k8sClient := fake.NewClientBuilder().WithScheme(parentBranchScheme(t)).Build() + workers := git.NewWorkerManager(k8sClient, logr.Discard(), git.BranchWorkerLimits{}, + types.SensitiveResourcePolicy{}) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = workers.Start(ctx) }() + time.Sleep(100 * time.Millisecond) // Start installs the context EnsureWorker starts workers under + router := watch.NewEventRouter(workers, nil, k8sClient, logr.Discard()) + return &GitTargetReconciler{Client: k8sClient, WorkerManager: workers, EventRouter: router}, workers +} + +// TestEnsureEventStream_AParentBranchChangeKeepsTheWorker: changing spec.parentBranch abandons +// nothing written, so it is not a repoint. The worker stays and re-reads the remote. +func TestEnsureEventStream_AParentBranchChangeKeepsTheWorker(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "", time.Now()) + + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + before, ok := workers.GetWorkerForTarget("repo1", "shop", "edits") + require.True(t, ok) + before.SeedRemoteObservationForTest("", time.Now(), "Fetch") + + target.Spec.ParentBranch = "release" + _, err = r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + + after, ok := workers.GetWorkerForTarget("repo1", "shop", "edits") + require.True(t, ok) + assert.Same(t, before, after, "the worker and its clone are kept") + assert.Equal(t, "release", after.ParentBranch()) + assert.False(t, after.BaseTrustedForTest(), "and its checkout of the old parent is no longer trusted") +} + +// TestParentBranchReadiness: a fetch that found neither the write branch nor its configured parent +// stalls the GitTarget as ParentBranchNotFound, for that parent only. +func TestParentBranchReadiness(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) + + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status, + "nothing has looked yet") + + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + worker.RecordMissingParentForTest("release") + + got := r.parentBranchReadiness(target, "shop", repo) + assert.Equal(t, metav1.ConditionFalse, got.Status) + assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) + assert.Contains(t, got.Message, "'release'") + + rd := newGitTargetReadiness() + gitTargetReadinessGates(rd, dataPlaneObservation{axes: gitTargetAxes{ + Streams: conditionValue{Status: metav1.ConditionTrue}, + GitPath: conditionValue{Status: metav1.ConditionTrue}, + Render: conditionValue{Status: metav1.ConditionTrue}, + }}, got, healthyDependency(), healthyDependency(), healthyDependency()) + trio := rd.trio() + assert.Equal(t, metav1.ConditionTrue, trio.Stalled.Status, "it needs somebody to create the branch") + assert.Equal(t, GitTargetReasonParentBranchNotFound, trio.Ready.Reason) + + target.Spec.ParentBranch = "other" + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status, + "the observation is about another parent") + target.Spec.ParentBranch = "" + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status) +} + +// TestParentBranchReadiness_AnOmittedParentCanStall: with spec.parentBranch omitted, a remote that +// is not empty but whose default branch does not resolve stalls the target too, with a message +// that names what to fix. An observation made under a configured parent says nothing about it. +func TestParentBranchReadiness_AnOmittedParentCanStall(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "", time.Now()) + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + + worker.RecordUnresolvedDefaultBranchForTest("master") + got := r.parentBranchReadiness(target, "shop", repo) + assert.Equal(t, metav1.ConditionFalse, got.Status) + assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) + assert.Contains(t, got.Message, "HEAD -> 'master'") + assert.Contains(t, got.Message, "spec.parentBranch") + + worker.RecordUnresolvedDefaultBranchForTest("") + got = r.parentBranchReadiness(target, "shop", repo) + assert.Equal(t, GitTargetReasonParentBranchNotFound, got.Reason) + assert.Contains(t, got.Message, "no default branch") + + worker.RecordMissingParentForTest("release") + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", repo).Status, + "an observation made under a configured parent is about another configuration") +} + +// TestParentRecovery_ASnapshotRequestForcesOneRecheck: the controller acts on the branch worker's +// snapshot requests, not on the Ready reason. A request forces exactly one recheck; requests made +// entirely between two reconciles are still acted on; an unchanged sequence forces nothing. +func TestParentRecovery_ASnapshotRequestForcesOneRecheck(t *testing.T) { + r, workers := startParentBranchWorkers(t) + repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} + target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) + _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + ref := types.NewResourceReference(target.Name, target.Namespace) + + recheck, _ := r.parentRecovery(target, "shop") + assert.False(t, recheck, "nothing asked") + + worker.BumpSnapshotRequestForTest(ref) + recheck, _ = r.parentRecovery(target, "shop") + assert.True(t, recheck) + recheck, _ = r.parentRecovery(target, "shop") + assert.False(t, recheck, "exactly one recheck per request") + + worker.BumpSnapshotRequestForTest(ref) + worker.BumpSnapshotRequestForTest(ref) // a whole episode between two reconciles + recheck, _ = r.parentRecovery(target, "shop") + assert.True(t, recheck, "still acted on") + + for range 10 { + recheck, _ = r.parentRecovery(target, "shop") + assert.False(t, recheck) + } + + r.snapshotRequests.forget(ref) + recheck, _ = r.parentRecovery(target, "shop") + assert.True(t, recheck, "a forgotten target acts on the standing request again") +} + +func TestParentRecoveryReadiness_IsProgressNotAStall(t *testing.T) { + rd := newGitTargetReadiness() + parentRecoveryReadiness(rd, true) + trio := rd.trio() + assert.Equal(t, metav1.ConditionFalse, trio.Ready.Status) + assert.Equal(t, GitTargetReasonRecoveringParentBranch, trio.Ready.Reason) + assert.Equal(t, metav1.ConditionTrue, trio.Reconciling.Status) + assert.NotEqual(t, metav1.ConditionTrue, trio.Stalled.Status) + assert.Equal(t, RequeueStreamSettleInterval, gitTargetRequeue(rd), "it keeps the not-Ready requeue") + + rd = newGitTargetReadiness() + parentRecoveryReadiness(rd, false) + assert.True(t, rd.converged()) +} + +// TestCheckForConflicts_ParentBranchAppliesToInvalidPaths: a target whose path its writer will +// reject still wires the shared worker, so it must not escape the parent check in either role and +// repoint the healthy sibling's parent. +func TestCheckForConflicts_ParentBranchAppliesToInvalidPaths(t *testing.T) { + earlier := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + healthy := parentBranchTarget("first", "edits", "apps/a", "main", earlier) + invalid := parentBranchTarget("second", "edits", "../bad", "release", earlier.Add(time.Hour)) + + r, workers := startParentBranchWorkers(t) + require.NoError(t, r.Client.Create(context.Background(), healthy)) + _, err := r.ensureEventStream(context.Background(), healthy, "shop", firstRepo, logr.Discard()) + require.NoError(t, err) + + conflict, _, reason, err := r.checkForConflicts(context.Background(), invalid, "shop") + require.NoError(t, err) + assert.True(t, conflict, "an invalid path must not bypass the parent conflict") + assert.Equal(t, GitTargetReasonTargetConflict, reason) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + assert.Equal(t, "main", worker.ParentBranch()) + + // And as the earlier object, the invalid target still wins the parent. + k8sClient := fake.NewClientBuilder().WithScheme(parentBranchScheme(t)). + WithObjects(parentBranchTarget("older", "edits", "../bad", "release", earlier)).Build() + conflict, _, _, err = (&GitTargetReconciler{Client: k8sClient}).checkForConflicts( + context.Background(), parentBranchTarget("newer", "edits", "apps/a", "main", earlier.Add(time.Hour)), "shop") + require.NoError(t, err) + assert.True(t, conflict) +} + +// TestParentBranchReadiness_IgnoresAnotherRepository: the manager keeps a branch's observation across +// a repoint, and a missing parent in the old repository says nothing about the new one. +func TestParentBranchReadiness_IgnoresAnotherRepository(t *testing.T) { + r, workers := startParentBranchWorkers(t) + target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) + _, err := r.ensureEventStream(context.Background(), target, "shop", firstRepo, logr.Discard()) + require.NoError(t, err) + worker, _ := workers.GetWorkerForTarget("repo1", "shop", "edits") + worker.RecordMissingParentForTest("release") + require.Equal(t, metav1.ConditionFalse, r.parentBranchReadiness(target, "shop", firstRepo).Status) + + _, err = r.ensureEventStream(context.Background(), target, "shop", secondRepo, logr.Discard()) + require.NoError(t, err) + assert.Equal(t, metav1.ConditionTrue, r.parentBranchReadiness(target, "shop", secondRepo).Status, + "the old repository's missing parent is not evidence about the replacement") +} + +// TestParentStatusOf: status.remote.parent is present only while the write branch is absent and +// a parent was named, and a moved parent is news even when the write branch is still absent. +func TestParentStatusOf(t *testing.T) { + absent := git.RemoteObservation{ParentState: git.ParentFound, ParentBranch: "main", ParentCommit: "aaaa"} + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{ + State: configbutleraiv1alpha3.GitTargetParentFound, Branch: "main", Commit: "aaaa", + }, parentStatusOf(absent)) + assert.Nil(t, parentStatusOf(git.RemoteObservation{Commit: "bbbb"}), + "an existing write branch does not follow its parent") + assert.Nil(t, parentStatusOf(git.RemoteObservation{}), "an observation that names no parent state") + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{State: configbutleraiv1alpha3.GitTargetParentUnborn}, + parentStatusOf(git.RemoteObservation{ParentState: git.ParentUnborn}), "an empty repository") + assert.Equal(t, &configbutleraiv1alpha3.GitTargetParentStatus{ + State: configbutleraiv1alpha3.GitTargetParentMissing, Branch: "release", + }, parentStatusOf(git.RemoteObservation{ParentState: git.ParentMissing, ParentBranch: "release"}), + "a missing parent is named with no commit") + + at := metav1.NewTime(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + published := &configbutleraiv1alpha3.GitTargetRemoteStatus{LastVerifiedAt: &at, Parent: parentStatusOf(absent)} + moved := published.DeepCopy() + moved.Parent.Commit = "cccc" + assert.True(t, remoteStatusIsNews(published, moved), "the parent moved") + assert.False(t, remoteStatusIsNews(published, published.DeepCopy())) +} diff --git a/internal/controller/gittarget_parent_recovery.go b/internal/controller/gittarget_parent_recovery.go new file mode 100644 index 00000000..33266bf4 --- /dev/null +++ b/internal/controller/gittarget_parent_recovery.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "sync" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// GitTargetReasonRecoveringParentBranch is the Ready reason while the parent branch has been found +// again but the branch worker has not yet published the work it held back. It is progress, not a +// stall, and keeps the not-Ready requeue cadence so the controller acts on the worker's snapshot +// requests within one requeue. +const GitTargetReasonRecoveringParentBranch = "RecoveringParentBranch" + +// snapshotRequestTracker remembers, per GitTarget, the last snapshot request of its branch worker +// that this reconciler acted on. A missing-parent episode that starts and ends between two +// reconciles still leaves the worker's sequence higher, so it is never lost; an unchanged sequence +// forces nothing. Its memory is empty after a restart, which is harmless: every stream starts with +// an initial list, which is itself the snapshot. Its zero value is usable. +type snapshotRequestTracker struct { + mu sync.Mutex + acted map[string]uint64 +} + +// take reports whether seq is a request not acted on yet, and records it as acted on. +func (t *snapshotRequestTracker) take(ref types.ResourceReference, seq uint64) bool { + t.mu.Lock() + defer t.mu.Unlock() + if seq == 0 || t.acted[ref.Key()] >= seq { + return false + } + if t.acted == nil { + t.acted = map[string]uint64{} + } + t.acted[ref.Key()] = seq + return true +} + +// forget drops a deleted object's record. +func (t *snapshotRequestTracker) forget(ref types.ResourceReference) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.acted, ref.Key()) +} + +// parentRecovery reads the branch worker's obligation for work a missing parent held back. The +// first result is whether it asks for a new snapshot of this target (true once per rise of its +// sequence), the second whether the parent is back while work is still owed. It reads memory only, +// and costs no connection. +func (r *GitTargetReconciler) parentRecovery( + target *configbutleraiv1alpha3.GitTarget, + providerNS string, +) (bool, bool) { + if r.WorkerManager == nil { + return false, false + } + worker, ok := r.WorkerManager.GetWorkerForTarget(target.Spec.GitProviderRef.Name, providerNS, target.Spec.Branch) + if !ok { + return false, false + } + ref := types.NewResourceReference(target.Name, target.Namespace) + open, found := worker.ParentRecovery() + return r.snapshotRequests.take(ref, worker.SnapshotRequestSeq(ref)), open && found +} + +// parentRecoveryReadiness is the progress gate for a target whose parent is back but whose +// held-back work is not published yet. +func parentRecoveryReadiness(rd *readiness, recovering bool) { + rd.progressingIf(recovering, metav1.ConditionFalse, GitTargetReasonRecoveringParentBranch, + "The parent branch is on the remote again; the branch worker is publishing the work it held back") +} diff --git a/internal/controller/gittarget_remote.go b/internal/controller/gittarget_remote.go index 397f4d8b..ee9e45f0 100644 --- a/internal/controller/gittarget_remote.go +++ b/internal/controller/gittarget_remote.go @@ -148,6 +148,7 @@ func (r *GitTargetReconciler) publishRemote( Commit: observed.Commit, LastVerifiedAt: &metav1.Time{Time: observed.At}, VerifiedBy: string(observed.By), + Parent: parentStatusOf(observed), } if !remoteStatusIsNews(target.Status.Remote, next) { return @@ -203,10 +204,30 @@ func remoteStatusIsNews(published, next *configbutleraiv1alpha3.GitTargetRemoteS if published == nil || published.LastVerifiedAt == nil || next.LastVerifiedAt == nil { return true } - if published.Commit != next.Commit { + if published.Commit != next.Commit || !equalParentStatus(published.Parent, next.Parent) { return true } // metav1.Time is second-granular on the wire, so this compares against what was PUBLISHED: a // sub-second difference the API server would round away is not movement. return next.LastVerifiedAt.Time.After(published.LastVerifiedAt.Time) } + +// parentStatusOf is the parent stanza of status.remote: present only while the observation found +// the write branch absent, which is when it says what the branch would be created from. +func parentStatusOf(observed git.RemoteObservation) *configbutleraiv1alpha3.GitTargetParentStatus { + if observed.Commit != "" || observed.ParentState == "" { + return nil + } + return &configbutleraiv1alpha3.GitTargetParentStatus{ + State: configbutleraiv1alpha3.GitTargetParentState(observed.ParentState), + Branch: observed.ParentBranch, + Commit: observed.ParentCommit, + } +} + +func equalParentStatus(a, b *configbutleraiv1alpha3.GitTargetParentStatus) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} diff --git a/internal/controller/gittarget_status_test.go b/internal/controller/gittarget_status_test.go index 714abe0c..e5a1d656 100644 --- a/internal/controller/gittarget_status_test.go +++ b/internal/controller/gittarget_status_test.go @@ -164,7 +164,7 @@ func TestGitTargetReadinessGates(t *testing.T) { st.setValue(GitTargetConditionRenderMatchesLive, observed.axes.Render) rd := newGitTargetReadiness() - gitTargetReadinessGates(rd, observed, tt.provider, tt.clusterProvider, tt.sourceReach) + gitTargetReadinessGates(rd, observed, conditionValue{}, tt.provider, tt.clusterProvider, tt.sourceReach) st.applyReadiness(rd) ready := findCondition(target.Status.Conditions, GitTargetConditionReady) diff --git a/internal/controller/gittarget_worker_cleanup_test.go b/internal/controller/gittarget_worker_cleanup_test.go index a354e6b8..a79fd7ec 100644 --- a/internal/controller/gittarget_worker_cleanup_test.go +++ b/internal/controller/gittarget_worker_cleanup_test.go @@ -45,7 +45,7 @@ func TestCleanupDeletedGitTarget_StopsAWorkerNothingNeedsAnyMore(t *testing.T) { time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() _, exists := workers.GetWorkerForTarget("repo1", "shop", "main") @@ -81,7 +81,7 @@ func TestCleanupDeletedGitTarget_LeavesAWorkerAnotherGitTargetStillUses(t *testi go func() { _ = workers.Start(ctx) }() time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() @@ -125,7 +125,7 @@ func TestCleanupDeletedGitTarget_RetriesWhenTheSweepCannotRead(t *testing.T) { go func() { _ = workers.Start(ctx) }() time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() @@ -158,7 +158,7 @@ func TestHandleFetchError_RequeuesAFailedCleanup(t *testing.T) { go func() { _ = workers.Start(ctx) }() time.Sleep(100 * time.Millisecond) func() { - err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}) + err := workers.EnsureWorker(ctx, "repo1", "shop", "main", git.RepoIdentity{URL: "file:///repo1.git"}, "") require.NoError(t, err) }() diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index aaa5f840..cd69863a 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -56,7 +56,14 @@ var ( //nolint:gochecknoglobals fetchRemoteBranchHashFn = fetchRemoteBranchHash //nolint:gochecknoglobals - syncToRemoteFn = syncToRemote + syncToRemoteFn = syncToRemoteFrom + // beforePushAdmission runs right before a push cycle compares its root's parent generation + // with the current one. Nil in production; a test changes the parent here to land a change + // just before admission. + //nolint:gochecknoglobals + beforePushAdmission func() + //nolint:gochecknoglobals + advertiseRemoteBranchFn = advertiseRemoteBranch ) // BranchWorker processes events for a single (GitProvider, Branch) combination. @@ -155,6 +162,11 @@ type BranchWorker struct { // block on the lock, and send after Stop had finished draining, into a queue with no loop left // to read it — and still answer "accepted". stoppingState bool + // exitedState is set once the event loop has returned, for whatever reason, and the worker + // has settled every CommitRequest it knew. From then on nothing can act on a request, so a + // withdraw is answered at once instead of being queued for a loop that is gone. Guarded by + // pendingResyncsMu, like stoppingState. + exitedState bool // lastObservation is what this worker last PROVED about the target branch on the remote, and // when. See RemoteObservation: it replaces a fetch-only trio (branchExists, lastCommitSHA, @@ -227,6 +239,30 @@ type BranchWorker struct { // Cleared after a successful push. Protected by repoMu. pushCycleRootHash plumbing.Hash + // newBranchParent and newBranchParentHash are the parent the local write branch was created + // from, while the remote does not carry the write branch. They let a cycle that finds HEAD + // already on the local write branch still root itself on the parent: a no-op cycle (a resync + // or a live write with nothing to commit) creates the local branch and leaves HEAD on it, and + // local HEAD alone cannot say that the branch is absent on the remote. + // + // Cleared as soon as the remote is known to carry the branch: a fetch that found it, or a push + // that created or confirmed it. Protected by repoMu. + newBranchParent plumbing.ReferenceName + newBranchParentHash plumbing.Hash + + // parentState is the GitTarget's spec.parentBranch, with a generation that SetParentBranch + // raises on every change. It is set by the reconcile and read by the event loop, hence atomic. + // See parentConfig. + parentState atomic.Pointer[parentConfig] + // baseParentGen is the generation of the parent configuration the checkout was last proven + // against: the snapshot whose fetch produced it, or the current one once the checkout is known + // not to depend on the parent (the write branch exists). Trust is scoped to it, so a push or a + // fetch that finishes after a change can never vouch for the new configuration. + baseParentGen atomic.Uint64 + // pushCycleRootGen is the parent generation pushCycleRootBranch was chosen under. The push + // admission check compares it with the current one. Protected by repoMu. + pushCycleRootGen uint64 + // firsts surfaces the first successful commit and push at default verbosity. firsts branchWorkerLogFirsts @@ -279,6 +315,26 @@ type BranchWorker struct { // crPhases holds where each unresolved CommitRequest stands, for the controller to report // while it waits. Guarded by crOutcomesMu; the event loop is the only writer. crPhases map[commitRequestID]CommitRequestPhase + // crQueuedAttaches counts, per request, the attaches on the FIFO that the loop has not handled + // yet. An outcome is not collected while any are queued: the outcome is what turns a late attach + // into a no-op, so collecting it first would let that attach register a withdrawn request + // again, however long the outcome had been kept. Guarded by crOutcomesMu. + crQueuedAttaches map[commitRequestID]int + // crOwners is the manager's table of which worker holds each request. See commitRequestOwners. + // Set by the WorkerManager before Start; nil for a worker built on its own. + crOwners *commitRequestOwners + + // clock is the time source of the parent-recovery schedule; nil is time.Now. See now(). + clock func() time.Time + // parentRecoveryOpen, parentRecoveryFound and parentProbeHold mirror the event loop's + // recovery latch for readers outside it. See parent_recovery.go. + parentRecoveryOpen atomic.Bool + parentRecoveryFound atomic.Bool + parentProbeHold atomic.Pointer[parentProbeHold] + // snapshotRequests counts, per GitTarget, the snapshots the worker has asked for. See + // SnapshotRequestSeq. + snapshotRequestsMu sync.Mutex + snapshotRequests map[itypes.ResourceReference]uint64 } // branchWorkerLogFirsts logs the first successful commit and push of a worker's @@ -466,6 +522,7 @@ func (w *BranchWorker) Start(parentCtx context.Context) error { go func() { defer w.wg.Done() w.processEvents() + w.settleCommitRequestsAfterExit() }() return nil @@ -509,6 +566,35 @@ func (w *BranchWorker) EnqueueRequest(request *WriteRequest) { w.enqueueRequest(request) } +// EnqueueWithdraw asks the loop to cancel a CommitRequest it has not acted on. It never blocks: a +// full queue or a stopping worker drops it, and the controller sends it again on its next poll. A +// worker whose loop has exited answers at once, because nothing is left that could act on it. +func (w *BranchWorker) EnqueueWithdraw(req *AttachCommitRequest) { + if req == nil { + return + } + w.pendingResyncsMu.Lock() + if w.exitedState { + w.pendingResyncsMu.Unlock() + w.answerWithdrawAfterExit(req.id()) + return + } + defer w.pendingResyncsMu.Unlock() + if w.stoppingLocked() { + // Still draining: its last push may yet carry the request, so only the exited worker can + // answer. The controller asks again on its next poll. + return + } + w.inflightItems.Add(1) + select { + case w.eventQueue <- WorkItem{Withdraw: req}: + default: + w.inflightItems.Add(-1) + w.Log.V(1).Info("Queue full, CommitRequest withdraw dropped (the controller re-sends)", + "request", req.Namespace+"/"+req.Name) + } +} + // EnqueueAttach adds a CommitRequest attach to this worker's queue. Riding the // same queue as resource events is what makes it process in audit order, after // every earlier write. The attach is fire-and-forget: the controller polls the @@ -534,7 +620,18 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { w.Log.V(1).Info("Worker is stopping, CommitRequest attach refused (the controller re-sends)") return } + id := req.id() + if !w.crOwners.claim(id, w) { + // The router routes to the owner, so only a request that moved workers between two polls + // gets here. The owner answers for it; registering it twice could commit it twice. + w.pendingResyncsMu.Unlock() + w.inflightItems.Add(-1) + w.Log.Info("CommitRequest attach refused: another branch worker holds the request", + "request", req.Namespace+"/"+req.Name) + return + } w.markResyncTailForTargetLocked(req.GitTargetNamespace, req.GitTargetName) + w.countQueuedAttach(id) select { case w.eventQueue <- WorkItem{Attach: req}: w.pendingResyncsMu.Unlock() @@ -548,6 +645,7 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { // time, so an enqueue is visible to the next scrape whether or not the // loop has woken to notice it. default: + w.uncountQueuedAttach(id) w.pendingResyncsMu.Unlock() w.inflightItems.Add(-1) w.recordQueueDrop(queueDropAttach) @@ -912,7 +1010,7 @@ func (w *BranchWorker) prepareBootstrapRepository( repoPath := w.repoPath() w.recordFetch(fetchReasonBootstrap) - pullReport, err := PrepareBranch(ctx, w.repo.URL, repoPath, w.Branch, auth) + pullReport, err := w.prepareBranch(ctx, repoPath, auth) if err != nil { return "", fmt.Errorf("failed to prepare repository: %w", err) } @@ -1018,6 +1116,10 @@ type branchWorkerEventLoop struct { // that no window reaches still resolves on time. An attached request needs no timer of its own: // its window's timers close it. attachTimer *time.Timer + + // recovery is the obligation to write what a missing parent branch held back, and the probe + // that drives it. See parent_recovery.go. + recovery parentRecovery } // commitWindowDefaults are a GitTarget's commit window timers when it declares none. @@ -1063,6 +1165,9 @@ func (l *branchWorkerEventLoop) run() { case <-refusalC: l.refusalTimer = nil l.flushPendingRefusalTouch() + case <-l.recoveryTimerC(): + l.recovery.timer = nil + l.runParentProbe() } // After every wake: bind any waiting CommitRequest to an open window, // resolve any whose attach deadline has passed, and re-arm the deadline timer. @@ -1126,6 +1231,11 @@ func (l *branchWorkerEventLoop) handleQueueItem(item WorkItem) { return } + if item.Withdraw != nil { + l.handleWithdrawCommitRequest(item.Withdraw) + return + } + if item.Resync != nil { // Take the CURRENT request for this scope: a newer one may have replaced // it while this marker waited in the FIFO. Running it here keeps the @@ -1273,6 +1383,7 @@ func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) l.w.Log.Error(err, "Failed to recover a dirty worktree; dropping atomic request", "events", len(request.Events)) + l.noteParentUnavailable(err, atomicScope(request)) return } @@ -1300,6 +1411,7 @@ func (l *branchWorkerEventLoop) handleAtomicRequest(request *WriteRequest) { l.w.recordCommitFailure(commitFailureKindAtomic, commitFailureError) l.w.Log.Error(err, "Atomic commit failed; dropping request", "events", len(request.Events)) } + l.noteParentUnavailable(err, atomicScope(request)) return } @@ -1342,15 +1454,22 @@ func (l *branchWorkerEventLoop) failUnpushedCommitRequests() { } } -// drainUnhandledQueueItems clears items the exiting loop will never handle. Each was counted into -// inflightItems at enqueue and decremented only after handling, so without this the final depth -// would stay non-zero forever: the loop has stopped, so nothing republishes a corrected value. A -// buffered CommitRequest attach is dropped; the controller re-sends on its next poll. -func (l *branchWorkerEventLoop) drainUnhandledQueueItems() { +// drainUnhandledQueueItems clears items the exiting loop will never handle. See drainQueue. +func (l *branchWorkerEventLoop) drainUnhandledQueueItems() { l.w.drainQueue() } + +// drainQueue clears items no loop will handle. Each was counted into inflightItems at enqueue and +// decremented only after handling, so without this the final depth would stay non-zero forever: +// the loop has stopped, so nothing republishes a corrected value. A buffered CommitRequest attach +// is dropped, and the request given back unless the worker knows it otherwise; the controller +// re-sends on its next poll, to whichever worker holds it then. +func (w *BranchWorker) drainQueue() { for { select { - case <-l.w.eventQueue: - l.w.inflightItems.Add(-1) + case item := <-w.eventQueue: + if item.Attach != nil { + w.uncountQueuedAttach(item.Attach.id()) + } + w.inflightItems.Add(-1) default: return } @@ -1398,15 +1517,24 @@ func (l *branchWorkerEventLoop) recoverRetainedWrites() error { l.w.markReplayComplete() return nil } - dirty, needsReplay := l.w.worktreeDirty(), l.w.replayRequired() - if !dirty && !needsReplay { + // A parent change is read from the generations, not taken from a flag: a rebuild that fails + // leaves them apart, so the next attempt rebuilds again. + dirty, needsReplay, parentChanged := l.w.worktreeDirty(), l.w.replayRequired(), l.w.rootParentStale() + if !dirty && !needsReplay && !parentChanged { return nil } + // The rebuild fetches, and a parent known to be missing is not fetched again before its probe + // is due: every commit path calls this, so without the check each write arriving in the + // meantime would cost a connection. + if l.w.awaitingParentProbe() { + return errAwaitingParentProbe + } l.w.Log.Info("Rebuilding retained writes onto the remote tip", "pendingWrites", len(l.pendingWrites), "worktreeDirty", dirty, - "replayRequired", needsReplay) + "replayRequired", needsReplay, + "parentBranchChanged", parentChanged) // fetchReasonRecovery, the same series the no-retained-writes case records in // ensureBaseForCycle: this is one event, and which half of it an operator sees must not depend // on whether a push happened to be in cooldown at the time. @@ -1491,7 +1619,7 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali l.w.recordCommitFailure(commitFailureKindWindow, commitFailureError) l.w.Log.Error(err, "Failed to recover a dirty worktree; dropping open window", "reason", string(reason), "windowTarget", windowTarget) - l.dropOpenWindow(pendingCR, err) + l.dropFailedWindow(err, pendingCR, err, windowScopes(targetNamespace, targetName, events)) return false } @@ -1540,7 +1668,8 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali "windowTarget", windowTarget, "events", len(events)) } - l.dropOpenWindow(pendingCR, fmt.Errorf("commit failed: %w", err)) + l.dropFailedWindow(err, pendingCR, fmt.Errorf("commit failed: %w", err), + windowScopes(targetNamespace, targetName, events)) return false } @@ -1580,6 +1709,15 @@ func (l *branchWorkerEventLoop) finalizeOpenWindowWithReason(reason windowFinali // dropOpenWindow discards a window whose finalize failed, resolving any attached // CommitRequest as Failed so the controller does not poll forever. +// dropFailedWindow drops a window whose write failed with err, and when the parent branch is what +// failed it, remembers the scopes the dropped writes belong to so they are recovered. +func (l *branchWorkerEventLoop) dropFailedWindow( + err error, pendingCR *commitRequestID, cause error, scopes []recoveryScope, +) { + l.noteParentUnavailable(err, scopes...) + l.dropOpenWindow(pendingCR, cause) +} + func (l *branchWorkerEventLoop) dropOpenWindow(pendingCR *commitRequestID, cause error) { l.openWindow = nil l.windowBytes = 0 @@ -1626,6 +1764,11 @@ func (l *branchWorkerEventLoop) pushPending() { l.stopPushTimer() return } + if l.w.awaitingParentProbe() { + // The parent is known missing and the probe is not due: it publishes these when it is. + l.stopPushTimer() + return + } // A reset may have discarded the local commits behind these writes while the replay that // rebuilds them did not finish. Pushing now would find the branch already at the remote tip, @@ -1635,10 +1778,12 @@ func (l *branchWorkerEventLoop) pushPending() { l.w.Log.Error(err, "Cannot publish until the retained writes are rebuilt; keeping them", "pendingWrites", len(l.pendingWrites)) l.stopPushTimer() + l.noteParentUnavailable(err) return } if err := l.w.pushPendingCommits(l.pendingWrites); err != nil { + l.noteParentUnavailable(err) l.w.Log.Error(err, "Push failed; pending writes retained for retry", "pendingWrites", len(l.pendingWrites)) // Leave pendingWrites in place; do NOT advance lastPushAt — the @@ -1656,6 +1801,7 @@ func (l *branchWorkerEventLoop) pushPending() { l.pendingWritesBytes = 0 l.lastPushAt = time.Now() l.stopPushTimer() + l.noteRecoveryPublished() } // resolvePushedCommitRequests settles every CommitRequest carried by a just-pushed write, now @@ -1726,6 +1872,10 @@ func (l *branchWorkerEventLoop) stopTimers() { l.stopPushTimer() l.stopAttachTimer() l.stopRefusalTimer() + if l.recovery.timer != nil { + l.recovery.timer.Stop() + l.recovery.timer = nil + } } func (l *branchWorkerEventLoop) stopRefusalTimer() { @@ -1740,7 +1890,13 @@ func (l *branchWorkerEventLoop) stopRefusalTimer() { // // The failure direction is the safe one. A stale false costs one fetch. A stale true is caught by // the compare-and-swap on the next push, which is the mechanism the write path already relies on. -func (w *BranchWorker) baseTrusted() bool { return w.baseTrustedState.Load() } +// +// Trust is scoped to the parent configuration: a checkout proven under an older one is not +// trusted, whatever the flag says, because the flag can be set by a push or a fetch that started +// before the change. +func (w *BranchWorker) baseTrusted() bool { + return w.baseTrustedState.Load() && w.baseParentGen.Load() == w.parentSnapshot().gen +} // worktreeDirty reports whether the worktree may hold a partial write. // @@ -1826,9 +1982,16 @@ func (w *BranchWorker) ensureBaseForCycle( repoPath string, hasPendingCommits bool, ) error { - if hasPendingCommits || (w.baseTrusted() && !w.worktreeDirty()) { + if hasPendingCommits { return nil } + w.restampParentGenIfIndependent() + if w.baseTrusted() && !w.worktreeDirty() { + return nil + } + if w.awaitingParentProbe() { + return errAwaitingParentProbe + } // Resolve credentials only on the branch that touches the remote. A cycle planning on a // trusted base never reads the credentials Secret at all, which is a saved API GET per cycle @@ -1847,7 +2010,7 @@ func (w *BranchWorker) ensureBaseForCycle( } w.recordFetch(reason) - pullReport, err := PrepareBranch(w.ctx, w.repo.URL, repoPath, w.Branch, auth) + pullReport, err := w.prepareBranch(w.ctx, repoPath, auth) if err != nil { return fmt.Errorf("prepare repository: %w", err) } @@ -1896,6 +2059,7 @@ func (w *BranchWorker) commitPendingWrites(pendingWrites []PendingWrite, hasPend if !hasPendingCommits { w.pushCycleRootBranch = baseBranch w.pushCycleRootHash = baseHash + w.pushCycleRootGen = w.baseParentGen.Load() } commitsCreated, err := w.executePendingWrites(w.ctx, repo, pendingWrites) @@ -1965,15 +2129,32 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { } const maxRetries = 3 - rootHash := w.pushCycleRootHash var lastErr error for range maxRetries { + // Re-read every attempt: a replay below re-roots the cycle. + rootHash := w.pushCycleRootHash rootBranch := w.pushCycleRootBranch if rootBranch == "" { rootBranch = plumbing.NewBranchReferenceName(w.Branch) } + // The admission boundary for a parent change: compared here, under repoMu and right before + // the push, so a change made after the cycle was planned never leaves on the old root. A + // change after this point is in flight, and the push is not recalled. + if beforePushAdmission != nil { + beforePushAdmission() + } + if !w.admitPushRoot(rootBranch) { + w.Log.Info("Parent branch changed since this cycle was planned; rebuilding before the push", + "branch", w.Branch, "parentBranch", w.ParentBranch()) + w.recordFetch(fetchReasonRecovery) + if err := w.replayOntoRemote(repo, pendingWrites, auth); err != nil { + return err + } + continue + } + outcome, err := pushAtomicFn(w.ctx, repo, rootHash, rootBranch, auth) if err == nil { // A push that returns without an error is an OBSERVATION of the remote, on the @@ -1989,15 +2170,8 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { w.setBaseTrusted(true) // The half a fetch-only record could not carry: on an active branch the push is the // event that MOVES the revision. - revision := "" - if !outcome.Head.IsZero() { - revision = outcome.Head.String() - } - w.recordRemoteObservation(revision, ObservedByPush) - w.Log.V(1).Info("Remote observed by push", - "branch", w.Branch, "outcome", string(outcome.Kind), "head", revision) - w.pushCycleRootBranch = "" - w.pushCycleRootHash = plumbing.ZeroHash + w.recordPushObservation(outcome, rootBranch, rootHash) + w.endPushCycle(outcome.Kind) w.firsts.push.Do(func() { w.Log.Info("First push to remote completed", "branch", w.Branch, @@ -2017,34 +2191,95 @@ func (w *BranchWorker) runPushCycle(pendingWrites []PendingWrite) error { return err } w.recordPushRetry(pushRetryRemoteMoved) - - // Marked BEFORE the reset, not after it. A reset moves the branch ref and then rewrites - // the worktree, so it can fail with the ref already moved and the commits behind these - // writes already unreachable. Marking afterwards misses exactly that case, and the cost of - // marking too eagerly is one fetch on the next cycle if the reset turns out to have - // changed nothing. - w.markReplayRequired() w.recordFetch(fetchReasonContention) - pullReport, syncErr := syncToRemoteFn(w.ctx, repo, plumbing.NewBranchReferenceName(w.Branch), auth) - if syncErr != nil { - w.invalidateBase("sync during replay failed") - return fmt.Errorf("sync remote during replay: %w", syncErr) - } - w.updateBranchMetadataFromPullReport(pullReport) - - rootBranch, rootHash, err = w.rebuildPendingWrites(repo, pendingWrites) - if err != nil { - w.invalidateBase("replay rebuild failed") + if err := w.replayOntoRemote(repo, pendingWrites, auth); err != nil { return err } - w.pushCycleRootBranch = rootBranch - w.pushCycleRootHash = rootHash - w.markReplayComplete() } + if lastErr == nil { + return fmt.Errorf("push not admitted after %d attempts: the parent branch kept changing", maxRetries) + } return fmt.Errorf("push failed after %d attempts: %w", maxRetries, lastErr) } +// replayOntoRemote syncs to the remote, rebuilds the pending writes on what it found, and makes +// that the cycle's root. The caller owns repoMu and has recorded the fetch. +func (w *BranchWorker) replayOntoRemote( + repo *gogit.Repository, + pendingWrites []PendingWrite, + auth []gitclient.Option, +) error { + // Marked BEFORE the reset, not after it. A reset moves the branch ref and then rewrites + // the worktree, so it can fail with the ref already moved and the commits behind these + // writes already unreachable. Marking afterwards misses exactly that case, and the cost of + // marking too eagerly is one fetch on the next cycle if the reset turns out to have + // changed nothing. + w.markReplayRequired() + pullReport, err := w.syncToRemote(w.ctx, repo, auth) + if err != nil { + w.invalidateBase("sync during replay failed") + return fmt.Errorf("sync remote during replay: %w", err) + } + w.updateBranchMetadataFromPullReport(pullReport) + + rootBranch, rootHash, err := w.rebuildPendingWrites(repo, pendingWrites) + if err != nil { + w.invalidateBase("replay rebuild failed") + return err + } + w.pushCycleRootBranch = rootBranch + w.pushCycleRootHash = rootHash + w.pushCycleRootGen = w.baseParentGen.Load() + w.markReplayComplete() + return nil +} + +// admitPushRoot reports whether a push rooted on rootBranch may leave under the current parent +// configuration: the root was chosen under it, or the root is the write branch itself, which a +// parent change cannot move. In that second case it re-stamps the root, at no cost. The caller +// owns repoMu. +func (w *BranchWorker) admitPushRoot(rootBranch plumbing.ReferenceName) bool { + gen := w.parentSnapshot().gen + if w.pushCycleRootGen == gen { + return true + } + if rootBranch != plumbing.NewBranchReferenceName(w.Branch) || !w.writeBranchProvenPresent() { + return false + } + w.pushCycleRootGen = gen + w.restampParentGenIfIndependent() + return true +} + +// writeBranchProvenPresent reports whether the last observation of the remote found the write +// branch. A parent change cannot alter a cycle rooted on an existing write branch, and the push's +// compare-and-swap still guards the branch if the observation went stale. +func (w *BranchWorker) writeBranchProvenPresent() bool { + observed := w.lastObservation.Load() + return observed != nil && observed.Commit != "" +} + +// restampParentGenIfIndependent carries trust across a parent change when the checkout does not +// depend on the parent: the write branch exists, and HEAD is not a local branch still rooted on a +// parent. It costs no connection, which is what keeps an existing write branch's cycles free. +func (w *BranchWorker) restampParentGenIfIndependent() { + if w.newBranchParent == "" && w.writeBranchProvenPresent() { + w.baseParentGen.Store(w.parentSnapshot().gen) + } +} + +// rootParentStale reports whether the retained cycle's root was chosen under an older parent +// configuration, and the root depends on the parent. +func (w *BranchWorker) rootParentStale() bool { + w.repoMu.Lock() + defer w.repoMu.Unlock() + if w.pushCycleRootGen == w.parentSnapshot().gen { + return false + } + return w.pushCycleRootBranch != plumbing.NewBranchReferenceName(w.Branch) || !w.writeBranchProvenPresent() +} + // remoteMovedDuringPush reports whether a failed push was contention — the remote branch no // longer sitting at the hash our commits were based on — which is the only push failure a replay // can fix. @@ -2063,7 +2298,26 @@ func (w *BranchWorker) remoteMovedDuringPush( rootHash plumbing.Hash, auth []gitclient.Option, ) bool { + // A push that creates the write branch is rooted on its parent, so a rejection naming the + // write branch itself is not about the root: somebody created the branch since we based our + // work. That is contention all the same, and the replay builds on their commit. + var moved *RemoteMovedError + if errors.As(pushErr, &moved) && moved.Branch != rootBranch && + moved.Branch == plumbing.NewBranchReferenceName(w.Branch) { + return true + } + // An empty repository gained refs since the cycle was planned on it. The replay's fetch + // resolves the new default branch, or refuses, and the rebuild re-plans from the retained + // writes instead of reusing the orphan commits. + var notEmpty *RepositoryNotEmptyError + if errors.As(pushErr, ¬Empty) { + return true + } + remoteHash, known := advertisedRootHash(pushErr, rootBranch) + if !known && rootBranch != plumbing.NewBranchReferenceName(w.Branch) { + return w.newBranchMovedDuringPush(rootBranch, rootHash, auth) + } if !known { var fetchErr error w.recordFetch(fetchReasonPushFailureProbe) @@ -2076,6 +2330,29 @@ func (w *BranchWorker) remoteMovedDuringPush( return remoteHash != rootHash } +// newBranchMovedDuringPush is the fallback for a push that was creating the write branch and failed +// without an advertisement to read: a dropped connection, or the server refusing an upload whose +// Old was zero because somebody created the branch after our advertisement. Probing the root alone +// would find the parent unmoved and conclude nothing happened, and the work would never replay. So +// it reads both refs from one advertisement, and a write branch that is now there is contention, +// exactly like the typed rejection naming it. It runs only on this exceptional path. +func (w *BranchWorker) newBranchMovedDuringPush( + rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, auth []gitclient.Option, +) bool { + advertisement, err := advertiseRemoteBranchFn(w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), + rootBranch.Short(), auth) + if err != nil { + w.invalidateBase("remote-state probe failed") + return false + } + if !advertisement.branch.IsZero() { + w.Log.Info("The write branch was created by somebody else during our push; replaying onto it", + "branch", w.Branch) + return true + } + return advertisement.parent != rootHash +} + // advertisedRootHash reads the remote's own answer out of a failed push, when the push got one. // // A compare-and-swap rejection is the remote telling us where the branch is: validatePushState @@ -2166,7 +2443,7 @@ func (w *BranchWorker) refreshRemoteAndRebuildPendingWrites( // false, which is precisely the state no other flag describes. w.markReplayRequired() w.recordFetch(reason) - pullReport, err := syncToRemoteFn(ctx, repo, plumbing.NewBranchReferenceName(w.Branch), auth) + pullReport, err := w.syncToRemote(ctx, repo, auth) if err != nil { w.invalidateBase("sync before replay failed") return fmt.Errorf("sync remote before replay: %w", err) @@ -2180,6 +2457,7 @@ func (w *BranchWorker) refreshRemoteAndRebuildPendingWrites( } w.pushCycleRootBranch = rootBranch w.pushCycleRootHash = rootHash + w.pushCycleRootGen = w.baseParentGen.Load() w.markReplayComplete() return nil } @@ -2300,11 +2578,139 @@ func (w *BranchWorker) ensureWriteBranch(repo *gogit.Repository) (plumbing.Refer if err := switchOrCreateBranch(repo, targetBranch, w.Log, w.Branch, baseHash); err != nil { return "", plumbing.ZeroHash, err } + // HEAD was on another branch, which happens only when a fetch found the write branch + // absent and checked out its parent instead. Remember it, so the next cycle on this local + // branch is still rooted on the parent. + w.newBranchParent, w.newBranchParentHash = baseBranch, baseHash + return baseBranch, baseHash, nil + } + + // Already on the local write branch, with nothing committed on top of the parent it was + // created from: the remote still does not carry it, so the parent remains the root. Only while + // the parent configuration is the one that checkout was made under: after a change, the + // remembered parent is the old one, and rooting on it would let the push create the branch + // there. The root then names the local branch, which the admission check refuses. + if w.newBranchParent != "" && baseHash == w.newBranchParentHash && + w.baseParentGen.Load() == w.parentSnapshot().gen { + return w.newBranchParent, w.newBranchParentHash, nil } return baseBranch, baseHash, nil } +// recordPushObservation records what a push that returned without an error observed. +func (w *BranchWorker) recordPushObservation( + outcome PushOutcome, rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, +) { + revision := "" + if !outcome.Head.IsZero() { + revision = outcome.Head.String() + } + if outcome.Kind == PushNoBranch && rootBranch != plumbing.NewBranchReferenceName(w.Branch) { + // Settled without creating the branch, after confirming its parent against this + // advertisement: the root IS the parent's tip. + w.recordAbsentBranch(rootBranch.Short(), rootHash.String(), ObservedByPush) + } else { + w.recordRemoteObservation(revision, ObservedByPush) + } + w.Log.V(1).Info("Remote observed by push", + "branch", w.Branch, "outcome", string(outcome.Kind), "head", revision) +} + +// endPushCycle clears the cycle's root after a push the remote answered. Only a push that left the +// branch absent keeps the parent it would be created from. +func (w *BranchWorker) endPushCycle(kind PushOutcomeKind) { + w.pushCycleRootBranch = "" + w.pushCycleRootHash = plumbing.ZeroHash + if kind != PushNoBranch { + w.clearNewBranchParent() + } +} + +// parentConfig is one immutable snapshot of the parent configuration. Every operation that +// consults the parent loads it once, at its start, and uses that name and generation throughout: +// nothing re-reads the current name after a network call returns, so an old lookup can never +// report a result about a newly configured parent. +type parentConfig struct { + // name is spec.parentBranch; empty means the remote's default branch. + name string + // gen rises on every change. Zero is the configuration a worker starts with. + gen uint64 +} + +// parentSnapshot returns the current parent configuration. +func (w *BranchWorker) parentSnapshot() parentConfig { + if p := w.parentState.Load(); p != nil { + return *p + } + return parentConfig{} +} + +// ParentBranch is the configured parent branch, or empty for the remote's default branch. +func (w *BranchWorker) ParentBranch() string { return w.parentSnapshot().name } + +// SetParentBranch configures the parent branch, and reports whether the value changed. A change +// raises the generation, which is what every later decision compares against: trust earned under +// the old one no longer counts, a cycle planned under it is rebuilt before its push is admitted, +// and retained writes are replayed on the new parent. The worker and its clone are kept, and an +// existing write branch, which a parent cannot move, keeps its trust at no cost. +// spec.parentBranch is immutable, so a change reaches a live worker only when the targets on its +// branch were replaced. +func (w *BranchWorker) SetParentBranch(name string) bool { + current := w.parentSnapshot() + if current.name == name { + return false + } + w.parentState.Store(&parentConfig{name: name, gen: current.gen + 1}) + w.Log.V(1).Info("Parent branch changed", "branch", w.Branch, "parentBranch", name) + return true +} + +// prepareBranch is PrepareBranch for this worker's branch and parent. A configured parent the +// remote does not carry is recorded as an observation, because it is one: the advertisement said +// so, and the GitTarget reports it as ParentBranchNotFound. +func (w *BranchWorker) prepareBranch( + ctx context.Context, repoPath string, auth []gitclient.Option, +) (*PullReport, error) { + parent := w.parentSnapshot() + report, err := PrepareBranchFrom(ctx, w.repo.URL, repoPath, w.Branch, parent.name, auth) + w.noteFetchParent(parent, err) + return report, err +} + +// syncToRemote is syncToRemoteFn for this worker's branch and parent; see prepareBranch. +func (w *BranchWorker) syncToRemote( + ctx context.Context, repo *gogit.Repository, auth []gitclient.Option, +) (*PullReport, error) { + parent := w.parentSnapshot() + report, err := syncToRemoteFn(ctx, repo, plumbing.NewBranchReferenceName(w.Branch), parent.name, auth) + w.noteFetchParent(parent, err) + return report, err +} + +// noteFetchParent records what a fetch under one parent snapshot proved: on success, that the +// checkout now matches that configuration; on a missing parent, the observation, naming the +// parent the fetch actually asked for. +func (w *BranchWorker) noteFetchParent(parent parentConfig, err error) { + if err == nil { + w.baseParentGen.Store(parent.gen) + return + } + var unresolved *DefaultBranchUnresolvedError + switch { + case errors.As(err, &unresolved): + w.recordMissingParent(parent.name, unresolved.Head) + case errors.Is(err, ErrParentBranchNotFound): + w.recordMissingParent(parent.name, parent.name) + } +} + +// clearNewBranchParent forgets the parent once the remote carries the write branch. +func (w *BranchWorker) clearNewBranchParent() { + w.newBranchParent = "" + w.newBranchParentHash = plumbing.ZeroHash +} + // Push cycle outcomes and retry reasons. // // A push CYCLE is one call to pushPendingCommits, replay retries included, and it ends exactly @@ -2664,10 +3070,44 @@ func (w *BranchWorker) LastRemoteObservation() (RemoteObservation, bool) { // accepted — because both prove the same kind of fact. An error path must NOT call it: a push // that died mid-upload or an advertisement that never arrived observed nothing, and recording a // guess there is how a stale revision reaches status. +// +// An empty revision is a branch the remote does not carry, observed without a parent to name: an +// empty repository, which leaves the first commit unborn. func (w *BranchWorker) recordRemoteObservation(revision string, by ObservationSource) { + if revision == "" { + w.recordAbsentBranch("", "", by) + return + } + w.publishObservation(RemoteObservation{Commit: revision, At: time.Now(), By: by, Repo: w.repo}) +} + +// recordMissingParent records that the write branch is absent and so is the parent it would be +// created from. requested is spec.parentBranch as the observing operation read it (empty for the +// remote's default branch); branch is the parent that is missing: the configured one, or the +// branch an unresolved HEAD names. Only a look at the remote can prove it, so it is by Fetch. +func (w *BranchWorker) recordMissingParent(requested, branch string) { + w.publishObservation(RemoteObservation{ + At: time.Now(), By: ObservedByFetch, Repo: w.repo, + ParentState: ParentMissing, ParentRequested: requested, ParentBranch: branch, + }) +} + +// recordAbsentBranch records that the write branch is absent, and where the parent it would be +// created from is. An empty parent is an empty repository, which has none. +func (w *BranchWorker) recordAbsentBranch(parentBranch, parentCommit string, by ObservationSource) { + state := ParentFound + if parentBranch == "" { + state, parentCommit = ParentUnborn, "" + } + w.publishObservation(RemoteObservation{ + At: time.Now(), By: by, Repo: w.repo, + ParentState: state, ParentBranch: parentBranch, ParentCommit: parentCommit, + }) +} + +func (w *BranchWorker) publishObservation(observed RemoteObservation) { // Stamped with the repository it is about, so that whoever publishes it can tell whether it // still describes the repository the GitTarget points at. See RemoteObservation.Repo. - observed := RemoteObservation{Commit: revision, At: time.Now(), By: by, Repo: w.repo} w.lastObservation.Store(&observed) // Delivered here, at the one point where an observation is made, rather than by each caller // against whichever GitTargets it happened to be holding. The fact is about the BRANCH, so it @@ -2705,7 +3145,7 @@ func (w *BranchWorker) syncWithRemote(ctx context.Context, reason string) error // PrepareBranch handles both initial and update cases w.recordFetch(reason) - report, err := PrepareBranch(ctx, w.repo.URL, repoPath, w.Branch, auth) + report, err := w.prepareBranch(ctx, repoPath, auth) if err != nil { return fmt.Errorf("failed to sync with remote: %w", err) } @@ -2740,7 +3180,7 @@ func (w *BranchWorker) ensureRepositoryInitialized(ctx context.Context) error { } // Use new PrepareBranch abstraction - pullReport, err := PrepareBranch(ctx, w.repo.URL, repoPath, w.Branch, auth) + pullReport, err := w.prepareBranch(ctx, repoPath, auth) if err != nil { return fmt.Errorf("failed to prepare repository: %w", err) } @@ -2767,11 +3207,11 @@ func (w *BranchWorker) shouldReadRetainedLocalRepository(repoPath string) bool { func (w *BranchWorker) updateBranchMetadataFromPullReport(report *PullReport) { // A branch the remote does not have is recorded as an observation with no revision, which is // the honest answer rather than a missing one: a git branch without a commit does not exist. - revision := "" if report.ExistsOnRemote { - revision = report.HEAD.Sha + w.recordRemoteObservation(report.HEAD.Sha, ObservedByFetch) + } else { + w.recordAbsentBranch(report.ParentBranch, report.HEAD.Sha, ObservedByFetch) } - w.recordRemoteObservation(revision, ObservedByFetch) // This is the ONE place trust can be gained by a fetch, because it is the one place a fetch // is known to have been followed by a reset: every call site is a PrepareBranch or a @@ -2786,12 +3226,17 @@ func (w *BranchWorker) updateBranchMetadataFromPullReport(report *PullReport) { // based on the default branch (or an empty one) IS that state. There is nothing on the remote // left to learn, so a fetch per cycle could only ever return the same answer. // - // What makes it safe is the same thing that makes it safe everywhere else: the - // compare-and-swap. A push onto a branch we believe is absent declares Old = zero, which the - // server rejects if somebody has since created it, and the rejection invalidates the base and - // fetches. So the cost of being wrong is one rejection, not a bad write. + // What makes it safe is the push. For a branch we believe absent, the push is rooted on the + // parent the branch is created from, and validatePushState refuses it if that parent moved or + // if somebody has since created the branch; either rejection invalidates the base and replays + // onto what the advertisement showed. The push then declares Old = zero, so the server also + // refuses a branch created after the advertisement. So the cost of being wrong is one + // rejection, not a bad write. w.setBaseTrusted(true) w.markWorktreeClean() + if report.ExistsOnRemote { + w.clearNewBranchParent() + } // Log if this was an unborn branch if report.HEAD.Unborn { @@ -2824,5 +3269,18 @@ func (w *BranchWorker) SeedRemoteObservationForTest(revision string, at time.Tim w.lastObservation.Store(&observed) } +// RecordMissingParentForTest records the observation a fetch makes when neither the write branch +// nor its configured parent exists, for tests in another package. +func (w *BranchWorker) RecordMissingParentForTest(parent string) { + w.recordMissingParent(parent, parent) +} + +// RecordUnresolvedDefaultBranchForTest records the observation a fetch makes when the write branch +// is absent, no parent is configured, and the remote's HEAD names head, which it does not carry +// (empty: no HEAD at all), for tests in another package. +func (w *BranchWorker) RecordUnresolvedDefaultBranchForTest(head string) { + w.recordMissingParent("", head) +} + // BaseTrustedForTest exposes the flag to tests in another package. func (w *BranchWorker) BaseTrustedForTest() bool { return w.baseTrusted() } diff --git a/internal/git/branch_worker_base_trust_test.go b/internal/git/branch_worker_base_trust_test.go index 437efe52..bcb9bd21 100644 --- a/internal/git/branch_worker_base_trust_test.go +++ b/internal/git/branch_worker_base_trust_test.go @@ -188,7 +188,7 @@ func TestBaseTrust_LostWhenSomebodySaysTheRemoteMoved(t *testing.T) { syncFailed := errors.New("sync failed") original := syncToRemoteFn syncToRemoteFn = func( - _ context.Context, _ *gogit.Repository, _ plumbing.ReferenceName, _ []gitclient.Option, + _ context.Context, _ *gogit.Repository, _ plumbing.ReferenceName, _ string, _ []gitclient.Option, ) (*PullReport, error) { return nil, syncFailed } diff --git a/internal/git/branch_worker_empty_repo_test.go b/internal/git/branch_worker_empty_repo_test.go new file mode 100644 index 00000000..36bc17fd --- /dev/null +++ b/internal/git/branch_worker_empty_repo_test.go @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// An empty repository is one whose advertisement carries no hash refs at all. These pin what the +// worker does at the edges of that definition, against canonical git over HTTP: a repository that +// stops being empty while work is planned on it (§4.2), and one that is not empty but whose +// default branch does not resolve (§4.3). Neither may ever produce an orphan branch. + +import ( + "os/exec" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// realServerNewBranch is a worker for the absent write branch `feature`, against a real server. +type realServerNewBranch struct { + *ledgerFixture + + loop *branchWorkerEventLoop +} + +func newRealServerNewBranch(t *testing.T, slug string, seed func(repoDir string)) *realServerNewBranch { + t.Helper() + f := newLedgerFixtureOnBranch(t, slug, false, "feature") + if seed != nil { + seed(f.repoDir) + } + f.worker.mapper = configMapMapper() + createGitTarget(t, f.worker, newBranchTarget, newBranchFolder, + &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneAlways}) + loop := newBranchWorkerEventLoop(f.worker, 0) + t.Cleanup(loop.stopTimers) + return &realServerNewBranch{ledgerFixture: f, loop: loop} +} + +func gitIn(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput() + require.NoError(t, err, "git %v: %s", args, out) + return strings.TrimSpace(string(out)) +} + +// ref is where a branch is on the remote, or zero when the remote does not carry it. +func (f *realServerNewBranch) ref(name string) plumbing.Hash { + f.t.Helper() + repo, err := gogit.PlainOpen(f.repoDir) + require.NoError(f.t, err) + ref, err := repo.Reference(plumbing.NewBranchReferenceName(name), true) + if err != nil { + return plumbing.ZeroHash + } + return ref.Hash() +} + +func (f *realServerNewBranch) parentOf(commit plumbing.Hash) []plumbing.Hash { + f.t.Helper() + repo, err := gogit.PlainOpen(f.repoDir) + require.NoError(f.t, err) + c, err := repo.CommitObject(commit) + require.NoError(f.t, err) + return c.ParentHashes +} + +func (f *realServerNewBranch) observed() RemoteObservation { + f.t.Helper() + o, known := f.worker.LastRemoteObservation() + require.True(f.t, known) + return o +} + +// tagOnly gives the remote a tag on a commit no branch carries: not empty, and no default branch. +func tagOnly(t *testing.T, repoDir string) { + t.Helper() + tree := gitIn(t, repoDir, "mktree") + commit := gitIn(t, repoDir, "-c", "user.name=Outside", "-c", "user.email=o@example.com", + "commit-tree", tree, "-m", "tagged") + gitIn(t, repoDir, "tag", "v1", commit) +} + +// branchOnDisk creates a branch with one README commit on the bare repository directly, which works +// whatever the repository holds already. +func branchOnDisk(t *testing.T, repoDir, branch, content string) plumbing.Hash { + t.Helper() + cmd := exec.Command("git", "-C", repoDir, "hash-object", "-w", "--stdin") + cmd.Stdin = strings.NewReader(content) + out, err := cmd.Output() + require.NoError(t, err) + mk := exec.Command("git", "-C", repoDir, "mktree") + mk.Stdin = strings.NewReader("100644 blob " + strings.TrimSpace(string(out)) + "\tREADME.md\n") + tree, err := mk.Output() + require.NoError(t, err) + commit := gitIn(t, repoDir, "-c", "user.name=Outside", "-c", "user.email=o@example.com", + "commit-tree", strings.TrimSpace(string(tree)), "-m", "outside") + gitIn(t, repoDir, "update-ref", "refs/heads/"+branch, commit) + return plumbing.NewHash(commit) +} + +// standbyRequest waits for a window and closes it at once: the write it attaches to is committed +// and, in a cooldown, retained. +func standbyRequest() *AttachCommitRequest { + req := attachReq("alice", time.Hour) + req.GitTargetName = newBranchTarget + req.MaxDuration = 0 + return req +} + +// §4.2 test 1. The work was planned on an empty repository; main appears before publication. The +// push must not create feature as an orphan: it is refused at the advertisement, and the replay +// builds on main. +func TestBranchWorker_AnEmptyRepositoryThatGainsABranchIsNotOrphaned(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newRealServerNewBranch(t, "empty-gains-main", nil) + + f.loop.lastPushAt = time.Now() // cooldown: the write is retained + req := standbyRequest() + serviceAttach(f.loop, req) + liveWrite(f.loop, "cm1") + require.Len(t, f.loop.pendingWrites, 1) + contention := fetchCount(t, reader, f.worker, fetchReasonContention) + + main := simulateClientCommitOnDisk(t, f.repoDir, "main", "README.md", "main appeared\n") + f.loop.pushPending() + + tip := f.ref("feature") + require.False(t, tip.IsZero(), "the write was published") + assert.Equal(t, []plumbing.Hash{main}, f.parentOf(tip), "on main's tip, not as a root commit") + assert.NotEmpty(t, gitIn(t, f.repoDir, "merge-base", "feature", "main")) + assert.Contains(t, gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "feature"), cmFile("cm1")) + assert.Equal(t, contention+1, fetchCount(t, reader, f.worker, fetchReasonContention), "exactly one replay") + res, resolved := outcome(t, f.worker) + require.True(t, resolved) + assert.Equal(t, FinalizeCommitted, res.Outcome) +} + +// §4.2 test 2. The same with a write that changes nothing: the branch stays absent. +func TestBranchWorker_AnEmptyRepositoryThatGainsABranchKeepsANoOpAbsent(t *testing.T) { + f := newRealServerNewBranch(t, "empty-gains-main-noop", nil) + + f.loop.lastPushAt = time.Now() + deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) + deleted.Operation = "DELETE" + deleted.Object = nil + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) + require.Len(t, f.loop.pendingWrites, 1) + + simulateClientCommitOnDisk(t, f.repoDir, "main", "README.md", "main appeared\n") + f.loop.pushPending() + + assert.Empty(t, f.loop.pendingWrites, "settled") + assert.True(t, f.ref("feature").IsZero(), "nothing to publish: the branch stays absent") +} + +// §4.2 test 3. Only a tag appears: not empty, and no default branch to start from. Nothing is +// orphaned, the work stays retained and the parent is reported missing, until a branch exists. +func TestBranchWorker_AnEmptyRepositoryThatGainsOnlyATagWaits(t *testing.T) { + f := newRealServerNewBranch(t, "empty-gains-tag", nil) + + f.loop.lastPushAt = time.Now() + liveWrite(f.loop, "cm1") + require.Len(t, f.loop.pendingWrites, 1) + + tagOnly(t, f.repoDir) + f.loop.pushPending() + + assert.True(t, f.ref("feature").IsZero(), "no orphan") + assert.Len(t, f.loop.pendingWrites, 1, "the work is retained") + o := f.observed() + assert.Equal(t, ParentMissing, o.ParentState) + assert.Empty(t, o.ParentRequested) + + main := branchOnDisk(t, f.repoDir, "main", "a branch at last\n") + pastProbeDeadline(f.worker) + f.loop.runParentProbe() + tip := f.ref("feature") + require.False(t, tip.IsZero()) + assert.Equal(t, []plumbing.Hash{main}, f.parentOf(tip)) +} + +// §4.2 test 4 and §4.3 test 4. A truly empty repository is still Unborn, and the first push still +// creates the root commit. +func TestBranchWorker_ATrulyEmptyRepositoryStillBootstraps(t *testing.T) { + f := newRealServerNewBranch(t, "empty-throughout", nil) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + assert.Equal(t, ParentUnborn, f.observed().ParentState) + + liveWrite(f.loop, "cm1") + + tip := f.ref("feature") + require.False(t, tip.IsZero()) + assert.Empty(t, f.parentOf(tip), "a root commit") +} + +// §4.3 test 1. HEAD names a branch the repository does not carry. That is not an empty repository, +// so nothing may start an orphan there; the fetch and the refresh both report the parent missing. +func TestBranchWorker_ADanglingRemoteHeadIsMissingNotUnborn(t *testing.T) { + f := newRealServerNewBranch(t, "dangling-head", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "symbolic-ref", "HEAD", "refs/heads/master") + }) + + err := f.worker.ensureRepositoryInitialized(f.worker.ctx) + require.ErrorIs(t, err, ErrDefaultBranchUnresolved) + o := f.observed() + assert.Equal(t, ParentMissing, o.ParentState, "the fetch: not Unborn") + assert.Empty(t, o.ParentRequested) + // Canonical git advertises neither a HEAD that resolves to nothing nor its symref, so the name + // it dangles at (master) never reaches the client: the status says the remote has no default + // branch. A server that does send the symref gets the name in the message. + assert.Empty(t, o.ParentBranch) + + provider, err := f.worker.getGitProvider(f.worker.ctx) + require.NoError(t, err) + require.NoError(t, f.loop.refreshFromRemote(provider)) + assert.Equal(t, ParentMissing, f.observed().ParentState, "the refresh agrees") + + liveWrite(f.loop, "cm1") + assert.True(t, f.ref("feature").IsZero(), "no orphan") + assert.True(t, f.loop.lastPushAt.IsZero(), "nothing was pushed") + + // §4.3 test 3: repairing HEAD is enough. + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") + pastProbeDeadline(f.worker) + liveWrite(f.loop, "cm2") + tip := f.ref("feature") + require.False(t, tip.IsZero()) + assert.Equal(t, []plumbing.Hash{f.ref("main")}, f.parentOf(tip)) +} + +// §4.3 test 2. A detached HEAD is resolved the way go-git resolves it, which this pins: master when +// it is at HEAD's commit, otherwise the alphabetically first branch that is. +func TestBranchWorker_ADetachedRemoteHeadFollowsGoGitsResolution(t *testing.T) { + for name, tc := range map[string]struct { + branches []string + want string + }{ + "master wins": {branches: []string{"trunk", "master"}, want: "master"}, + "first by name": {branches: []string{"trunk", "develop"}, want: "develop"}, + "one branch at HEAD": {branches: []string{"trunk"}, want: "trunk"}, + } { + t.Run(name, func(t *testing.T) { + f := newRealServerNewBranch(t, "detached-"+strings.ReplaceAll(name, " ", "-"), func(repoDir string) { + first := simulateClientCommitOnDisk(t, repoDir, tc.branches[0], "README.md", "x\n") + for _, b := range tc.branches[1:] { + gitIn(t, repoDir, "branch", b, first.String()) + } + gitIn(t, repoDir, "update-ref", "--no-deref", "HEAD", first.String()) + }) + + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + o := f.observed() + assert.Equal(t, ParentFound, o.ParentState) + assert.Equal(t, tc.want, o.ParentBranch, "status names the branch go-git chose") + }) + } +} + +// noOpWrite sends a live write that changes nothing: the delete of an object Git never had. +func noOpWrite(loop *branchWorkerEventLoop) { + deleted := configMapTargetEvent("never-written", "alice", newBranchTarget) + deleted.Operation = "DELETE" + deleted.Object = nil + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{deleted}, CommitMode: CommitModePerEvent}}) +} + +// §4.7 R5. With spec.parentBranch omitted the parent is the default branch as last discovered. A +// refresh that sees HEAD switch from main to trunk at the same commit follows it with one fetch, so +// the next publication validates trunk and pushes report the same parent the refresh did. Without +// a refresher the worker keeps main until its next discovery, which the push advertisement cannot +// be: it carries no HEAD. +func TestBranchWorker_ADefaultBranchSwitchIsFollowedAtTheNextDiscovery(t *testing.T) { + seed := func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "trunk", "main") + } + + t.Run("a refresh follows it", func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newRealServerNewBranch(t, "default-switch-refresh", seed) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + require.Equal(t, "main", f.observed().ParentBranch) + + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/trunk") + refreshes := fetchCount(t, reader, f.worker, fetchReasonRefresh) + f.loop.handleRefreshRequest(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + assert.Equal( + t, + refreshes+1, + fetchCount(t, reader, f.worker, fetchReasonRefresh), + "one fetch follows the switch", + ) + assert.Equal(t, "trunk", f.observed().ParentBranch) + + noOpWrite(f.loop) + assert.Equal(t, "trunk", f.observed().ParentBranch, "the next no-op push agrees with the refresh") + assert.Equal(t, ObservedByPush, f.observed().By) + }) + + t.Run("without a refresher it waits for the next discovery", func(t *testing.T) { + f := newRealServerNewBranch(t, "default-switch-no-refresh", seed) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/trunk") + noOpWrite(f.loop) + assert.Equal(t, "main", f.observed().ParentBranch, "publication cannot see HEAD") + }) +} + +// §4.7 R6. The write branch is created by somebody else after our advertisement, with the parent +// unmoved. The server refuses our upload (Old was zero) as an untyped error, and probing the parent +// alone would find it unmoved and conclude nothing happened. The fallback reads both refs, sees the +// write branch, and replays onto their commit. +func TestBranchWorker_AWriteBranchCreatedDuringOurUploadIsReplayedOnto(t *testing.T) { + f := newRealServerNewBranch(t, "refused-upload", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + }) + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + + var theirs plumbing.Hash + afterPushValidation = func() { + afterPushValidation = nil + theirs = simulateClientCommitOnDisk(t, f.repoDir, "feature", "THEIRS.md", "theirs\n") + } + t.Cleanup(func() { afterPushValidation = nil }) + roots := pushRoots(t, nil) + + liveWrite(f.loop, "cm1") + + require.False(t, theirs.IsZero(), "the branch was created between the advertisement and the upload") + require.Len(t, *roots, 2, "the refused upload, then the replay") + tip := f.ref("feature") + require.NotEqual(t, theirs, tip, "our commit was published") + gitIn(t, f.repoDir, "merge-base", "--is-ancestor", theirs.String(), tip.String()) + assert.Contains(t, gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "feature"), cmFile("cm1")) + assert.Empty(t, f.loop.pendingWrites) +} diff --git a/internal/git/branch_worker_new_branch_test.go b/internal/git/branch_worker_new_branch_test.go new file mode 100644 index 00000000..c40357a5 --- /dev/null +++ b/internal/git/branch_worker_new_branch_test.go @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// A write branch the remote does not carry yet is created from its parent, the remote's default +// branch. These tests pin the two halves of that contract on ONE warm worker, which is the case +// the cold-worker test (TestBranchWorker_CommitAndPushRequest_NewBranchStartsFromLatestMain) cannot +// reach: a worker that already trusts its checkout of the parent plans without fetching, so only +// the push can notice that the parent moved. +// +// - Publication checks the parent: the first commit on the new branch descends from the parent's +// tip as the push session advertised it, not from the checkout the plan ran on. +// - Standby is quiet: a cycle that commits nothing leaves the write branch absent on the remote. + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/config" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +const ( + newBranchTarget = "standby-target" + newBranchFolder = "standby" +) + +// newBranchFixture is a remote whose main is seeded and whose write branch `feature` is absent, +// a worker for `feature`, and a second clone that moves main from outside. +type newBranchFixture struct { + t *testing.T + worker *BranchWorker + server *git.Repository + seed *git.Repository + seedTree *git.Worktree + seedPath string + remote string +} + +func newNewBranchFixture(t *testing.T, seedFiles map[string]string) *newBranchFixture { + t.Helper() + ctx := context.Background() + tempDir := t.TempDir() + remotePath := filepath.Join(tempDir, "remote.git") + remoteURL := "file://" + remotePath + server := createBareRepo(t, remotePath) + + seedPath := filepath.Join(tempDir, "seed") + seed, seedTree := initLocalRepo(t, seedPath, remoteURL, "main") + f := &newBranchFixture{ + t: t, server: server, seed: seed, seedTree: seedTree, seedPath: seedPath, remote: remoteURL, + } + files := map[string]string{"README.md": "seed\n"} + for name, content := range seedFiles { + files[name] = content + } + f.pushToMain(files) + + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configv1alpha3.AddToScheme(scheme)) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + provider := &configv1alpha3.GitProvider{Spec: configv1alpha3.GitProviderSpec{URL: remoteURL}} + provider.Name = "test-repo" + provider.Namespace = "default" + require.NoError(t, k8sClient.Create(ctx, provider)) + + f.worker = NewBranchWorker( + k8sClient, logr.Discard(), "test-repo", "default", "feature", + RepoIdentity{URL: remoteURL}, nil, BranchWorkerLimits{}, + ) + f.worker.ctx = ctx + f.worker.mapper = configMapMapper() + t.Cleanup(func() { _ = os.RemoveAll(f.worker.repoRootPath()) }) + createGitTarget(t, f.worker, newBranchTarget, newBranchFolder, + &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneAlways}) + return f +} + +// pushToMain commits files to main from outside the worker and returns the new tip. +func (f *newBranchFixture) pushToMain(files map[string]string) plumbing.Hash { + f.t.Helper() + for name, content := range files { + full := filepath.Join(f.seedPath, name) + require.NoError(f.t, os.MkdirAll(filepath.Dir(full), 0o750)) + require.NoError(f.t, os.WriteFile(full, []byte(content), 0o600)) + _, err := f.seedTree.Add(name) + require.NoError(f.t, err) + } + hash, err := f.seedTree.Commit("outside", &git.CommitOptions{ + Author: &object.Signature{Name: "Outside", Email: "outside@example.com", When: time.Now()}, + }) + require.NoError(f.t, err) + require.NoError(f.t, f.seed.Push(&git.PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/main:refs/heads/main"}, + })) + return hash +} + +// mainTip is where main is on the remote. +func (f *newBranchFixture) mainTip() plumbing.Hash { + f.t.Helper() + ref, err := f.server.Reference(plumbing.NewBranchReferenceName("main"), true) + require.NoError(f.t, err) + return ref.Hash() +} + +// featureOnRemote reports where the write branch is on the remote, if anywhere. +func (f *newBranchFixture) featureOnRemote() (plumbing.Hash, bool) { + f.t.Helper() + ref, err := f.server.Reference(plumbing.NewBranchReferenceName("feature"), true) + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return plumbing.ZeroHash, false + } + require.NoError(f.t, err) + return ref.Hash(), true +} + +// warmOnParent is the trusted fetch/reset a new worker performs: HEAD on main, base trusted. +func (f *newBranchFixture) warmOnParent() plumbing.Hash { + f.t.Helper() + require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + require.True(f.t, f.worker.baseTrusted()) + branch, hash := f.localHead() + require.Equal(f.t, plumbing.NewBranchReferenceName("main"), branch, "an absent write branch checks out its parent") + return hash +} + +// noopResync runs a completed resync that finds nothing to change. +func (f *newBranchFixture) noopResync(loop *branchWorkerEventLoop) { + f.t.Helper() + req := &ResyncRequest{ + ResourceVersion: "1", + GitTargetName: newBranchTarget, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + loop.handleQueueItem(WorkItem{Resync: req}) + result := <-req.Result + require.NoError(f.t, result.Err) + require.Zero(f.t, result.Stats.Created+result.Stats.Updated+result.Stats.Deleted, "the resync must be a no-op") + require.Empty(f.t, loop.pendingWrites, "a no-op resync retains nothing") +} + +func (f *newBranchFixture) localHead() (plumbing.ReferenceName, plumbing.Hash) { + f.t.Helper() + repo, err := git.PlainOpen(f.worker.repoPath()) + require.NoError(f.t, err) + branch, hash, err := GetCurrentBranch(repo) + require.NoError(f.t, err) + return branch, hash +} + +// liveWrite sends one live ConfigMap event through the loop's commit window. +func liveWrite(loop *branchWorkerEventLoop, name string) { + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent(name, "alice", newBranchTarget)}, + CommitMode: CommitModePerEvent, + }}) +} + +// treeHas reports whether the commit's tree carries the file. +func treeHas(t *testing.T, repo *git.Repository, commit plumbing.Hash, file string) bool { + t.Helper() + c, err := repo.CommitObject(commit) + require.NoError(t, err) + _, err = c.File(file) + return err == nil +} + +func cmFile(name string) string { return newBranchFolder + "/default/configmaps/" + name + ".yaml" } + +// cmDocument is the document the worker writes for configMapTargetEvent(name, ...). +func cmDocument(name string) string { + return "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " + name + + "\n namespace: default\ndata:\n key: " + name + "\n" +} + +// TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip is the defect: a warm +// worker trusts its checkout of the parent at A, the parent moves to B, and a live write must +// still create the write branch on B. +// +// The two local states differ in what the cycle records as its push root, which is what decides +// whether the push primitive's own parent check is exercised. Both are inspected rather than +// assumed. +func TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip(t *testing.T) { + cases := []struct { + name string + setup func(f *newBranchFixture, loop *branchWorkerEventLoop) + }{ + { + name: "HEAD on the parent after a trusted fetch", + setup: func(f *newBranchFixture, _ *branchWorkerEventLoop) { f.warmOnParent() }, + }, + { + name: "a local write branch left by a no-op resync", + setup: func(f *newBranchFixture, loop *branchWorkerEventLoop) { + f.noopResync(loop) + branch, _ := f.localHead() + require.Equal(t, plumbing.NewBranchReferenceName("feature"), branch, + "a no-op resync prepares the local write branch before it learns there is nothing to do") + _, onRemote := f.featureOnRemote() + require.False(t, onRemote) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + hashA := f.mainTip() + tc.setup(f, loop) + _, local := f.localHead() + require.Equal(t, hashA, local, "the worker checked out A") + require.True(t, f.worker.baseTrusted(), "and trusts it, so the next plan does not fetch") + + hashB := f.pushToMain(map[string]string{"LATEST.md": "from main at B\n"}) + + // Hold the push so the cycle's recorded root can be read before it is sent. + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + assert.Equal(t, plumbing.NewBranchReferenceName("main"), f.worker.pushCycleRootBranch, + "the cycle is rooted on the parent the write branch is created from") + assert.Equal(t, hashA, f.worker.pushCycleRootHash, "planned on the stale checkout, as designed") + + loop.pushPending() + require.Empty(t, loop.pendingWrites, "the push succeeded") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "the write created the branch") + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + require.Len(t, commit.ParentHashes, 1) + assert.Equal(t, hashB, commit.ParentHashes[0], "the new branch starts from the parent's current tip") + assert.True(t, treeHas(t, f.server, tip, "LATEST.md"), "and carries B's change") + assert.True(t, treeHas(t, f.server, tip, cmFile("cm1")), "and the captured edit") + }) + } +} + +// TestBranchWorker_InSyncSnapshotsOnAbsentBranchCreateNothing is standby across parent movement: +// two in-sync snapshots, with an unrelated change to main between them, push nothing at all. +func TestBranchWorker_InSyncSnapshotsOnAbsentBranchCreateNothing(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + f.noopResync(loop) + f.pushToMain(map[string]string{"UNRELATED.md": "outside the folder\n"}) + f.noopResync(loop) + + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "an in-sync target creates no branch") + assert.True(t, loop.lastPushAt.IsZero(), "and attempts no push") + assert.Zero(t, fetchCount(t, reader, f.worker, fetchReasonContention)) +} + +// TestBranchWorker_ResyncOnAbsentBranchDiffsAgainstTheMovedParent is the other side: once main +// moves so the cluster differs, the snapshot diffs against the new main and creates the branch +// from it. +func TestBranchWorker_ResyncOnAbsentBranchDiffsAgainstTheMovedParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + f.noopResync(loop) + moved := f.pushToMain(map[string]string{cmFile("stale"): cmDocument("stale")}) + + req := &ResyncRequest{ + ResourceVersion: "2", + GitTargetName: newBranchTarget, + GitTargetNamespace: "default", + Result: make(chan ResyncResult, 1), + } + loop.handleQueueItem(WorkItem{Resync: req}) + result := <-req.Result + require.NoError(t, result.Err) + assert.Equal(t, 1, result.Stats.Deleted, "the document main gained is not in the cluster") + loop.pushPending() + require.Empty(t, loop.pendingWrites) + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{moved}, commit.ParentHashes) + assert.False(t, treeHas(t, f.server, tip, cmFile("stale"))) +} + +// TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby is "keep normal deployments +// quiet". A live write the parent already holds commits nothing, and its push must not create the +// write branch merely to record that nothing changed. It still reaches the remote: "already +// present" is a claim only the advertisement can settle. +func TestBranchWorker_NoDiffLiveWriteOnAbsentBranchStaysOnStandby(t *testing.T) { + t.Run("the parent already holds the document", func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + liveWrite(loop, "cm1") + + require.Empty(t, loop.pendingWrites, "the no-diff write was published") + assert.False(t, loop.lastPushAt.IsZero(), "it reached the remote to confirm") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "and the write branch stays absent") + observed, known := f.worker.LastRemoteObservation() + require.True(t, known) + assert.Empty(t, observed.Commit, "the push observed no write branch") + }) + + t.Run("the parent gained the document after the plan", func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + require.False(t, loop.pendingWrites[0].CommitSHA.IsZero(), "planned against A, the write is a commit") + + // The reconciler applied main, and main now holds exactly what the cluster holds. + f.pushToMain(map[string]string{cmFile("cm1"): cmDocument("cm1")}) + loop.pushPending() + + require.Empty(t, loop.pendingWrites) + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "replayed against the new parent, nothing is left to publish") + assert.Equal(t, int64(1), fetchCount(t, reader, f.worker, fetchReasonContention), + "the publication check is what caught the moved parent") + }) +} + +// TestBranchWorker_NewBranchPushCostsNoExtraFetchWhenTheParentHolds pins the price of the check: +// it is read from the advertisement the push session opens anyway. +func TestBranchWorker_NewBranchPushCostsNoExtraFetchWhenTheParentHolds(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + hashA := f.warmOnParent() + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{hashA}, commit.ParentHashes) + for _, reason := range []string{fetchReasonPublication, fetchReasonContention, fetchReasonPushFailureProbe} { + assert.Zero(t, fetchCount(t, reader, f.worker, reason), "no %s fetch", reason) + } +} + +// TestBranchWorker_ParentMovesWhileTheWindowIsOpen covers the window between planning and +// publication from the other end: the parent moves while the commit window is still collecting. +func TestBranchWorker_ParentMovesWhileTheWindowIsOpen(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + defer loop.stopTimers() + f.warmOnParent() + + liveWrite(loop, "cm1") + require.NotNil(t, loop.openWindow, "the window is still collecting") + hashB := f.pushToMain(map[string]string{"LATEST.md": "from main at B\n"}) + + require.True(t, loop.finalizeOpenWindow()) + loop.maybeSchedulePush() + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{hashB}, commit.ParentHashes) +} + +// TestBranchWorker_DeletedWriteBranchIsRecreatedFromTheParent is the return to standby after a +// review: the write branch is merged and deleted, main advances, and the next edit through the +// SAME worker recreates the branch on the new main with only the new edit on top. +func TestBranchWorker_DeletedWriteBranchIsRecreatedFromTheParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + liveWrite(loop, "cm1") + _, onRemote := f.featureOnRemote() + require.True(t, onRemote) + + // Merged by content and deleted, the way a squash merge leaves it. + merged := f.pushToMain(map[string]string{cmFile("cm1"): cmDocument("cm1"), "MERGED.md": "squashed\n"}) + require.NoError(t, f.server.Storer.RemoveReference(plumbing.NewBranchReferenceName("feature"))) + + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + require.Empty(t, loop.pendingWrites) + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "the next edit recreates the branch") + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{merged}, commit.ParentHashes, "on the parent's current tip") + assert.True(t, treeHas(t, f.server, tip, "MERGED.md")) + assert.True(t, treeHas(t, f.server, tip, cmFile("cm2"))) +} + +// TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent covers the two ways the check can +// fail to settle: a parent that keeps moving exhausts the bounded retries, and a remote that cannot +// be reached answers nothing. Either way the work stays retained and the write branch uncreated. +func TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent(t *testing.T) { + t.Run("the parent keeps moving", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + // Every reset onto the remote is immediately outrun by another outside commit, so each + // attempt is checked again and each one finds the parent moved. + original := syncToRemoteFn + t.Cleanup(func() { syncToRemoteFn = original }) + moves := 0 + syncToRemoteFn = func( + ctx context.Context, repo *git.Repository, branch plumbing.ReferenceName, parent string, + auth []gitclient.Option, + ) (*PullReport, error) { + report, err := original(ctx, repo, branch, parent, auth) + moves++ + f.pushToMain(map[string]string{"MOVING.md": time.Now().String() + string(rune('a'+moves))}) + return report, err + } + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + f.pushToMain(map[string]string{"FIRST.md": "moved before the first attempt\n"}) + loop.pushPending() + + assert.Len(t, loop.pendingWrites, 1, "the work is retained") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "and nothing unchecked was published") + assert.Positive(t, moves) + }) + + t.Run("the remote cannot be reached", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + original := pushAtomicFn + t.Cleanup(func() { pushAtomicFn = original }) + pushAtomicFn = func( + context.Context, *git.Repository, plumbing.Hash, plumbing.ReferenceName, []gitclient.Option, + ) (PushOutcome, error) { + return PushOutcome{}, errors.New("connection refused") + } + + liveWrite(loop, "cm1") + + assert.Len(t, loop.pendingWrites, 1, "the work is retained for retry") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote) + }) +} + +// TestBranchWorker_WriteBranchCreatedConcurrentlyIsBuiltOn is §2.1a through the worker: somebody +// creates the write branch between our plan and our push. Their commit must survive, and ours +// must be replayed on top of it rather than pushed over it. +func TestBranchWorker_WriteBranchCreatedConcurrentlyIsBuiltOn(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + f.warmOnParent() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + + theirs := simulateClientCommitOnDisk(t, f.remote, "feature", "THEIRS.md", "theirs\n") + loop.pushPending() + require.Empty(t, loop.pendingWrites) + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + commit, err := f.server.CommitObject(tip) + require.NoError(t, err) + assert.Equal(t, []plumbing.Hash{theirs}, commit.ParentHashes, "our commit descends from theirs") + assert.True(t, treeHas(t, f.server, tip, "THEIRS.md")) + assert.True(t, treeHas(t, f.server, tip, cmFile("cm1"))) +} diff --git a/internal/git/branch_worker_parent_branch_test.go b/internal/git/branch_worker_parent_branch_test.go new file mode 100644 index 00000000..e5e0617c --- /dev/null +++ b/internal/git/branch_worker_parent_branch_test.go @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// spec.parentBranch names the branch a write branch the remote does not carry is created from. +// Omitted keeps the remote's default branch, which the tests in branch_worker_new_branch_test.go +// cover; these cover a configured one: it is followed while the write branch is absent, it must +// exist for the write branch to be created, and it stops mattering once the write branch exists. + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// pushToRelease commits one file to `release` from outside, creating it from main when absent. +func (f *newBranchFixture) pushToRelease(file, content string) plumbing.Hash { + f.t.Helper() + return simulateClientCommitOnDisk(f.t, f.remote, "release", file, content) +} + +func (f *newBranchFixture) commitParent(tip plumbing.Hash) plumbing.Hash { + f.t.Helper() + // Opened afresh: a handle opened before the pushes can miss a packfile they added. + server, err := gogit.PlainOpen(strings.TrimPrefix(f.remote, "file://")) + require.NoError(f.t, err) + commit, err := server.CommitObject(tip) + require.NoError(f.t, err) + require.Len(f.t, commit.ParentHashes, 1) + return commit.ParentHashes[0] +} + +func TestSmartFetchFrom_ConfiguredParent(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + remoteURL := "file://" + filepath.Join(tempDir, "remote.git") + createBareRepo(t, filepath.Join(tempDir, "remote.git")) + simulateClientCommitOnDisk(t, remoteURL, "main", "README.md", "main\n") + simulateClientCommitOnDisk(t, remoteURL, "release", "RELEASE.md", "release\n") + simulateClientCommitOnDisk(t, remoteURL, "existing", "EXISTING.md", "existing\n") + + cases := []struct { + name, target, parent string + want plumbing.ReferenceName + wantErr error + }{ + {name: "omitted falls back to the default branch", target: "feature", want: "refs/heads/main"}, + {name: "a configured parent replaces the default", target: "feature", parent: "release", + want: "refs/heads/release"}, + {name: "a missing configured parent is refused", target: "feature", parent: "typo", + wantErr: ErrParentBranchNotFound}, + {name: "an existing target ignores a missing parent", target: "existing", parent: "typo", + want: "refs/heads/existing"}, + {name: "a parent equal to an absent target is refused", target: "feature", parent: "feature", + wantErr: ErrParentBranchNotFound}, + {name: "a parent equal to an existing target is that branch", target: "existing", parent: "existing", + want: "refs/heads/existing"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + repo, _ := initLocalRepo(t, filepath.Join(t.TempDir(), "local"), remoteURL, "") + got, err := SmartFetchFrom(ctx, repo, plumbing.NewBranchReferenceName(tc.target), tc.parent, nil) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } + + t.Run("an empty repository refuses a configured parent", func(t *testing.T) { + emptyURL := "file://" + filepath.Join(t.TempDir(), "empty.git") + createBareRepo(t, emptyURL[len("file://"):]) + repo, _ := initLocalRepo(t, filepath.Join(t.TempDir(), "local"), emptyURL, "") + _, err := SmartFetchFrom(ctx, repo, plumbing.NewBranchReferenceName("main"), "main", nil) + require.ErrorIs(t, err, ErrParentBranchNotFound) + + got, err := SmartFetchFrom(ctx, repo, plumbing.NewBranchReferenceName("main"), "", nil) + require.NoError(t, err) + assert.Empty(t, got, "an omitted parent keeps the empty-repository bootstrap") + }) +} + +// TestBranchWorker_NewBranchStartsFromTheConfiguredParent: the write branch is created on the +// configured parent's tip as publication saw it, even when the parent moved after the checkout. +func TestBranchWorker_NewBranchStartsFromTheConfiguredParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release at R1\n") + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + branch, _ := f.localHead() + require.Equal( + t, + plumbing.NewBranchReferenceName("release"), + branch, + "the absent write branch checks out its parent", + ) + r2 := f.pushToRelease("RELEASE-2.md", "release at R2\n") + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, r2, f.commitParent(tip), "created on the configured parent's current tip") + assert.True(t, treeHas(t, f.server, tip, "RELEASE-2.md")) +} + +// TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists: a configured parent the +// remote does not carry refuses the write, starts no orphan branch, and is reported; once it is +// pushed the next write creates the branch on it. +func TestBranchWorker_MissingConfiguredParentWritesNothingUntilItExists(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + assert.Empty(t, loop.pendingWrites, "nothing was committed") + assert.True(t, loop.lastPushAt.IsZero(), "and nothing was pushed") + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote, "no orphan branch") + observed, known := f.worker.LastRemoteObservation() + require.True(t, known) + assert.Equal(t, ParentMissing, observed.ParentState) + + r1 := f.pushToRelease("RELEASE.md", "now it exists\n") + pastProbeDeadline(f.worker) // until then a known-missing parent is not fetched again + liveWrite(loop, "cm2") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "it recovers once the parent exists") + assert.Equal(t, r1, f.commitParent(tip)) + observed, _ = f.worker.LastRemoteObservation() + assert.Empty(t, observed.ParentState, "the write branch exists, so no parent is reported") +} + +// TestBranchWorker_ParentBranchEqualToTheWriteBranch: legal when the branch exists, and then it is +// simply that branch; refused when it does not, because nothing names where to start it. +func TestBranchWorker_ParentBranchEqualToTheWriteBranch(t *testing.T) { + t.Run("the branch exists", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + existing := simulateClientCommitOnDisk(t, f.remote, "feature", "FEATURE.md", "feature\n") + f.worker.SetParentBranch("feature") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + tip, _ := f.featureOnRemote() + assert.Equal(t, existing, f.commitParent(tip)) + }) + + t.Run("the branch is absent", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.worker.SetParentBranch("feature") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + _, onRemote := f.featureOnRemote() + assert.False(t, onRemote) + observed, _ := f.worker.LastRemoteObservation() + assert.Equal(t, ParentMissing, observed.ParentState) + assert.Equal(t, "feature", observed.ParentBranch) + }) + + t.Run("an empty repository", func(t *testing.T) { + worker, server, _ := setupCommitPushSplitWorkerOnEmptyRemote(t) + worker.mapper = configMapMapper() + createGitTarget(t, worker, newBranchTarget, newBranchFolder, + &configv1alpha3.PrunePolicy{Mode: configv1alpha3.PruneAlways}) + worker.SetParentBranch("main") + loop := newBranchWorkerEventLoop(worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + _, err := server.Reference(plumbing.NewBranchReferenceName("main"), true) + require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "an explicit parent never bootstraps") + }) +} + +// TestBranchWorker_AnExistingWriteBranchDoesNotFollowItsParent is the rule kept from Flux and Argo +// CD: the parent decides where the branch starts, and nothing after that. +func TestBranchWorker_AnExistingWriteBranchDoesNotFollowItsParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release at R1\n") + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + first, onRemote := f.featureOnRemote() + require.True(t, onRemote) + + f.pushToRelease("RELEASE-2.md", "release at R2\n") + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + tip, _ := f.featureOnRemote() + assert.Equal(t, first, f.commitParent(tip), "the write branch builds on itself") + assert.False(t, treeHas(t, f.server, tip, "RELEASE-2.md")) +} + +// TestBranchWorker_SetParentBranchInvalidatesTheBase: a changed parent is re-read on the next +// cycle; an unchanged one costs nothing. +func TestBranchWorker_SetParentBranchInvalidatesTheBase(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.warmOnParent() + + assert.False(t, f.worker.SetParentBranch(""), "unchanged") + assert.True(t, f.worker.baseTrusted()) + + assert.True(t, f.worker.SetParentBranch("release")) + assert.Equal(t, "release", f.worker.ParentBranch()) + assert.False(t, f.worker.baseTrusted(), "the checkout holds the old parent") +} + +// TestBranchWorker_AParentChangeRebuildsRetainedWrites: writes retained during the push cooldown +// were planned against the old parent, and an invalidated base does not reach them. The change is +// taken by the event loop, which refetches and replays before the next commit or push, so the +// branch starts on the new parent, or is not created at all when the new parent is missing. +func TestBranchWorker_AParentChangeRebuildsRetainedWrites(t *testing.T) { + for _, parent := range []string{"release", "missing"} { + t.Run(parent, func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + require.True(t, loop.pendingWrites[0].CommitSHA.IsZero(), "a no-op is retained during the cooldown") + + require.True(t, f.worker.SetParentBranch(parent)) + liveWrite(loop, "cm2") + loop.pushPending() + + tip, exists := f.featureOnRemote() + if parent == "missing" { + assert.False(t, exists, "a missing configured parent prevents branch creation") + assert.NotEmpty(t, loop.pendingWrites, "and the retained work is kept") + return + } + require.True(t, exists) + assert.Equal(t, release, f.commitParent(tip), "the first push uses the configured parent") + }) + } +} + +// TestBranchWorker_AParentChangeSurvivesAFailedRebuild: the rebuild a parent change asks for can +// fail before it resets anything, here on reading the GitProvider. The change must still be +// pending on the next attempt; taking it at the first try would leave the retained writes planned +// on the old parent, and the retry would publish from it, or create the branch although the new +// parent is missing. +func TestBranchWorker_AParentChangeSurvivesAFailedRebuild(t *testing.T) { + for _, parent := range []string{"release", "missing"} { + t.Run(parent, func(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + require.True(t, f.worker.SetParentBranch(parent)) + + inner := f.worker.Client.(client.WithWatch) + f.worker.Client = interceptor.NewClient(inner, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, + opts ...client.GetOption) error { + if _, ok := obj.(*configv1alpha3.GitProvider); ok { + return errors.New("transient API failure") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + loop.pushPending() + require.Len(t, loop.pendingWrites, 1, "a failed rebuild keeps the work") + + f.worker.Client = inner + loop.pushPending() + + tip, exists := f.featureOnRemote() + if parent == "missing" { + assert.False(t, exists, "the missing new parent still prevents branch creation on the retry") + return + } + require.True(t, exists) + assert.Equal(t, release, f.commitParent(tip), "the retry starts from the new parent") + }) + } +} + +// TestBranchWorker_ObservesTheParentOfAnAbsentBranch: every way the worker looks at the remote +// records, alongside the write branch's absence, the parent it would be created from, which +// status.remote.parent publishes. With spec.parentBranch omitted that names the remote's default +// branch. Once the write branch exists, there is no parent to report. +func TestBranchWorker_ObservesTheParentOfAnAbsentBranch(t *testing.T) { + observed := func(f *newBranchFixture) RemoteObservation { + t.Helper() + o, known := f.worker.LastRemoteObservation() + require.True(t, known) + return o + } + + t.Run("a fetch names the default branch when the parent is omitted", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + hashA := f.warmOnParent() + o := observed(f) + assert.Equal(t, ObservedByFetch, o.By) + assert.Empty(t, o.Commit) + assert.Equal(t, ParentFound, o.ParentState) + assert.Equal(t, "main", o.ParentBranch) + assert.Equal(t, hashA.String(), o.ParentCommit) + }) + + t.Run("an empty repository is unborn", func(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorkerOnEmptyRemote(t) + require.NoError(t, worker.ensureRepositoryInitialized(worker.ctx)) + o, known := worker.LastRemoteObservation() + require.True(t, known) + assert.Equal(t, ParentUnborn, o.ParentState) + assert.Empty(t, o.ParentBranch) + assert.Empty(t, o.ParentCommit) + }) + + t.Run("a push that leaves the branch absent confirms the parent", func(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + hashA := f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + + o := observed(f) + assert.Equal(t, ObservedByPush, o.By) + assert.Equal(t, "main", o.ParentBranch) + assert.Equal(t, hashA.String(), o.ParentCommit) + + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + o = observed(f) + assert.NotEmpty(t, o.Commit, "the branch exists now") + assert.Empty(t, o.ParentBranch, "and its parent is no longer followed") + }) + + t.Run("a configured parent, present and missing", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + liveWrite(loop, "cm1") + o := observed(f) + assert.Equal(t, "release", o.ParentBranch) + assert.Empty(t, o.ParentCommit, "the parent is not on the remote") + assert.Equal(t, ParentMissing, o.ParentState) + + r1 := f.pushToRelease("RELEASE.md", "release\n") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + o = observed(f) + assert.Equal(t, "release", o.ParentBranch) + assert.Equal(t, r1.String(), o.ParentCommit) + assert.Equal(t, ParentFound, o.ParentState) + }) +} diff --git a/internal/git/branch_worker_parent_change_test.go b/internal/git/branch_worker_parent_change_test.go new file mode 100644 index 00000000..c697f91e --- /dev/null +++ b/internal/git/branch_worker_parent_change_test.go @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// A parent change reaches a live worker from the reconcile goroutine while the event loop may be in +// the middle of a fetch or a push. These pin the rules that make that safe: every operation uses +// one snapshot of the parent configuration, trust is scoped to the snapshot's generation, and a +// push is admitted only when its root was chosen under the current generation. A push already +// admitted is not recalled. + +import ( + "context" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// allFetchReasons is every series a fetch can be counted under. +var allFetchReasons = []string{ + fetchReasonBootstrap, fetchReasonPublication, fetchReasonRecovery, fetchReasonContention, + fetchReasonPushFailureProbe, fetchReasonRefresh, +} + +func totalFetches(t *testing.T, reader *sdkmetric.ManualReader, worker *BranchWorker) int64 { + t.Helper() + var total int64 + for _, reason := range allFetchReasons { + total += fetchCount(t, reader, worker, reason) + } + return total +} + +// onSync runs hook after the n-th sync (1-based) the worker makes, inside that fetch. +func onSync(t *testing.T, n int, hook func()) { + t.Helper() + original := syncToRemoteFn + calls := 0 + syncToRemoteFn = func(ctx context.Context, repo *gogit.Repository, target plumbing.ReferenceName, + parent string, auth []gitclient.Option) (*PullReport, error) { + report, err := original(ctx, repo, target, parent, auth) + calls++ + if calls == n { + hook() + } + return report, err + } + t.Cleanup(func() { syncToRemoteFn = original }) +} + +// pushRoots records the root of every push attempt, and runs hook inside the first one, before it +// reaches the remote. +func pushRoots(t *testing.T, hook func()) *[]plumbing.ReferenceName { + t.Helper() + original := pushAtomicFn + var roots []plumbing.ReferenceName + pushAtomicFn = func(ctx context.Context, repo *gogit.Repository, rootHash plumbing.Hash, + rootBranch plumbing.ReferenceName, auth []gitclient.Option) (PushOutcome, error) { + roots = append(roots, rootBranch) + if len(roots) == 1 && hook != nil { + hook() + } + return original(ctx, repo, rootHash, rootBranch, auth) + } + t.Cleanup(func() { pushAtomicFn = original }) + return &roots +} + +// pastProbeDeadline moves the worker's clock past any parent-probe deadline, so a test can stand in +// for the time a held-back parent waits before it is looked for again. +func pastProbeDeadline(w *BranchWorker) { + w.clock = func() time.Time { return time.Now().Add(time.Hour) } +} + +// observations collects every observation the worker reports. +func observations(f *newBranchFixture) *[]RemoteObservation { + var seen []RemoteObservation + f.worker.remoteReporter = func(o RemoteObservation) { seen = append(seen, o) } + return &seen +} + +// Test 1: a change that lands inside a fetch. The fetch finishes under the old snapshot, so its +// observation names the parent it used, and the push that follows is not admitted on that root. +func TestBranchWorker_AParentChangeDuringAFetchIsNotPushedOnTheOldParent(t *testing.T) { + t.Run("the initial fetch", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + seen := observations(f) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + onSync(t, 1, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip), "the first commit sits on the new parent") + require.NotEmpty(t, *seen) + assert.Equal(t, "main", (*seen)[0].ParentBranch, "the fetch reports the parent it actually used") + }) + + t.Run("a contention fetch", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + f.pushToMain(map[string]string{"MOVED.md": "main moved\n"}) // the push is refused: contention + seen := observations(f) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + onSync(t, 1, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) + require.NotEmpty(t, *seen) + assert.Equal(t, "main", (*seen)[0].ParentBranch, "the contention fetch reports the parent it used") + }) +} + +// Test 2, the review's reproduction: a change inside a no-op push that settles as PushNoBranch. +// The push sets trust after the change, and the remembered parent is the old one; neither may +// root the next real write. +func TestBranchWorker_AParentChangeDuringANoOpPushIsTakenByTheNextWrite(t *testing.T) { + f := newNewBranchFixture(t, map[string]string{cmFile("cm1"): cmDocument("cm1")}) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + roots := pushRoots(t, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") // already in Git: settles without creating the branch + _, onRemote := f.featureOnRemote() + require.False(t, onRemote) + require.Len(t, *roots, 1) + + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip), "the next real write lands on the new parent") +} + +// Test 3: a change just before admission, with retained writes. No push leaves on the old root. +func TestBranchWorker_AParentChangeBeforeAdmissionReplaysOntoTheNewParent(t *testing.T) { + f := newNewBranchFixture(t, nil) + release := f.pushToRelease("RELEASE.md", "release\n") + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + loop.lastPushAt = time.Now() // cooldown: the write is retained + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + + changed := false + beforePushAdmission = func() { + if !changed { + changed = true + f.worker.SetParentBranch("release") + } + } + t.Cleanup(func() { beforePushAdmission = nil }) + roots := pushRoots(t, nil) + + loop.pushPending() + + require.Empty(t, loop.pendingWrites, "the replay published") + assert.NotContains(t, *roots, plumbing.NewBranchReferenceName("main"), "no push left on the old root") + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) +} + +// Test 4: a change after admission is in flight. The push that creates the branch from the old +// parent is not recalled; the branch then exists, and an existing branch does not follow its +// parent, so later writes build on it with no attempt to re-root it. +func TestBranchWorker_AParentChangeAfterAdmissionIsNotRecalled(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release\n") + mainTip := f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + roots := pushRoots(t, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + first, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, mainTip, f.commitParent(first), "the admitted push created the branch from main") + + fetches := totalFetches(t, reader, f.worker) + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + tip, _ := f.featureOnRemote() + assert.Equal(t, first, f.commitParent(tip), "later writes go to the existing branch") + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "and nothing re-reads the parent") + assert.Len(t, *roots, 2) + assert.Equal(t, plumbing.NewBranchReferenceName("feature"), (*roots)[1]) +} + +// Test 5: the new parent is missing, in cases 1 and 3. The branch is never created, the work stays +// retained, and it lands once the parent exists. +func TestBranchWorker_AParentChangeToAMissingParentKeepsTheWork(t *testing.T) { + t.Run("during a fetch", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + onSync(t, 1, func() { f.worker.SetParentBranch("release") }) + + liveWrite(loop, "cm1") + + _, onRemote := f.featureOnRemote() + require.False(t, onRemote, "no branch on the old parent") + require.NotEmpty(t, loop.pendingWrites, "the work is retained") + + release := f.pushToRelease("RELEASE.md", "now it exists\n") + pastProbeDeadline(f.worker) + loop.runParentProbe() + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote, "the probe published the retained work") + assert.Equal(t, release, f.commitParent(tip)) + }) + + t.Run("before admission", func(t *testing.T) { + f := newNewBranchFixture(t, nil) + f.warmOnParent() + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + loop.lastPushAt = time.Now() + liveWrite(loop, "cm1") + require.Len(t, loop.pendingWrites, 1) + beforePushAdmission = func() { f.worker.SetParentBranch("release") } + t.Cleanup(func() { beforePushAdmission = nil }) + + loop.pushPending() + + _, onRemote := f.featureOnRemote() + require.False(t, onRemote) + require.Len(t, loop.pendingWrites, 1, "the work is retained") + + beforePushAdmission = nil + release := f.pushToRelease("RELEASE.md", "now it exists\n") + pastProbeDeadline(f.worker) + loop.runParentProbe() + tip, onRemote := f.featureOnRemote() + require.True(t, onRemote) + assert.Equal(t, release, f.commitParent(tip)) + }) +} + +// Test 6: a parent cannot move an existing write branch, so a change costs it nothing: no fetch, +// and the next push goes to the write branch. +func TestBranchWorker_AParentChangeCostsAnExistingWriteBranchNothing(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + f := newNewBranchFixture(t, nil) + f.pushToRelease("RELEASE.md", "release\n") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + liveWrite(loop, "cm1") + first, onRemote := f.featureOnRemote() + require.True(t, onRemote) + + require.True(t, f.worker.SetParentBranch("release")) + fetches := totalFetches(t, reader, f.worker) + roots := pushRoots(t, nil) + loop.lastPushAt = time.Time{} + liveWrite(loop, "cm2") + + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "zero extra fetches") + require.Len(t, *roots, 1, "one push") + assert.Equal(t, plumbing.NewBranchReferenceName("feature"), (*roots)[0]) + tip, _ := f.featureOnRemote() + assert.Equal(t, first, f.commitParent(tip)) +} diff --git a/internal/git/branch_worker_queue_depth_test.go b/internal/git/branch_worker_queue_depth_test.go index 30f6ce2e..f4ba4e0b 100644 --- a/internal/git/branch_worker_queue_depth_test.go +++ b/internal/git/branch_worker_queue_depth_test.go @@ -130,7 +130,7 @@ func TestEnsureWorker_WaitsForTheWorkerItReplacesToStop(t *testing.T) { ensured := make(chan struct{}) go func() { - _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", RepoIdentity{}) + _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", RepoIdentity{}, "") close(ensured) }() @@ -178,7 +178,7 @@ func TestReconcileWorkers_DoesNotStopAWorkerEnsureWorkerJustHandedOut(t *testing // The SAME repository the retiring worker is about, deliberately: that is what makes // an unguarded EnsureWorker take its "already there" path, which is the regression // this test is here to catch. - _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", retiring.repo) + _ = manager.EnsureWorker(context.Background(), "test-provider", "test-ns", "main", retiring.repo, "") }() // Long enough for an unguarded EnsureWorker to finish inside the window; a guarded one is // blocked on lifecycleMu and finishes after the sweep instead. @@ -243,7 +243,7 @@ func TestReconcileWorkers_DoesNotStopAWorkerCreatedAfterItsSnapshot(t *testing.T go func() { defer close(ensured) _ = manager.EnsureWorker( - context.Background(), "newcomer-provider", "test-ns", "main", RepoIdentity{}) + context.Background(), "newcomer-provider", "test-ns", "main", RepoIdentity{}, "") }() time.Sleep(200 * time.Millisecond) }) diff --git a/internal/git/branch_worker_split_test.go b/internal/git/branch_worker_split_test.go index 8d7ee5c6..3bcb70cf 100644 --- a/internal/git/branch_worker_split_test.go +++ b/internal/git/branch_worker_split_test.go @@ -615,6 +615,7 @@ func TestBranchWorker_TransientPushFailure_RetriesSameLocalCommits(t *testing.T) _ context.Context, _ *git.Repository, _ plumbing.ReferenceName, + _ string, _ []gitclient.Option, ) (*PullReport, error) { syncCalled = true @@ -694,6 +695,7 @@ func TestBranchWorker_PushFollowedByFetchFailure_TreatsAsTransient(t *testing.T) _ context.Context, _ *git.Repository, _ plumbing.ReferenceName, + _ string, _ []gitclient.Option, ) (*PullReport, error) { syncCalled = true diff --git a/internal/git/commit_request_attach.go b/internal/git/commit_request_attach.go index dd2d0c32..97b65be0 100644 --- a/internal/git/commit_request_attach.go +++ b/internal/git/commit_request_attach.go @@ -62,6 +62,18 @@ const ( PhaseWaitingForPush CommitRequestPhase = "WaitingForPush" ) +// Held reports whether the worker has acted on a request in this phase: it is attached to a window +// or committed. From there only the worker can say how it ends, so the controller waits for it +// however long that takes, instead of failing the request on a clock of its own. +func (p CommitRequestPhase) Held() bool { + return p == PhaseCollectingWindow || p == PhaseWaitingForPush +} + +// ErrCommitRequestWithdrawn is the outcome of a request the controller withdrew before the worker +// acted on it. The worker records it, so a late re-send of the attach can never register the +// request again, and the controller's failure and the worker's answer are the same answer. +var ErrCommitRequestWithdrawn = errors.New("the CommitRequest was withdrawn before the worker acted on it") + // FinalizeResult carries the resolved outcome of a CommitRequest back to the // controller, polled via LookupCommitRequestOutcome. type FinalizeResult struct { diff --git a/internal/git/commit_request_attach_loop.go b/internal/git/commit_request_attach_loop.go index 9b665ed1..433ab7e1 100644 --- a/internal/git/commit_request_attach_loop.go +++ b/internal/git/commit_request_attach_loop.go @@ -29,25 +29,115 @@ import ( // commitRequestOutcomeTTL bounds how long a resolved CommitRequest outcome is // retained for the controller to poll before it is GC'd. It comfortably exceeds -// the controller's poll cadence and safety bound. +// the controller's poll cadence. It is not what keeps a withdrawn request withdrawn: +// an outcome is never collected while an attach for it is still queued (crQueuedAttaches), +// and the controller sends no attach once it has started withdrawing. const commitRequestOutcomeTTL = 15 * time.Minute +// ErrBranchWorkerStopped is the outcome of a request whose branch worker stopped before it could +// finish it. Once the loop has exited nothing can publish the request, so this answer is final. +var ErrBranchWorkerStopped = errors.New("the GitTarget's branch worker stopped before it finished the CommitRequest") + // recordCommitRequestOutcome stores a resolved outcome and GCs stale entries. The // event loop is the only caller, but it takes the mutex because the controller // reads concurrently via LookupCommitRequestOutcome. func (w *BranchWorker) recordCommitRequestOutcome(id commitRequestID, result FinalizeResult) { w.crOutcomesMu.Lock() defer w.crOutcomesMu.Unlock() + w.recordCommitRequestOutcomeLocked(id, result) +} + +// recordCommitRequestOutcomeLocked is recordCommitRequestOutcome for a caller holding crOutcomesMu. +func (w *BranchWorker) recordCommitRequestOutcomeLocked(id commitRequestID, result FinalizeResult) { if w.crOutcomes == nil { w.crOutcomes = map[commitRequestID]commitRequestOutcomeEntry{} } now := time.Now() w.crOutcomes[id] = commitRequestOutcomeEntry{result: result, resolvedAt: now} for k, entry := range w.crOutcomes { - if now.Sub(entry.resolvedAt) > commitRequestOutcomeTTL { + if now.Sub(entry.resolvedAt) > commitRequestOutcomeTTL && w.crQueuedAttaches[k] == 0 { delete(w.crOutcomes, k) + w.crOwners.release(k, w) + } + } +} + +// countQueuedAttach records an attach entering the FIFO. The caller holds pendingResyncsMu, so the +// count is raised before the loop can receive the item. +func (w *BranchWorker) countQueuedAttach(id commitRequestID) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if w.crQueuedAttaches == nil { + w.crQueuedAttaches = map[commitRequestID]int{} + } + w.crQueuedAttaches[id]++ +} + +// uncountQueuedAttach records an attach leaving the FIFO: handled by the loop, dropped on a full +// queue, or drained by an exiting worker. A request the worker then knows nothing else about is +// given back, so the controller's next poll finds the worker its GitTarget names. +func (w *BranchWorker) uncountQueuedAttach(id commitRequestID) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if n := w.crQueuedAttaches[id]; n > 1 { + w.crQueuedAttaches[id] = n - 1 + return + } + delete(w.crQueuedAttaches, id) + _, resolved := w.crOutcomes[id] + _, registered := w.crPhases[id] + if !resolved && !registered { + w.crOwners.release(id, w) + } +} + +// settleCommitRequestsAfterExit runs once the event loop has returned: after a shutdown, after a +// retirement, or when the loop never started because its GitProvider could not be read. Either way +// nothing will act on a request again, and every request the worker knew must get an answer +// rather than wait for one that cannot come. +// +// - A request the worker held (attached, or committed) fails with ErrBranchWorkerStopped. A clean +// shutdown has already settled those through its last push; this catches what it could not. +// - A request it never acted on is given back. The old loop can no longer commit it, so the +// controller is free to send it to the worker the GitTarget names now, or to withdraw it. +// +// From then on a withdraw is answered at once (answerWithdrawAfterExit). +func (w *BranchWorker) settleCommitRequestsAfterExit() { + w.pendingResyncsMu.Lock() + w.stoppingState = true + w.pendingResyncsMu.Unlock() + + // A loop that never ran left its queue as the producers filled it. + w.drainQueue() + + w.crOutcomesMu.Lock() + for id, phase := range w.crPhases { + if phase.Held() { + w.recordCommitRequestOutcomeLocked(id, FinalizeResult{Branch: w.Branch, Err: ErrBranchWorkerStopped}) + } else { + w.crOwners.release(id, w) } + delete(w.crPhases, id) + } + w.crOutcomesMu.Unlock() + + w.pendingResyncsMu.Lock() + w.exitedState = true + w.pendingResyncsMu.Unlock() +} + +// answerWithdrawAfterExit resolves a withdraw sent to a worker whose loop has exited. Nothing can +// act on the request any more, so it is withdrawn for good, unless it already has an outcome. +func (w *BranchWorker) answerWithdrawAfterExit(id commitRequestID) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + if _, resolved := w.crOutcomes[id]; resolved { + return } + w.recordCommitRequestOutcomeLocked(id, FinalizeResult{ + Branch: w.Branch, + Err: errors.Join(ErrCommitRequestWithdrawn, ErrBranchWorkerStopped), + }) } // LookupCommitRequestOutcome returns a resolved CommitRequest outcome, or ok=false @@ -95,6 +185,9 @@ func (w *BranchWorker) hasCommitRequestOutcome(id commitRequestID) bool { // an idempotent re-send restarts nothing and closes nothing. func (l *branchWorkerEventLoop) handleAttachCommitRequest(req *AttachCommitRequest) { id := req.id() + // Uncounted only once the request is registered or resolved, so the worker never looks as if it + // had forgotten a request it is in the middle of taking. + defer l.w.uncountQueuedAttach(id) if l.w.hasCommitRequestOutcome(id) { return // already resolved: a late idempotent re-send. } @@ -365,6 +458,24 @@ func (w *BranchWorker) buildRequestRecordWrite( }, nil } +// handleWithdrawCommitRequest cancels a request the worker has not acted on: one that is only +// waiting for a window, or that was never registered at all. A request attached to a window or +// already committed is held, and withdrawing it is a no-op; the controller reads the held phase and +// keeps waiting. Either way the worker's answer and the controller's agree. +func (l *branchWorkerEventLoop) handleWithdrawCommitRequest(req *AttachCommitRequest) { + id := req.id() + if l.w.hasCommitRequestOutcome(id) { + return // already resolved, or already withdrawn. + } + if pcr, ok := l.pendingCRs[id]; ok && (pcr.attached || pcr.committed) { + l.w.Log.Info("CommitRequest withdraw ignored: the worker already holds it", + "request", id.Namespace+"/"+id.Name) + return + } + l.resolveCommitRequest(id, FinalizeResult{Err: ErrCommitRequestWithdrawn}) + l.rearmAttachTimer() +} + // resolveCommitRequest records a request's terminal outcome for the controller to // poll and forgets it from the pending set. func (l *branchWorkerEventLoop) resolveCommitRequest(id commitRequestID, result FinalizeResult) { diff --git a/internal/git/commit_request_owners.go b/internal/git/commit_request_owners.go new file mode 100644 index 00000000..132b2131 --- /dev/null +++ b/internal/git/commit_request_owners.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import "sync" + +// commitRequestOwners records which branch worker accepted each CommitRequest's attach, so the +// controller's questions about a request reach the worker that can still act on it. +// +// The GitTarget cannot answer that. A worker serves every GitTarget on its (provider, branch), so it +// outlives a deleted target, and a retired worker is detached from the manager before it stops, +// so its last push runs while the manager no longer lists it. Asking "which worker does the target +// name now?" in either case finds none, and reading that as "nothing holds the request" is how a +// request was reported withdrawn while its worker could still publish it. +// +// An entry is claimed when an attach enters a worker's queue, and released when that worker +// forgets the request: when its outcome is collected, or when the worker exits without having +// acted on it. A request with no entry is therefore held by no worker, and only then may the +// controller fall back to the GitTarget. +// +// It is shared by the manager and every worker it creates. The lock is a leaf: callers may hold +// a worker's pendingResyncsMu or crOutcomesMu when they take it, never the other way round. +type commitRequestOwners struct { + mu sync.Mutex + owners map[commitRequestID]*BranchWorker +} + +// owner returns the worker holding the request, if any. A nil table (a worker built outside a +// manager) knows no owners. +func (o *commitRequestOwners) owner(id commitRequestID) (*BranchWorker, bool) { + if o == nil { + return nil, false + } + o.mu.Lock() + defer o.mu.Unlock() + w, ok := o.owners[id] + return w, ok +} + +// claim makes w the request's owner, unless another worker already is. It reports whether w owns +// the request afterwards. A nil table lets every worker claim. +func (o *commitRequestOwners) claim(id commitRequestID, w *BranchWorker) bool { + if o == nil { + return true + } + o.mu.Lock() + defer o.mu.Unlock() + if current, ok := o.owners[id]; ok { + return current == w + } + if o.owners == nil { + o.owners = map[commitRequestID]*BranchWorker{} + } + o.owners[id] = w + return true +} + +// release forgets the request, if w is still its owner. +func (o *commitRequestOwners) release(id commitRequestID, w *BranchWorker) { + if o == nil { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if o.owners[id] == w { + delete(o.owners, id) + } +} diff --git a/internal/git/commit_request_withdraw_test.go b/internal/git/commit_request_withdraw_test.go new file mode 100644 index 00000000..22609e8e --- /dev/null +++ b/internal/git/commit_request_withdraw_test.go @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// The controller withdraws a CommitRequest before failing it closed. These pin the worker's half of +// that agreement: a request the worker has not acted on is cancelled for good, and a request it +// holds (attached, or committed) is not, so a request reported failed is never committed later. + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +func TestWithdraw_AWaitingRequestIsCancelledForGood(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", time.Hour) + serviceAttach(loop, req) + require.Equal(t, PhaseWaitingForWindow, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + + loop.handleQueueItem(WorkItem{Withdraw: req}) + + res, ok := outcome(t, worker) + require.True(t, ok, "a withdrawn request resolves") + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + assert.Empty(t, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + + // A late re-send of the attach, and a window it would have attached to, change nothing. + serviceAttach(loop, req) + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("late", "alice", "team-a")}, + CommitMode: CommitModePerEvent, + }}) + require.NotNil(t, loop.openWindow) + assert.Nil(t, loop.openWindow.pendingCR, "a withdrawn request never attaches") + assert.Empty(t, loop.pendingCRs) + res, _ = outcome(t, worker) + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) +} + +// A withdraw that arrives before the worker ever saw the attach leaves a tombstone, so the attach +// that was still in flight cannot register afterwards. +func TestWithdraw_AnUnregisteredRequestCannotRegisterLater(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", time.Hour) + loop.handleQueueItem(WorkItem{Withdraw: req}) + serviceAttach(loop, req) + + res, ok := outcome(t, worker) + require.True(t, ok) + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + assert.Empty(t, loop.pendingCRs) +} + +// A request attached to a window is held: the withdraw is a no-op, and the request still resolves +// with its commit. This is the case the controller reads as "keep waiting". +func TestWithdraw_AnAttachedRequestIsHeldAndStillCommits(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("held", "alice", "team-a")}, + CommitMode: CommitModePerEvent, + }}) + req := attachReq("alice", time.Hour) + serviceAttach(loop, req) + require.Equal(t, PhaseCollectingWindow, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + + loop.handleQueueItem(WorkItem{Withdraw: req}) + + _, resolved := outcome(t, worker) + require.False(t, resolved, "a held request is not withdrawn") + assert.True(t, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName).Held()) + + forceDue(loop) + loop.pushPending() + + res, ok := outcome(t, worker) + require.True(t, ok) + require.NoError(t, res.Err) + assert.Equal(t, FinalizeCommitted, res.Outcome) +} + +// A request that already resolved keeps its outcome. +func TestWithdraw_AResolvedRequestKeepsItsOutcome(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", 0) + serviceAttach(loop, req) // no window: resolves NoOpenWindow at once + before, ok := outcome(t, worker) + require.True(t, ok) + + loop.handleQueueItem(WorkItem{Withdraw: req}) + + after, _ := outcome(t, worker) + assert.Equal(t, before, after) +} + +func TestEnqueueWithdraw_QueueFullIsDroppedAndNilIsANoOp(t *testing.T) { + w := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1)} + w.EnqueueWithdraw(nil) + assert.Empty(t, w.eventQueue) + + w.eventQueue <- WorkItem{} // saturate + w.EnqueueWithdraw(attachReq("alice", 0)) + assert.Zero(t, w.inflightItems.Load(), "a dropped withdraw must not leak an inflight count") +} + +// The withdrawn outcome is what turns a late attach into a no-op, so it must outlive every attach +// still on the FIFO, however long that attach waits: a TTL alone only moves the problem. +func TestWithdraw_AnOutcomeIsKeptWhileAnAttachForItIsQueued(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + loop := newBranchWorkerEventLoop(worker, time.Hour) + defer loop.stopTimers() + + req := attachReq("alice", time.Hour) + loop.handleQueueItem(WorkItem{Withdraw: req}) + worker.EnqueueAttach(req) // a late attach, still on the FIFO + + // The outcome runs past its TTL while the attach waits, and a GC pass runs. + ageCommitRequestOutcome(worker, req.id(), 2*commitRequestOutcomeTTL) + worker.recordCommitRequestOutcome(commitRequestID{Namespace: "default", Name: "other"}, FinalizeResult{}) + _, kept := outcome(t, worker) + require.True(t, kept, "an outcome with an attach still queued is not collected") + + loop.handleQueueItem(<-worker.eventQueue) + loop.serviceCommitRequests() + assert.Empty(t, loop.pendingCRs, "the late attach does not register the withdrawn request") + res, _ := outcome(t, worker) + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + + // With nothing queued for it any more, the outcome is collected as usual. + ageCommitRequestOutcome(worker, req.id(), 2*commitRequestOutcomeTTL) + worker.recordCommitRequestOutcome(commitRequestID{Namespace: "default", Name: "other"}, FinalizeResult{}) + _, kept = outcome(t, worker) + assert.False(t, kept) +} + +// A worker whose loop never ran, because its GitProvider could not be read, is still listed by the +// manager, so attaches and withdraws keep arriving. It must answer them rather than queue them for +// a loop that is gone. +func TestWithdraw_AWorkerThatExitedAtStartupAnswers(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, configv1alpha3.AddToScheme(scheme)) + worker := NewBranchWorker(fake.NewClientBuilder().WithScheme(scheme).Build(), logr.Discard(), + "no-such-provider", "default", "main", RepoIdentity{URL: "file:///nowhere"}, nil, BranchWorkerLimits{}) + owners := &commitRequestOwners{} + worker.crOwners = owners + + req := attachReq("alice", time.Hour) + worker.EnqueueAttach(req) // queued before the loop gives up + require.NoError(t, worker.Start(context.Background())) + worker.wg.Wait() // the loop has exited + + _, owned := owners.owner(req.id()) + assert.False(t, owned, "a request the worker never acted on is given back") + assert.Zero(t, worker.inflightItems.Load(), "the queue it never read is drained") + + worker.EnqueueAttach(req) + _, resolved := outcome(t, worker) + assert.False(t, resolved, "an attach is refused, not queued") + + worker.EnqueueWithdraw(req) + res, resolved := outcome(t, worker) + require.True(t, resolved, "a withdraw is answered at once") + require.ErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + require.ErrorIs(t, res.Err, ErrBranchWorkerStopped) + assert.Zero(t, worker.inflightItems.Load()) +} + +// A worker that is retired keeps answering for the request it held: its shutdown settles it, and +// that outcome stays reachable through the owner table after the manager stopped listing it. +// Reading "no worker" as "nothing holds it" is what reported such a request withdrawn. +func TestWithdraw_ARetiredWorkerStillAnswersForWhatItHeld(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + owners := &commitRequestOwners{} + worker.crOwners = owners + require.NoError(t, worker.Start(context.Background())) + + require.True(t, worker.Enqueue(configMapTargetEvent("held", "alice", "team-a"))) + req := attachReq("alice", time.Hour) + worker.EnqueueAttach(req) + require.Eventually(t, func() bool { + return worker.LookupCommitRequestPhase("default", crName, "uid-"+crName).Held() + }, 5*time.Second, 10*time.Millisecond) + + worker.Stop() + + owner, owned := owners.owner(req.id()) + require.True(t, owned, "the retired worker still holds the request") + assert.Same(t, worker, owner) + res, resolved := outcome(t, worker) + require.True(t, resolved, "its shutdown settled the request, by its last push or by failing it") + require.NotErrorIs(t, res.Err, ErrCommitRequestWithdrawn) + + worker.EnqueueWithdraw(req) + after, _ := outcome(t, worker) + assert.Equal(t, res, after, "a withdraw after the fact keeps the real outcome") +} + +// A request still waiting for a window when its worker stops is given back: the old loop can no +// longer commit it, so the controller may send it to the GitTarget's next worker. +func TestWithdraw_AStoppedWorkerGivesBackWhatItNeverActedOn(t *testing.T) { + worker, _, _ := setupCommitPushSplitWorker(t) + createPlainGitTarget(t, worker, "team-a", "team-a") + owners := &commitRequestOwners{} + worker.crOwners = owners + require.NoError(t, worker.Start(context.Background())) + + req := attachReq("alice", time.Hour) + worker.EnqueueAttach(req) + require.Eventually(t, func() bool { + return worker.LookupCommitRequestPhase("default", crName, "uid-"+crName) == PhaseWaitingForWindow + }, 5*time.Second, 10*time.Millisecond) + + worker.Stop() + + _, owned := owners.owner(req.id()) + assert.False(t, owned) + assert.Empty(t, worker.LookupCommitRequestPhase("default", crName, "uid-"+crName)) + _, resolved := outcome(t, worker) + assert.False(t, resolved, "giving it back is not an outcome") +} + +func TestCommitRequestOwners_OneOwnerAtATime(t *testing.T) { + a, b := &BranchWorker{}, &BranchWorker{} + id := commitRequestID{Namespace: "default", Name: crName} + owners := &commitRequestOwners{} + + require.True(t, owners.claim(id, a)) + assert.True(t, owners.claim(id, a), "claiming again is idempotent") + assert.False(t, owners.claim(id, b), "a second worker cannot take a held request") + owners.release(id, b) + got, ok := owners.owner(id) + require.True(t, ok, "only the owner can release") + assert.Same(t, a, got) + owners.release(id, a) + assert.True(t, owners.claim(id, b)) + + var none *commitRequestOwners + assert.True(t, none.claim(id, a), "a worker outside a manager owns what it is sent") + _, ok = none.owner(id) + assert.False(t, ok) + none.release(id, a) +} + +func TestEnqueueAttach_AnotherWorkersRequestIsRefused(t *testing.T) { + owners := &commitRequestOwners{} + holder := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1), crOwners: owners} + other := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1), crOwners: owners} + req := attachReq("alice", 0) + + holder.EnqueueAttach(req) + other.EnqueueAttach(req) + + assert.Len(t, holder.eventQueue, 1) + assert.Empty(t, other.eventQueue, "the request belongs to the worker that accepted it first") + assert.Zero(t, other.inflightItems.Load()) +} + +// A dropped attach gives the request back unless the worker knows it otherwise. +func TestEnqueueAttach_AQueueFullDropGivesTheRequestBack(t *testing.T) { + owners := &commitRequestOwners{} + w := &BranchWorker{Log: logr.Discard(), Branch: "main", eventQueue: make(chan WorkItem, 1), crOwners: owners} + w.eventQueue <- WorkItem{} // saturate + + req := attachReq("alice", 0) + w.EnqueueAttach(req) + + _, owned := owners.owner(req.id()) + assert.False(t, owned) + assert.Empty(t, w.crQueuedAttaches) +} + +func ageCommitRequestOutcome(w *BranchWorker, id commitRequestID, by time.Duration) { + w.crOutcomesMu.Lock() + defer w.crOutcomesMu.Unlock() + entry := w.crOutcomes[id] + entry.resolvedAt = entry.resolvedAt.Add(-by) + w.crOutcomes[id] = entry +} + +func TestCommitRequestPhase_Held(t *testing.T) { + assert.True(t, PhaseCollectingWindow.Held()) + assert.True(t, PhaseWaitingForPush.Held()) + assert.False(t, PhaseWaitingForWindow.Held()) + assert.False(t, PhaseWaitingForWorker.Held()) + assert.False(t, CommitRequestPhase("").Held()) +} diff --git a/internal/git/git.go b/internal/git/git.go index 3e4f1529..7fc97eca 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -98,9 +98,20 @@ func PrepareBranch( ctx context.Context, repoURL, repoPath, targetBranchName string, auth []gitclient.Option, +) (*PullReport, error) { + return PrepareBranchFrom(ctx, repoURL, repoPath, targetBranchName, "", auth) +} + +// PrepareBranchFrom is PrepareBranch with an explicit parent branch for a target branch the remote +// does not carry; empty means the remote's default branch. See SmartFetchFrom. +func PrepareBranchFrom( + ctx context.Context, + repoURL, repoPath, targetBranchName, parentBranch string, + auth []gitclient.Option, ) (*PullReport, error) { logger := log.FromContext(ctx) - logger.Info("Preparing branch for operations", "url", repoURL, "path", repoPath, "branch", targetBranchName) + logger.Info("Preparing branch for operations", "url", repoURL, "path", repoPath, "branch", targetBranchName, + "parentBranch", parentBranch) // Ensure the directory exists if err := os.MkdirAll(filepath.Dir(repoPath), 0750); err != nil { @@ -136,7 +147,7 @@ func PrepareBranch( } targetBranch := plumbing.NewBranchReferenceName(targetBranchName) - pullReport, err := syncToRemote(ctx, repo, targetBranch, auth) + pullReport, err := syncToRemoteFn(ctx, repo, targetBranch, parentBranch, auth) if err != nil { return nil, err } @@ -366,13 +377,24 @@ func syncToRemote( repo *git.Repository, branch plumbing.ReferenceName, auth []gitclient.Option, +) (*PullReport, error) { + return syncToRemoteFrom(ctx, repo, branch, "", auth) +} + +// syncToRemoteFrom is syncToRemote with an explicit parent branch; see SmartFetchFrom. +func syncToRemoteFrom( + ctx context.Context, + repo *git.Repository, + branch plumbing.ReferenceName, + parent string, + auth []gitclient.Option, ) (*PullReport, error) { _, currentHash, err := GetCurrentBranch(repo) if err != nil { return nil, fmt.Errorf("unexpected fail to read HEAD: %w", err) } - availableBranch, err := SmartFetch(ctx, repo, branch, auth) + availableBranch, err := SmartFetchFrom(ctx, repo, branch, parent, auth) if err != nil { return nil, fmt.Errorf("failed to fetch: %w", err) } @@ -384,7 +406,11 @@ func syncToRemote( } remoteExists := availableBranch.Short() == branch.Short() - return createPullReport(branch.Short(), currentHash, newHash, remoteExists, false), nil + report := createPullReport(branch.Short(), currentHash, newHash, remoteExists, false) + if !remoteExists { + report.ParentBranch = availableBranch.Short() + } + return report, nil } // Failed to fetch from both sources, so let's configure head to be unborn at targetbranch. diff --git a/internal/git/git_atomic_push.go b/internal/git/git_atomic_push.go index f190ca5c..a921eb30 100644 --- a/internal/git/git_atomic_push.go +++ b/internal/git/git_atomic_push.go @@ -104,6 +104,22 @@ type RemoteMovedError struct { Missing bool } +// RepositoryNotEmptyError reports a push planned against an empty repository that is no longer +// empty: the cycle's root is the zero hash, the write branch is not advertised, and the +// advertisement carries other refs. Creating the branch now would make an orphan beside them, so +// the push is refused and the worker replays onto what discovery finds. +// +// It is its own type rather than a RemoteMovedError, whose Branch and Advertised mean "the root, +// compared": filling them with another branch's hash would make them lie. +type RepositoryNotEmptyError struct { + // Refs is how many hash refs the advertisement carried. + Refs int +} + +func (e *RepositoryNotEmptyError) Error() string { + return fmt.Sprintf("the repository is no longer empty: the advertisement carries %d refs", e.Refs) +} + func (e *RemoteMovedError) Error() string { if e.Missing { // Unchanged from the untyped error this replaces: it is the string in the logs operators @@ -190,8 +206,7 @@ func validatePushState( // Determine the "old" hash for the push command and validate state var oldHash = plumbing.ZeroHash remoteHash, found := refs[branch] - currentRootHash, rootFound := refs[rootBranch] - if !rootFound && !rootHash.IsZero() { + if _, rootFound := refs[rootBranch]; !rootFound && !rootHash.IsZero() { return pushPlan{}, &RemoteMovedError{ Branch: rootBranch, Expected: rootHash, @@ -200,36 +215,84 @@ func validatePushState( } } - if found { - // Target branch exists on remote - oldHash = remoteHash + if found && localHash == remoteHash { + logger.Info("remote already up2date", "branch", branchName, "hash", localHash) + return pushPlan{settled: PushUpToDate, head: remoteHash}, nil + } - // Check if we are already up2date - if localHash == remoteHash { - logger.Info("remote already up2date", "branch", branchName, "hash", localHash) - return pushPlan{settled: PushUpToDate, head: remoteHash}, nil + if rootHash.IsZero() && !found { + if n := countHashRefs(refs); n > 0 { + logger.Info("Planned against an empty repository that has gained refs", "branch", branchName, "refs", n) + return pushPlan{}, &RepositoryNotEmptyError{Refs: n} } + } - // Check if the remoteHash is what we based our work on - if rootHash != currentRootHash { - logger.Info("Remote branch not in expected state", "branch", branchName) - return pushPlan{}, &RemoteMovedError{ - Branch: rootBranch, - Expected: rootHash, - Advertised: currentRootHash, - } - } + if err := remoteMovedSinceBase(branch, rootBranch, rootHash, refs); err != nil { + logger.Info("Remote branch not in expected state", "branch", branchName, "root", rootBranch.Short()) + return pushPlan{}, err } - // Nothing on the remote and nothing local to send. The advertisement still answered the - // question the caller asked: this branch is not there. - if !found && localHash.IsZero() { + if found { + oldHash = remoteHash + } else if nothingToPublish(branch, localHash, rootBranch, rootHash) { + // The advertisement still answered the question the caller asked: this branch is not there. + logger.Info("Nothing to publish; the branch stays absent", "branch", branchName) return pushPlan{settled: PushNoBranch}, nil } + // Old stays zero for a new branch, so the server also refuses it if somebody creates it after + // this advertisement. return pushPlan{old: oldHash, new: localHash}, nil } +// countHashRefs counts the advertised hash refs other than HEAD; see advertisesNoRefs. +func countHashRefs(refs map[plumbing.ReferenceName]plumbing.Hash) int { + n := 0 + for name := range refs { + if name != plumbing.HEAD { + n++ + } + } + return n +} + +// nothingToPublish reports, for a branch the remote does not carry, that there is nothing to +// create it with: no local commit at all, or a new branch (rooted on another branch, its parent) +// whose local tip is still the parent's, which the caller has just confirmed. Creating it would +// publish nothing but a name, so it stays absent until a write has something to commit. +func nothingToPublish(branch plumbing.ReferenceName, localHash plumbing.Hash, + rootBranch plumbing.ReferenceName, rootHash plumbing.Hash, +) bool { + if localHash.IsZero() { + return true + } + return rootBranch != branch && localHash == rootHash +} + +// remoteMovedSinceBase reports a remote that is no longer where the local commits were based, or +// nil. Both checks hold whether or not the pushed branch exists, because the case they guard +// hardest is the one where it does not. +func remoteMovedSinceBase( + branch, rootBranch plumbing.ReferenceName, + rootHash plumbing.Hash, + refs map[plumbing.ReferenceName]plumbing.Hash, +) error { + // Somebody created the branch we believed absent. Pushing with Old = their commit would + // overwrite it with a commit that does not descend from it, on any server that accepts a + // non-fast-forward update; the root check below cannot see it, because the root is the parent. + if remoteHash, found := refs[branch]; found && rootBranch != branch { + return &RemoteMovedError{Branch: branch, Expected: plumbing.ZeroHash, Advertised: remoteHash} + } + + // The commits were built on rootHash. For an existing branch that is the branch itself; for a + // new one it is the parent, which must not have moved either: a branch born on a stale parent + // is born behind it. + if advertised := refs[rootBranch]; advertised != rootHash { + return &RemoteMovedError{Branch: rootBranch, Expected: rootHash, Advertised: advertised} + } + return nil +} + // performPush executes the packfile creation and push operation. func performPush( ctx context.Context, @@ -309,6 +372,12 @@ func performPush( // // The returned PushOutcome is an observation of the remote, made on the connection the push was // opening anyway. An error means nothing was observed; see PushOutcome for what each kind proves. +// afterPushValidation runs between the advertisement check and the upload. Nil in production; a test +// changes the remote here to make the server, not our client, refuse the upload. +// +//nolint:gochecknoglobals // a test seam, like pushAtomicFn +var afterPushValidation func() + func PushAtomic( ctx context.Context, repo *git.Repository, @@ -341,6 +410,9 @@ func PushAtomic( return PushOutcome{}, fmt.Errorf("failed to get current branch: %w", err) } + if afterPushValidation != nil { + afterPushValidation() + } if err := performPush(ctx, session, repo, rootHash, plan.new, plan.old, branch, logger); err != nil { return PushOutcome{}, err } diff --git a/internal/git/git_atomic_push_test.go b/internal/git/git_atomic_push_test.go index 433e089f..b3b9e29f 100644 --- a/internal/git/git_atomic_push_test.go +++ b/internal/git/git_atomic_push_test.go @@ -97,6 +97,71 @@ func TestAtomicPush_PushToOther(t *testing.T) { require.Equal(t, createdCommit, ref.Hash()) } +// TestAtomicPush_PushToOther_DetectsMovedRoot covers the push that creates a branch. Its root is +// the parent it was built from, and a parent that moved since the checkout must refuse the push: +// otherwise the new branch is born on a stale tree. +func TestAtomicPush_PushToOther_DetectsMovedRoot(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + serverPath := filepath.Join(tempDir, "server") + remoteURL := "file://" + serverPath + + localPath := filepath.Join(tempDir, "local") + remoteRepo := createBareRepo(t, serverPath) + commitA := simulateClientCommitOnDisk(t, remoteURL, "main", "README.md", "A") + + localRepo, worktree := initLocalRepo(t, localPath, remoteURL, "feature") + commitFileChange(t, worktree, localPath, "README.md", "ours, built on A") + + commitB := simulateClientCommitOnDisk(t, remoteURL, "main", "OTHER.md", "B") + + mainBranch := plumbing.NewBranchReferenceName("main") + featureBranch := plumbing.NewBranchReferenceName("feature") + outcome, err := PushAtomic(ctx, localRepo, commitA, mainBranch, nil) + require.Error(t, err) + assert.Empty(t, outcome.Kind, "an error means nothing was observed") + + var moved *RemoteMovedError + require.ErrorAs(t, err, &moved) + assert.Equal(t, RemoteMovedError{Branch: mainBranch, Expected: commitA, Advertised: commitB}, *moved) + + _, err = remoteRepo.Reference(featureBranch, true) + require.ErrorIs(t, err, plumbing.ErrReferenceNotFound, "the new branch must not be created") +} + +// TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch covers a branch somebody else created +// between our checkout and our push. We believed it absent, so we must not push over their commit. +func TestAtomicPush_PushToOther_DetectsConcurrentlyCreatedBranch(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + serverPath := filepath.Join(tempDir, "server") + remoteURL := "file://" + serverPath + + localPath := filepath.Join(tempDir, "local") + remoteRepo := createBareRepo(t, serverPath) + commitA := simulateClientCommitOnDisk(t, remoteURL, "main", "README.md", "A") + + localRepo, worktree := initLocalRepo(t, localPath, remoteURL, "feature") + commitFileChange(t, worktree, localPath, "README.md", "ours, built on A") + + theirs := simulateClientCommitOnDisk(t, remoteURL, "feature", "THEIRS.md", "theirs") + + mainBranch := plumbing.NewBranchReferenceName("main") + featureBranch := plumbing.NewBranchReferenceName("feature") + outcome, err := PushAtomic(ctx, localRepo, commitA, mainBranch, nil) + require.Error(t, err) + assert.Empty(t, outcome.Kind, "an error means nothing was observed") + + var moved *RemoteMovedError + require.ErrorAs(t, err, &moved) + assert.Equal(t, RemoteMovedError{Branch: featureBranch, Expected: plumbing.ZeroHash, Advertised: theirs}, *moved, + "we expected the branch to be absent") + + ref, err := remoteRepo.Reference(featureBranch, true) + require.NoError(t, err) + assert.Equal(t, theirs, ref.Hash(), "their commit must be untouched") +} + // TestAtomicPush_DetectsMissingBranch tests that push detects when remote branch doesn't exist anymore. func TestAtomicPush_DetectsMissingBranch(t *testing.T) { ctx := context.Background() diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 658768a2..e8a419b8 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -61,12 +61,21 @@ type ledgerFixture struct { // pending is the retained-write slice the event loop would own. The ops drive commit and push // directly, so they carry it here. pending []PendingWrite + // loop is the event loop, for the rows whose state lives on it. + loop *branchWorkerEventLoop } // newLedgerFixture builds that world. seeded chooses between an empty remote and one that already // has a commit on main, which is the difference between ledger rows 1 and 2. func newLedgerFixture(t *testing.T, slug string, seeded bool) *ledgerFixture { t.Helper() + return newLedgerFixtureOnBranch(t, slug, seeded, "main") +} + +// newLedgerFixtureOnBranch is newLedgerFixture with a worker for another branch than main, which +// the remote does not carry: the write branch of a target that has not written yet. +func newLedgerFixtureOnBranch(t *testing.T, slug string, seeded bool, branch string) *ledgerFixture { + t.Helper() projectRoot := t.TempDir() repoDir := filepath.Join(projectRoot, "repo.git") @@ -104,7 +113,7 @@ func newLedgerFixture(t *testing.T, slug string, seeded bool) *ledgerFixture { logr.Discard(), providerName, "default", - "main", + branch, RepoIdentity{URL: sim.RepoURL}, nil, BranchWorkerLimits{}, @@ -182,6 +191,9 @@ type ledgerOp struct { slug string // seeded starts the remote with a commit on main. seeded bool + // branch is the worker's write branch; empty is main. A branch the remote does not carry is a + // target on standby. + branch string // prime puts the fixture in the state the row describes. It runs BEFORE the measurement // starts, so a row about steady-state publication is not charged for the initial clone. prime func(f *ledgerFixture) @@ -359,6 +371,82 @@ func ledgerOperations() []ledgerOp { <-done }, }, + { + // H3: a target on standby, and a live write that changes nothing (a deployment's echo, + // here the delete of an object Git never had). The branch stays absent, and the push + // advertisement that confirms the parent is the only connection. + name: "13. standby, a live write that changes nothing", + slug: "standby-noop", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) }, + run: func(f *ledgerFixture) { + deleted := configMapEvent("never-written", "alice", "team-a") + deleted.Operation = "DELETE" + deleted.Object = nil + pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, []Event{deleted}) + require.NoError(f.t, err) + require.NoError(f.t, f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false)) + f.pending = append(f.pending, *pendingWrite) + f.push() + }, + }, + { + // H5: a target on standby whose parent moved since its checkout. The advertisement + // refuses the stale root client-side, one contention fetch replays onto the new tip, + // and the push creates the branch there. It is row 6, for a new branch. + name: "14. standby, the first write after the parent moved", + slug: "standby-parent-moved", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { + require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + f.contend("MOVED.md", "the parent moved\n") + }, + run: func(f *ledgerFixture) { f.publish("first") }, + }, + { + // §4.4: while the parent is missing, the worker's probe is one advertisement per + // deadline and never a fetch. How often it runs is test 4 of parent_recovery_test.go. + name: "15. parent probe while the parent is missing", + slug: "parent-probe", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { + f.worker.SetParentBranch("release") + f.loop = newBranchWorkerEventLoop(f.worker, time.Hour) + f.t.Cleanup(f.loop.stopTimers) + pendingWrite, err := f.worker.buildGroupedPendingWrite(f.worker.ctx, + []Event{configMapEvent("held", "alice", "team-a")}) + require.NoError(f.t, err) + err = f.worker.commitPendingWrites([]PendingWrite{*pendingWrite}, false) + require.ErrorIs(f.t, err, ErrParentBranchNotFound) + f.loop.noteParentUnavailable(err) + }, + run: func(f *ledgerFixture) { + f.worker.clock = func() time.Time { return time.Now().Add(time.Hour) } + f.loop.runParentProbe() + }, + }, + { + // R6: somebody creates the write branch between our advertisement and our upload, with + // the parent unmoved. The server refuses the upload; one advertisement reads both refs + // and finds the branch; the replay fetches it, rebuilds, and pushes onto it. It costs + // more than row 6, which is refused at the advertisement, before any upload. + name: "16. standby, the write branch created during our upload", + slug: "refused-upload", + seeded: true, + branch: "feature", + prime: func(f *ledgerFixture) { require.NoError(f.t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) }, + run: func(f *ledgerFixture) { + afterPushValidation = func() { + afterPushValidation = nil + simulateClientCommitOnDisk(f.t, f.repoDir, "feature", "THEIRS.md", "theirs\n") + } + defer func() { afterPushValidation = nil }() + f.publish("first") + }, + }, } } @@ -413,7 +501,11 @@ func TestGitRoundTripLedger(t *testing.T) { for _, op := range ops { row := ledgerRow{Operation: op.name} ok := t.Run(op.name, func(t *testing.T) { - f := newLedgerFixture(t, op.slug, op.seeded) + branch := op.branch + if branch == "" { + branch = "main" + } + f := newLedgerFixtureOnBranch(t, op.slug, op.seeded, branch) if op.prime != nil { op.prime(f) } @@ -435,19 +527,20 @@ func TestGitRoundTripLedger(t *testing.T) { // A harness that measures nothing measures nothing consistently, and a golden file is happy // to record that. Row 12 is meant to be zero; everything else is not. - for _, row := range rows[:len(rows)-1] { + const idleRow = "12. an idle target held across several commit windows" + for _, row := range rows { + if row.Operation == idleRow { + // Row 12 is a property, not a measurement: an idle target is silent. It is asserted + // here directly so it can never be "accepted" by regenerating the golden file. + require.Zero(t, row.Snapshot.connections(), + "an idle target must not talk to the Git host at all; it opened %s", row.Snapshot.exactBytes()) + continue + } require.Positive(t, row.Snapshot.connections(), "%s recorded no traffic at all, so the harness — not the code — is what this row is "+ "describing", row.Operation) } - // Row 12 is a property, not a measurement: an idle target is silent. It is asserted here - // directly so it can never be "accepted" by regenerating the golden file. - idle := rows[len(rows)-1] - require.Equal(t, "12. an idle target held across several commit windows", idle.Operation) - require.Zero(t, idle.Snapshot.connections(), - "an idle target must not talk to the Git host at all; it opened %s", idle.Snapshot.exactBytes()) - assertLedgerGolden(t, rows) } diff --git a/internal/git/git_smart_fetch.go b/internal/git/git_smart_fetch.go index 5458afb1..97b2d79e 100644 --- a/internal/git/git_smart_fetch.go +++ b/internal/git/git_smart_fetch.go @@ -21,12 +21,86 @@ import ( // Return values (example with target="refs/heads/feature"): // - "refs/heads/feature", nil: Target found on remote, fetched, ready to checkout. // - "refs/heads/main", nil: Target missing on remote, fell back to default branch. -// - "", nil: No valid branches found (empty repo). +// - "", nil: The repository is empty (see advertisesNoRefs). +// - "", ErrDefaultBranchUnresolved: Not empty, and no default branch to fall back to. func SmartFetch( ctx context.Context, repo *git.Repository, target plumbing.ReferenceName, // e.g. "refs/heads/feature" or "HEAD" auth []gitclient.Option, +) (plumbing.ReferenceName, error) { + return SmartFetchFrom(ctx, repo, target, "", auth) +} + +// ErrParentBranchNotFound reports that a configured parent branch is absent on the remote while +// the target branch is absent too, so there is nothing to create the target branch from. It is +// never reported for an omitted parent: that falls back to the remote's default branch, and to +// an unborn branch in an empty repository. +var ErrParentBranchNotFound = errors.New("parent branch not found on the remote") + +// ErrDefaultBranchUnresolved reports a repository that is not empty but whose default branch does +// not resolve to a branch it carries, while no parent is configured and the target branch is +// absent. Starting the target branch there would make an orphan, so it is refused exactly like a +// missing configured parent. DefaultBranchUnresolvedError names what HEAD points at. +var ErrDefaultBranchUnresolved = errors.New("the remote's default branch does not resolve to a branch") + +// DefaultBranchUnresolvedError is ErrDefaultBranchUnresolved with the evidence: the branch the +// remote's HEAD names, or empty when the advertisement carries no symbolic HEAD at all. +// +// The resolution itself follows go-git. Remote.List rewrites a hash-only HEAD (a server without +// the symref capability, or a detached HEAD) into a symbolic one before returning: to master when +// master has that hash, otherwise to the alphabetically first branch that has it. So a HEAD this +// error reports is one even that heuristic could not place. +type DefaultBranchUnresolvedError struct { + // Head is the short name of the branch HEAD points at; empty when there is no symbolic HEAD. + Head string +} + +func (e *DefaultBranchUnresolvedError) Error() string { + if e.Head == "" { + return "the remote has no default branch" + } + return fmt.Sprintf("the remote's default branch (HEAD -> %s) does not exist", e.Head) +} + +// Is makes the error match ErrDefaultBranchUnresolved. +func (e *DefaultBranchUnresolvedError) Is(target error) bool { + return target == ErrDefaultBranchUnresolved +} + +// advertisesNoRefs reports an empty repository: an advertisement with no hash refs at all, no +// branch, no tag, nothing else. A symbolic HEAD pointing at an unborn branch does not count, and a +// repository with only tags is not empty. This is the one definition discovery, publication and +// the observation share. +func advertisesNoRefs(refs []*plumbing.Reference) bool { + for _, ref := range refs { + if ref.Type() == plumbing.HashReference && ref.Name() != plumbing.HEAD { + return false + } + } + return true +} + +// symbolicHeadTarget is the short name of the branch the advertisement's HEAD points at, or empty. +func symbolicHeadTarget(refs []*plumbing.Reference) string { + for _, ref := range refs { + if ref.Name() == plumbing.HEAD && ref.Type() == plumbing.SymbolicReference { + return ref.Target().Short() + } + } + return "" +} + +// SmartFetchFrom is SmartFetch with an explicit parent: the branch to fall back to when the target +// branch is absent. Empty means the remote's default branch. A configured parent that is absent is +// an error rather than a fallback, so a typo can never start an orphan branch; it does not matter +// while the target branch exists. +func SmartFetchFrom( + ctx context.Context, + repo *git.Repository, + target plumbing.ReferenceName, + parent string, + auth []gitclient.Option, ) (plumbing.ReferenceName, error) { remoteName := "origin" remote, err := repo.Remote(remoteName) @@ -39,12 +113,17 @@ func SmartFetch( if err != nil { return "", err } - if len(refs) == 0 { + if len(refs) == 0 && parent == "" { return "", nil } // 2. Analyze: Find default branch and check target existence - defaultFull, defaultShort, targetExists := analyzeRemoteRefs(ctx, refs, target.String()) + headFull, headShort, targetExists := analyzeRemoteRefs(ctx, refs, target.String()) + + defaultFull, defaultShort, err := fallbackBranch(refs, parent, headFull, headShort, targetExists) + if err != nil { + return "", err + } // 3. Plan: Build RefSpecs based on analysis refSpecs := buildSmartRefSpecs(remoteName, defaultFull, defaultShort, target, targetExists) @@ -75,12 +154,45 @@ func SmartFetch( } } - // 5. Repair: Fix local symbolic HEAD - repairRemoteSymbolicHead(repo, remoteName, defaultShort) + // 5. Repair: Fix local symbolic HEAD. It mirrors the remote's HEAD, never a configured parent. + repairRemoteSymbolicHead(repo, remoteName, headShort) return result, nil } +// fallbackBranch is the branch to fall back on, and to fetch as the safety net: the configured +// parent, or the remote's default branch when none is configured. A parent the remote does not +// carry is an error only while the target is absent too: a configured one is +// ErrParentBranchNotFound, and an unresolved default branch in a repository that is not empty is +// ErrDefaultBranchUnresolved, because an unborn start there would be an orphan beside the refs it +// ignored. +func fallbackBranch( + refs []*plumbing.Reference, parent, headFull, headShort string, targetExists bool, +) (string, string, error) { + if parent == "" { + if headFull == "" && !targetExists && !advertisesNoRefs(refs) { + return "", "", &DefaultBranchUnresolvedError{Head: symbolicHeadTarget(refs)} + } + return headFull, headShort, nil + } + full, short := configuredParent(refs, parent) + if full == "" && !targetExists { + return "", "", fmt.Errorf("%w: %q", ErrParentBranchNotFound, parent) + } + return full, short, nil +} + +// configuredParent finds a configured parent branch in the advertisement, or returns empty names. +func configuredParent(refs []*plumbing.Reference, parent string) (string, string) { + full := plumbing.NewBranchReferenceName(parent) + for _, ref := range refs { + if ref.Name() == full && ref.Type() == plumbing.HashReference { + return full.String(), parent + } + } + return "", "" +} + func listRemoteRefs(remote *git.Remote, auth []gitclient.Option) ([]*plumbing.Reference, error) { refs, err := remote.List(&git.ListOptions{ClientOptions: auth}) if errors.Is(err, transport.ErrEmptyRemoteRepository) { diff --git a/internal/git/parent_recovery.go b/internal/git/parent_recovery.go new file mode 100644 index 00000000..b91c5b27 --- /dev/null +++ b/internal/git/parent_recovery.go @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "errors" + "fmt" + "time" + + "github.com/go-git/go-git/v6/plumbing" + + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// This file is the worker's recovery from a parent branch that is missing or does not resolve: +// work it could not write then, and the obligation to write it once the parent exists. +// +// An observation flip is not an obligation. "The parent is Found again" says nothing about whether +// the writes held back meanwhile have landed: the first attempt after it can fail transiently, the +// push timer is stopped by a failed push, the refresher skips a branch that holds work, and with +// --git-refresh-interval=0 there is no refresher at all. So the worker latches an obligation the +// moment a cycle fails on the parent, and clears it only when the work is published. +// +// The obligation has two halves: +// +// - Retained writes: the loop's pendingWrites. Nothing extra is needed to remember them; the +// obligation is open while any are held, and pushPending publishing them is what closes it. +// - Dropped writes, per (GitTarget, collection) scope: a live window or a resync the worker had +// to drop because the parent was missing. The worker cannot re-derive what the cluster holds, +// so it asks the controller for a snapshot (SnapshotRequestSeq), and a scope clears only when +// a resync of it is published: pushed, or settled as a no-op. A successful resync reply alone +// is not enough, because its write may still be waiting for the push. Scopes are per +// collection because a ConfigMap snapshot of target A proves nothing about a Deployment of A, +// or about target B on the same worker. +// +// The worker probes for the parent itself, on the event loop, with one advertisement per deadline: +// 10s, doubling, capped at 5m, shared by every target on the worker. Until the parent is found, +// nothing else may spend a connection on it: a refresh tick before the deadline is a no-op, and a +// live write or resync fails at once without a fetch. Reconciles never probe. + +const ( + parentProbeInitialBackoff = 10 * time.Second + parentProbeMaxBackoff = 5 * time.Minute + parentProbeBackoffFactor = 2 +) + +// errAwaitingParentProbe is the answer to a cycle that would fetch for a parent already known to be +// missing, before the next probe is due. It is a missing parent as far as every caller is +// concerned, and costs no connection. +var errAwaitingParentProbe = fmt.Errorf("%w (unchanged since the last look; waiting for the next probe)", + ErrParentBranchNotFound) + +// isParentUnavailable reports an error caused by the parent a new write branch would be created +// from: a configured parent that is absent, or a default branch that does not resolve. +func isParentUnavailable(err error) bool { + return errors.Is(err, ErrParentBranchNotFound) || errors.Is(err, ErrDefaultBranchUnresolved) +} + +// recoveryScope is one slice of a GitTarget whose writes were dropped. A zero collection is a write +// whose collection is unknown, which any resync of the target covers. +type recoveryScope struct { + target itypes.ResourceReference + collection itypes.CollectionKey +} + +// parentRecovery is the loop's half of the obligation. Loop-goroutine only. +type parentRecovery struct { + // active is the latch. + active bool + // found records that the parent was seen since the latch was set (or since it was last lost). + found bool + // gen is the parent generation the latch was set under; a change makes the probe due at once. + gen uint64 + + // scopes are the dropped writes still owed a published snapshot. + scopes map[recoveryScope]struct{} + // awaitingPush are scopes a resync has re-derived into a commit that is not pushed yet. + awaitingPush map[recoveryScope]struct{} + + backoff time.Duration + nextProbeAt time.Time + timer *time.Timer +} + +// now is the loop's clock, injectable so the probe schedule can be tested without sleeping. +func (w *BranchWorker) now() time.Time { + if w.clock != nil { + return w.clock() + } + return time.Now() +} + +// noteParentUnavailable latches the obligation when err was caused by the parent, and remembers +// the scopes whose writes were dropped for it. Any other error is not this obligation's concern. +func (l *branchWorkerEventLoop) noteParentUnavailable(err error, scopes ...recoveryScope) { + if !isParentUnavailable(err) { + return + } + r := &l.recovery + switch { + case !r.active: + r.active = true + r.backoff = parentProbeInitialBackoff + r.nextProbeAt = l.w.now().Add(r.backoff) + r.gen = l.w.parentSnapshot().gen + case r.found: + // Found, then lost again on the attempt that followed: wait a whole backoff before asking. + r.nextProbeAt = l.w.now().Add(r.backoff) + } + r.found = false + for _, scope := range scopes { + if r.scopes == nil { + r.scopes = map[recoveryScope]struct{}{} + } + r.scopes[scope] = struct{}{} + // A snapshot of this scope that is still waiting for its push was taken before this write + // was dropped, so publishing it proves nothing about this write. The scope needs a new one. + delete(r.awaitingPush, scope) + } + l.armRecoveryTimer() + l.publishRecovery() +} + +// windowScopes are the scopes of a dropped live window: one per collection its events came from. +func windowScopes(namespace, name string, events []Event) []recoveryScope { + target := itypes.NewResourceReference(name, namespace) + seen := map[itypes.CollectionKey]bool{} + var scopes []recoveryScope + for i := range events { + collection := events[i].SourceCollection + if !seen[collection] { + seen[collection] = true + scopes = append(scopes, recoveryScope{target: target, collection: collection}) + } + } + return scopes +} + +// atomicScope is the scope of a dropped atomic write. +func atomicScope(request *WriteRequest) recoveryScope { + name, namespace := atomicRefusalTarget(request) + return recoveryScope{target: itypes.NewResourceReference(name, namespace), collection: request.sourceCollection()} +} + +// resyncScope is the scope of a resync. +func resyncScope(req *ResyncRequest) recoveryScope { + return recoveryScope{ + target: itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), + collection: req.refusalCollection(), + } +} + +// noteResyncApplied records that a resync of one scope succeeded. A resync succeeds only after +// fetching the parent, so it is also proof the parent is back. A no-op settles the scope now; one +// that committed settles it when its commit is pushed. +func (l *branchWorkerEventLoop) noteResyncApplied(namespace, name string, collection itypes.CollectionKey, + committed bool) { + r := &l.recovery + if !r.active { + return + } + target := itypes.NewResourceReference(name, namespace) + for scope := range r.scopes { + if scope.target != target || !resyncCovers(collection, scope.collection) { + continue + } + if committed { + if r.awaitingPush == nil { + r.awaitingPush = map[recoveryScope]struct{}{} + } + r.awaitingPush[scope] = struct{}{} + } else { + delete(r.scopes, scope) + } + } + l.closeRecoveryIfDone() +} + +// resyncCovers reports whether a resync of one collection re-derives a dropped scope: the same +// collection, or either side unknown (a resync of the whole target, or a write whose collection is +// not known). +func resyncCovers(resynced, dropped itypes.CollectionKey) bool { + var unknown itypes.CollectionKey + return resynced == dropped || resynced == unknown || dropped == unknown +} + +// noteRecoveryPublished runs after every successful push: the scopes whose snapshot it carried are +// settled, and the retained writes it published were the other half. +func (l *branchWorkerEventLoop) noteRecoveryPublished() { + r := &l.recovery + if !r.active { + return + } + for scope := range r.awaitingPush { + delete(r.scopes, scope) + } + r.awaitingPush = nil + l.closeRecoveryIfDone() +} + +// closeRecoveryIfDone clears the latch once nothing is owed. +func (l *branchWorkerEventLoop) closeRecoveryIfDone() { + r := &l.recovery + if !r.active || len(l.pendingWrites) > 0 || len(r.scopes) > 0 { + return + } + if r.timer != nil { + r.timer.Stop() + } + *r = parentRecovery{} + l.publishRecovery() +} + +// probeDue reports whether the next background look at the remote may be spent now. +func (l *branchWorkerEventLoop) probeDue() bool { + r := &l.recovery + return r.active && (!l.w.now().Before(r.nextProbeAt) || r.gen != l.w.parentSnapshot().gen) +} + +// runParentProbe is the deadline: while the parent is missing it spends one advertisement; once it +// is found it retries the retained writes and asks again for the snapshots still owed. Whatever +// remains open is scheduled for the next deadline. +func (l *branchWorkerEventLoop) runParentProbe() { + r := &l.recovery + if !r.active { + return + } + if gen := l.w.parentSnapshot().gen; gen != r.gen { + r.gen, r.found, r.backoff = gen, false, parentProbeInitialBackoff + } + if !r.found { + missing, err := l.probeParent() + if err != nil || missing { + if err != nil { + l.w.Log.V(1).Info("Parent branch probe failed", "branch", l.w.Branch, "error", err.Error()) + } + l.scheduleNextProbe() + return + } + r.found = true + r.backoff = parentProbeInitialBackoff + l.publishRecovery() + l.w.Log.Info("The parent branch is on the remote again; recovering held-back work", + "branch", l.w.Branch, "parentBranch", l.w.ParentBranch(), + "pendingWrites", len(l.pendingWrites), "scopes", len(r.scopes)) + } + if len(l.pendingWrites) > 0 { + l.pushPending() + } + if r.active && r.found { + for target := range l.recoveryTargets() { + l.w.bumpSnapshotRequest(target) + } + } + if r.active { + l.scheduleNextProbe() + } +} + +// recoveryTargets are the GitTargets owed a snapshot. +func (l *branchWorkerEventLoop) recoveryTargets() map[itypes.ResourceReference]struct{} { + targets := map[itypes.ResourceReference]struct{}{} + for scope := range l.recovery.scopes { + if _, waiting := l.recovery.awaitingPush[scope]; !waiting { + targets[scope.target] = struct{}{} + } + } + return targets +} + +// scheduleNextProbe doubles the backoff and moves the deadline that far on: the first deadline is +// parentProbeInitialBackoff after the latch, the next twice that after it, up to the cap. +func (l *branchWorkerEventLoop) scheduleNextProbe() { + r := &l.recovery + r.backoff = min(parentProbeBackoffFactor*r.backoff, parentProbeMaxBackoff) + r.nextProbeAt = l.w.now().Add(r.backoff) + l.armRecoveryTimer() + l.publishRecovery() +} + +// probeParent reads one advertisement and records what it says. It reports whether the parent is +// still missing; a write branch that now exists is not. +func (l *branchWorkerEventLoop) probeParent() (bool, error) { + w := l.w + provider, err := w.getGitProvider(w.ctx) + if err != nil { + return true, fmt.Errorf("get GitProvider: %w", err) + } + auth, err := getAuthFromSecret(w.ctx, w.Client, provider, w.sshHostKeys, w.credentialPolicy) + if err != nil { + return true, fmt.Errorf("get auth: %w", err) + } + parent := w.parentSnapshot() + advertisement, err := advertiseRemoteBranchFn( + w.repo.URL, + plumbing.NewBranchReferenceName(w.Branch), + parent.name, + auth, + ) + if err != nil { + return true, fmt.Errorf("read the remote advertisement: %w", err) + } + return w.recordAdvertisement(advertisement, parent.name), nil +} + +func (l *branchWorkerEventLoop) armRecoveryTimer() { + r := &l.recovery + if r.timer != nil { + r.timer.Stop() + r.timer = nil + } + if !r.active { + return + } + r.timer = time.NewTimer(max(r.nextProbeAt.Sub(l.w.now()), 0)) +} + +func (l *branchWorkerEventLoop) recoveryTimerC() <-chan time.Time { + if l.recovery.timer == nil { + return nil + } + return l.recovery.timer.C +} + +// publishRecovery mirrors the latch for the readers outside the loop: the controller, and the +// fetch gate the write path consults. +func (l *branchWorkerEventLoop) publishRecovery() { + r := &l.recovery + l.w.parentRecoveryOpen.Store(r.active) + l.w.parentRecoveryFound.Store(r.active && r.found) + if r.active && !r.found { + l.w.parentProbeHold.Store(&parentProbeHold{until: r.nextProbeAt, gen: r.gen}) + } else { + l.w.parentProbeHold.Store(nil) + } +} + +// parentProbeHold is the window in which a missing parent is not looked for again. +type parentProbeHold struct { + until time.Time + gen uint64 +} + +// awaitingParentProbe reports that the parent is known missing and the next probe is not due, so +// a cycle that would fetch for it must not. A parent change lifts it. +func (w *BranchWorker) awaitingParentProbe() bool { + hold := w.parentProbeHold.Load() + return hold != nil && hold.gen == w.parentSnapshot().gen && w.now().Before(hold.until) +} + +// bumpSnapshotRequest asks the controller for a fresh snapshot of one GitTarget. +func (w *BranchWorker) bumpSnapshotRequest(target itypes.ResourceReference) { + w.snapshotRequestsMu.Lock() + defer w.snapshotRequestsMu.Unlock() + if w.snapshotRequests == nil { + w.snapshotRequests = map[itypes.ResourceReference]uint64{} + } + w.snapshotRequests[target]++ +} + +// BumpSnapshotRequestForTest asks for a snapshot of the GitTarget, as the recovery probe does, for +// tests in another package. +func (w *BranchWorker) BumpSnapshotRequestForTest(target itypes.ResourceReference) { + w.bumpSnapshotRequest(target) +} + +// SnapshotRequestSeq is how many times the worker has asked for a fresh snapshot of the GitTarget, +// to re-derive writes it dropped while the parent branch was missing. The controller forces one +// recheck each time it rises, and remembers the value it acted on. +func (w *BranchWorker) SnapshotRequestSeq(target itypes.ResourceReference) uint64 { + w.snapshotRequestsMu.Lock() + defer w.snapshotRequestsMu.Unlock() + return w.snapshotRequests[target] +} + +// ParentRecovery reports whether the worker still owes work held back by a missing parent (the +// first result), and whether the parent has been found since (the second). +func (w *BranchWorker) ParentRecovery() (bool, bool) { + return w.parentRecoveryOpen.Load(), w.parentRecoveryFound.Load() +} diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go new file mode 100644 index 00000000..3c164920 --- /dev/null +++ b/internal/git/parent_recovery_test.go @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// Work a missing parent branch held back is an obligation, not an observation: it stays open until +// the work is published, so a transient failure after the parent reappears cannot strand it, and +// it is driven by the worker on its own deadline, whatever the refresh configuration. These pin +// the obligation, its per-scope bookkeeping, and the probe budget. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + gitclient "github.com/go-git/go-git/v6/plumbing/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// testClock is a clock the test moves by hand. +type testClock struct { + mu sync.Mutex + now time.Time +} + +func useTestClock(w *BranchWorker) *testClock { + c := &testClock{now: time.Now()} + w.clock = c.Now + return c +} + +func (c *testClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *testClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +// failNextSync makes the next sync the worker attempts fail as a network error would. +func failNextSync(t *testing.T) { + t.Helper() + original := syncToRemoteFn + failed := false + syncToRemoteFn = func(ctx context.Context, repo *gogit.Repository, target plumbing.ReferenceName, + parent string, auth []gitclient.Option) (*PullReport, error) { + if !failed { + failed = true + return nil, errors.New("connection reset by peer") + } + return original(ctx, repo, target, parent, auth) + } + t.Cleanup(func() { syncToRemoteFn = original }) +} + +// Test 1. Writes retained while the parent is missing are published once it exists, through the +// worker's own deadline, even when the first attempt after the parent reappears fails. No cluster +// edit is involved, and it works with the refresher on or off. +func TestParentRecovery_RetainedWritesLandAfterAFailedFirstAttempt(t *testing.T) { + for name, viaRefresh := range map[string]bool{"refresh off": false, "refresh on": true} { + t.Run(name, func(t *testing.T) { + f := newRealServerNewBranch( + t, + "recovery-retained-"+map[bool]string{false: "timer", true: "refresh"}[viaRefresh], + func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "release", "main") + }, + ) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + deadline := func() { + if viaRefresh { + f.loop.handleRefreshRequest(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + return + } + f.loop.runParentProbe() // what the probe timer does + } + + f.loop.lastPushAt = time.Now() + liveWrite(f.loop, "cm1") + require.Len(t, f.loop.pendingWrites, 1) + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") // the parent goes away + f.loop.pushPending() + require.Len(t, f.loop.pendingWrites, 1, "kept") + open, found := f.worker.ParentRecovery() + require.True(t, open) + require.False(t, found) + + if viaRefresh { + before := f.mark() + deadline() + assert.Zero(t, f.mark().since(before).connections(), "a refresh tick before the deadline costs nothing") + } + + release := simulateClientCommitOnDisk(t, f.repoDir, "release", "RELEASE.md", "back\n") + failNextSync(t) + clock.advance(parentProbeMaxBackoff) + deadline() + + assert.True(t, f.ref("feature").IsZero(), "the first attempt failed") + require.Len(t, f.loop.pendingWrites, 1, "and the work is still held") + open, found = f.worker.ParentRecovery() + require.True(t, open, "the obligation outlives a failed attempt") + assert.True(t, found) + + clock.advance(parentProbeMaxBackoff) + deadline() + + tip := f.ref("feature") + require.False(t, tip.IsZero(), "the next deadline published it, with no cluster edit") + assert.Equal(t, []plumbing.Hash{release}, f.parentOf(tip)) + assert.Empty(t, f.loop.pendingWrites) + open, _ = f.worker.ParentRecovery() + assert.False(t, open, "published: nothing is owed") + }) + } +} + +func refreshTarget() itypes.ResourceReference { + return itypes.NewResourceReference(newBranchTarget, "default") +} + +// Test 2. Writes dropped for two targets on one worker. Each scope clears only when a resync of it +// is published; a resync that committed but is not pushed yet does not clear it, and a target +// whose resync failed is asked again on the next deadline. +func TestParentRecovery_DroppedWritesClearPerScopeWhenPublished(t *testing.T) { + f := newNewBranchFixture(t, nil) + clock := useTestClock(f.worker) + createPlainGitTarget(t, f.worker, "other-target", "other") + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + + a := itypes.NewResourceReference(newBranchTarget, "default") + b := itypes.NewResourceReference("other-target", "default") + collectionA := itypes.CollectionKey{Resource: "configmaps", Namespace: "default"} + collectionB := itypes.CollectionKey{Resource: "configmaps", Namespace: "default", LabelSelector: "team=b"} + write := func(target string, collection itypes.CollectionKey, name string) { + event := configMapTargetEvent(name, "alice", target) + event.SourceCollection = collection + loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + } + resync := func(target itypes.ResourceReference, collection itypes.CollectionKey, name string) error { + scope := ResyncScope{Collection: collection} + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCM(name, "blue")}, + ResourceVersion: "1", + GitTargetName: target.Name, + GitTargetNamespace: target.Namespace, + Scope: &scope, + Result: make(chan ResyncResult, 1), + } + loop.applyResync(req) + return (<-req.Result).Err + } + + write(newBranchTarget, collectionA, "a1") + write("other-target", collectionB, "b1") + assert.Empty(t, loop.pendingWrites, "both were dropped") + assert.Len(t, loop.recovery.scopes, 2) + + f.pushToRelease("RELEASE.md", "release\n") + clock.advance(parentProbeMaxBackoff) + loop.runParentProbe() + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(a)) + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(b)) + + loop.lastPushAt = time.Now() // cooldown: A's resync commits, and its push waits + require.NoError(t, resync(a, collectionA, "a1")) + require.Len(t, loop.pendingWrites, 1) + assert.Contains(t, loop.recovery.scopes, recoveryScope{target: a, collection: collectionA}, + "a resync that is not pushed yet does not clear its scope") + + failNextSync(t) + require.Error(t, resync(b, collectionB, "b1")) + + clock.advance(parentProbeMaxBackoff) + loop.runParentProbe() // publishes A's commit, and asks for B again + + assert.Empty(t, loop.pendingWrites) + assert.NotContains(t, loop.recovery.scopes, recoveryScope{target: a, collection: collectionA}, "published") + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(a), "A is not asked again") + assert.Equal(t, uint64(2), f.worker.SnapshotRequestSeq(b), "B is") + + require.NoError(t, resync(b, collectionB, "b1")) + loop.pushPending() + open, _ := f.worker.ParentRecovery() + assert.False(t, open, "every scope is published") +} + +// Test 4. However many targets share the worker, and however often refreshes tick and live writes +// arrive, a parent that stays missing is asked about at most once per deadline of the shared +// schedule, and never fetched. That holds with work retained too: there, every write first wants to +// rebuild the retained writes, which is a fetch the deadline must hold back as well. +func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { + for name, retained := range map[string]bool{"dropped writes": false, "retained writes": true} { + t.Run(name, func(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + slug := map[bool]string{false: "recovery-probe-budget", true: "recovery-probe-budget-retained"}[retained] + f := newRealServerNewBranch(t, slug, func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "release", "main") + }) + clock := useTestClock(f.worker) + createPlainGitTarget(t, f.worker, "second", "second") + createPlainGitTarget(t, f.worker, "third", "third") + targets := []string{newBranchTarget, "second", "third"} + + if retained { + f.worker.SetParentBranch("release") + require.NoError(t, f.worker.ensureRepositoryInitialized(f.worker.ctx)) + f.loop.lastPushAt = time.Now() + liveWrite(f.loop, "first") + require.Len(t, f.loop.pendingWrites, 1) + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") + f.loop.pushPending() // the push that finds the parent gone + require.True(t, f.worker.replayRequired(), "the retained writes wait for a rebuild") + } else { + f.worker.SetParentBranch("missing") + liveWrite(f.loop, "first") // the one fetch that finds the parent missing + } + open, _ := f.worker.ParentRecovery() + require.True(t, open) + + fetches := totalFetches(t, reader, f.worker) + before := f.mark() + probes := 0 + const window = 10 * time.Minute + for elapsed := time.Duration(0); elapsed < window; elapsed += 2 * time.Second { + clock.advance(2 * time.Second) + if f.loop.probeDue() { + probes++ + f.loop.runParentProbe() // the timer + } + for _, target := range targets { + f.loop.handleRefreshRequest(&RefreshRequest{ + Target: itypes.NewResourceReference(target, "default"), MaxAge: time.Nanosecond, + }) + event := configMapTargetEvent("cm-"+target, "alice", target) + f.loop.handleQueueItem( + WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}, + ) + } + } + + // 10s, then doubling to the 5m cap: deadlines at 10s, 30s, 70s, 150s and 310s. + assert.Equal(t, 5, probes) + assert.LessOrEqual(t, f.mark().since(before).connections(), int64(probes), + "one advertisement per deadline, whatever else happens") + assert.Equal(t, fetches, totalFetches(t, reader, f.worker), "and no fetch at all") + }) + } +} + +// A second outage while a recovery snapshot waits for its push. The write dropped during it was +// made after that snapshot was taken, so publishing the snapshot must not settle its scope: the +// worker asks for another one, and the obligation stays open until that is published. +func TestParentRecovery_ASecondOutageKeepsTheWorkDroppedInIt(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-second-outage", func(dir string) { + simulateClientCommitOnDisk(t, dir, "main", "README.md", "main\n") + }) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + collection := itypes.CollectionKey{Resource: "configmaps", Namespace: "default"} + write := func(name string) { + event := configMapTargetEvent(name, "alice", newBranchTarget) + event.SourceCollection = collection + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{Events: []Event{event}, CommitMode: CommitModePerEvent}}) + } + + write("cm1") // dropped: release does not exist + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(parentProbeMaxBackoff) + f.loop.runParentProbe() + seq := f.worker.SnapshotRequestSeq(refreshTarget()) + require.Equal(t, uint64(1), seq) + + f.loop.lastPushAt = time.Now() // the snapshot commits, and its push waits for the cooldown + req := &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCM("cm1", "blue")}, + ResourceVersion: "1", GitTargetName: newBranchTarget, GitTargetNamespace: "default", + Scope: &ResyncScope{Collection: collection}, + Result: make(chan ResyncResult, 1), + } + f.loop.applyResync(req) + require.NoError(t, (<-req.Result).Err) + require.Len(t, f.loop.recovery.awaitingPush, 1) + + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") // the second outage + f.loop.pushPending() + write("cm2") // dropped during it + require.Len(t, f.loop.pendingWrites, 1, "cm2 was dropped; only the snapshot is retained") + + gitIn(t, f.repoDir, "branch", "release", "main") + clock.advance(parentProbeMaxBackoff) + f.loop.runParentProbe() + + require.False(t, f.ref("feature").IsZero(), "the snapshot was published") + open, _ := f.worker.ParentRecovery() + assert.True(t, open, "cm2 still needs a snapshot") + assert.Greater(t, f.worker.SnapshotRequestSeq(refreshTarget()), seq, "and the worker asks for one") +} + +// Test 5. An unresolved default branch, once repaired, recovers through the same path: the probe +// finds it and asks for the snapshot the dropped write needs. +func TestParentRecovery_ARepairedRemoteHeadAsksForTheSnapshot(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-dangling-head", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "symbolic-ref", "HEAD", "refs/heads/master") + }) + clock := useTestClock(f.worker) + + liveWrite(f.loop, "cm1") + assert.True(t, f.ref("feature").IsZero()) + + gitIn(t, f.repoDir, "symbolic-ref", "HEAD", "refs/heads/main") + clock.advance(parentProbeMaxBackoff) + f.loop.runParentProbe() + + assert.Equal(t, uint64(1), f.worker.SnapshotRequestSeq(refreshTarget())) + open, found := f.worker.ParentRecovery() + assert.True(t, open, "open until the snapshot is published") + assert.True(t, found) +} + +func TestParentRecovery_AParentChangeMakesTheProbeDue(t *testing.T) { + f := newNewBranchFixture(t, nil) + useTestClock(f.worker) + f.worker.SetParentBranch("release") + loop := newBranchWorkerEventLoop(f.worker, 0) + defer loop.stopTimers() + liveWrite(loop, "cm1") + require.False(t, loop.probeDue()) + require.True(t, f.worker.awaitingParentProbe()) + + f.worker.SetParentBranch("other") + + assert.True(t, loop.probeDue()) + assert.False(t, f.worker.awaitingParentProbe(), "a new parent is looked for at once") +} + +// A CommitRequest that rides a write held back by a missing parent stays held through the whole +// recovery, however long it takes, and resolves Committed when the work lands. This runs the real +// event loop on its own goroutine; the clock stands in for a recovery far longer than the bound the +// controller used to fail requests on (attachTimeout + maxDuration + 120s). +func TestParentRecovery_AHeldCommitRequestIsCommittedAfterALongRecovery(t *testing.T) { + f := newRealServerNewBranch(t, "recovery-held-commit-request", func(repoDir string) { + simulateClientCommitOnDisk(t, repoDir, "main", "README.md", "main\n") + gitIn(t, repoDir, "branch", "release", "main") + }) + clock := useTestClock(f.worker) + f.worker.SetParentBranch("release") + vanished := false + beforePushAdmission = func() { + if !vanished { // the parent disappears right before the first push + vanished = true + gitIn(t, f.repoDir, "update-ref", "-d", "refs/heads/release") + } + } + t.Cleanup(func() { beforePushAdmission = nil }) + require.NoError(t, f.worker.Start(context.Background())) + t.Cleanup(f.worker.Stop) + + require.True(t, f.worker.Enqueue(configMapTargetEvent("cm1", "alice", newBranchTarget))) + req := standbyRequest() + f.worker.EnqueueAttach(req) + phase := func() CommitRequestPhase { + return f.worker.LookupCommitRequestPhase(req.Namespace, req.Name, req.UID) + } + require.Eventually(t, func() bool { + open, _ := f.worker.ParentRecovery() + return open && phase() == PhaseWaitingForPush + }, 10*time.Second, 20*time.Millisecond, "the write is held while the parent is missing") + + clock.advance(10 * time.Minute) // far past attachTimeout + maxDuration + 120s + f.worker.EnqueueRefresh(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + require.Never(t, func() bool { + _, resolved := outcome(t, f.worker) + return resolved + }, 300*time.Millisecond, 20*time.Millisecond, "a held request is never failed for taking long") + assert.True(t, phase().Held()) + + release := simulateClientCommitOnDisk(t, f.repoDir, "release", "RELEASE.md", "back\n") + clock.advance(parentProbeMaxBackoff) + f.worker.EnqueueRefresh(&RefreshRequest{Target: refreshTarget(), MaxAge: time.Nanosecond}) + + require.Eventually(t, func() bool { + _, resolved := outcome(t, f.worker) + return resolved + }, 10*time.Second, 20*time.Millisecond) + res, _ := outcome(t, f.worker) + require.NoError(t, res.Err) + assert.Equal(t, FinalizeCommitted, res.Outcome) + tip := f.ref("feature") + assert.Equal(t, res.Commit, tip.String()) + assert.Equal(t, []plumbing.Hash{release}, f.parentOf(tip)) +} diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 96f6df15..4780a9d7 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -94,6 +94,15 @@ func (l *branchWorkerEventLoop) handleRefreshRequest(req *RefreshRequest) { // do no work at all, two lines below. observed, known := w.LastRemoteObservation() + // While work is held back by a missing parent, the refresher shares the worker's one probe + // deadline: a tick before it costs nothing, and a tick after it is the probe. + if l.recovery.active { + if l.probeDue() { + l.runParentProbe() + } + return + } + // 3. Not idle, so not the target this exists for. A reset here would destroy retained // commits, and the worktree may hold a partial write — which is also why this is the one exit // that does not re-read the folder: a layout resolved from a half-written tree is worse than @@ -190,24 +199,32 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr // is what makes it honest for the two targets a clone-first version skipped silently — one // declared but never published to, and one whose GitProvider was recreated against a // different repository. - advertised, err := advertiseRemoteBranch( - w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), auth) + parentBranch := w.ParentBranch() + advertisement, err := advertiseRemoteBranchFn( + w.repo.URL, plumbing.NewBranchReferenceName(w.Branch), parentBranch, auth) if err != nil { return fmt.Errorf("read the remote advertisement: %w", err) } + advertised := advertisement.branch + if missingParent := w.recordAdvertisement(advertisement, parentBranch); missingParent { + // Neither the branch nor the parent it would be created from: there is nothing to follow, + // and the GitTarget reports the parent as missing until somebody creates it. + return nil + } revision := "" if !advertised.IsZero() { revision = advertised.String() } - w.recordRemoteObservation(revision, ObservedByFetch) - // The remote does not carry this branch. That IS the answer, and there is nothing to fetch: a - // fetch would fall back to the default branch and teach us nothing about a branch nobody has - // created — the ordinary state of a target that has not written yet. A branch somebody - // DELETED is caught by the compare-and-swap on the next push. + // The remote does not carry this branch — the ordinary state of a target that has not written + // yet. That IS the observation. What the checkout holds instead is the parent the branch would + // be created from, so that is what is followed, from the same advertisement. This keeps the + // folder's published shape about the tree the next commit builds on; it is not what keeps + // that commit fresh, which the push checks for itself. A branch somebody DELETED is caught by + // the compare-and-swap on the next push. if advertised.IsZero() { w.Log.V(1).Info("Refresh found no such branch on the remote", "branch", w.Branch) - return nil + return l.followParent(advertisement) } // Everything below compares the advertisement against the local checkout, so a missing one is @@ -241,6 +258,77 @@ func (l *branchWorkerEventLoop) refreshFromRemote(provider *configv1alpha3.GitPr return nil } +// recordAdvertisement records what one advertisement said about the write branch and, while it +// is absent, its parent. It reports a configured parent that is missing as well. +func (w *BranchWorker) recordAdvertisement(advertisement remoteAdvertisement, parentBranch string) bool { + if !advertisement.branch.IsZero() { + w.recordRemoteObservation(advertisement.branch.String(), ObservedByFetch) + return false + } + if advertisement.parent.IsZero() { + if parentBranch != "" { + w.recordMissingParent(parentBranch, parentBranch) + return true + } + if !advertisement.empty { + // Not empty, and its HEAD resolves to no branch: as missing as a configured parent. + w.recordMissingParent("", advertisement.parentBranch) + return true + } + w.recordAbsentBranch("", "", ObservedByFetch) // an empty repository: no parent at all + return false + } + w.recordAbsentBranch(advertisement.parentBranch, advertisement.parent.String(), ObservedByFetch) + return false +} + +// followParent brings the checkout of an absent write branch onto its parent's advertised tip. +// It costs nothing beyond the advertisement already read when the parent has not moved, and it +// writes nothing either way. +// +// "Moved" includes a parent that is now a different branch: with spec.parentBranch omitted the +// parent is the remote's default branch as last discovered, and a remote whose HEAD switched from +// main to trunk at the same commit has moved as surely as one whose main got a new commit. One +// fetch follows it, so the next publication validates trunk, and pushes and refreshes stop +// reporting different parents. +func (l *branchWorkerEventLoop) followParent(advertisement remoteAdvertisement) error { + w := l.w + parent := advertisement.parent + // An empty repository has no parent to follow. + if parent.IsZero() { + return nil + } + repo, err := gogit.PlainOpen(w.repoPath()) + if errors.Is(err, gogit.ErrRepositoryNotExists) { + return nil + } + if err != nil { + return fmt.Errorf("open repository: %w", err) + } + if w.baseTrusted() && parent == localHead(repo) && advertisement.parentBranch == w.checkoutParentBranch(repo) { + w.Log.V(1).Info("Refresh confirmed the parent branch has not moved", "branch", w.Branch) + return nil + } + w.Log.Info("Refresh found the parent branch elsewhere; resetting onto it", + "branch", w.Branch, "parent", parent.String()) + return w.syncWithRemote(w.ctx, fetchReasonRefresh) +} + +// checkoutParentBranch is the short name of the branch the checkout of an absent write branch is +// rooted on: the parent the local write branch was created from, or the branch HEAD is on. +func (w *BranchWorker) checkoutParentBranch(repo *gogit.Repository) string { + w.repoMu.Lock() + defer w.repoMu.Unlock() + if w.newBranchParent != "" { + return w.newBranchParent.Short() + } + branch, _, err := GetCurrentBranch(repo) + if err != nil { + return "" + } + return branch.Short() +} + // rescanLayoutForTarget re-reads the target's folder, publishes what its shape resolves to, and // publishes whether the folder can be written at all. // @@ -315,7 +403,22 @@ func (w *BranchWorker) rescanLayoutForTarget(ctx context.Context, req *RefreshRe w.reportLayout(ctx, batch, worktreeRevision(worktree)) } -// advertiseRemoteBranch asks the remote where a branch is, and transfers nothing else. +// remoteAdvertisement is what one ref advertisement says about a write branch. +type remoteAdvertisement struct { + // branch is where the write branch is; zero when the remote does not carry it. + branch plumbing.Hash + // parent is where the parent a new write branch is created from is: the configured parent + // branch, or the remote's default branch when none is configured. Zero when the remote does + // not carry it, which includes an empty repository. + parent plumbing.Hash + // parentBranch is that parent's short name; empty when the remote's HEAD names no branch. + parentBranch string + // empty records an advertisement with no hash refs at all: see advertisesNoRefs. + empty bool +} + +// advertiseRemoteBranch asks the remote where a branch and its parent are, and transfers nothing +// else. An empty parentBranch means the branch the remote's HEAD names. // // It is built on a storage-less remote, the way CheckRepo is, so it needs no clone on disk. Zero // means the advertisement did not carry the branch, which includes an empty repository — an @@ -323,19 +426,31 @@ func (w *BranchWorker) rescanLayoutForTarget(ctx context.Context, req *RefreshRe func advertiseRemoteBranch( remoteURL string, branch plumbing.ReferenceName, + parentBranch string, auth []gitclient.Option, -) (plumbing.Hash, error) { +) (remoteAdvertisement, error) { remote := gogit.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{remoteURL}}) refs, err := listRemoteRefs(remote, auth) if err != nil { - return plumbing.ZeroHash, err + return remoteAdvertisement{}, err } + empty := advertisesNoRefs(refs) + hashes := map[plumbing.ReferenceName]plumbing.Hash{} + var defaultBranch plumbing.ReferenceName for _, ref := range refs { - if ref.Type() == plumbing.HashReference && ref.Name() == branch { - return ref.Hash(), nil + switch { + case ref.Type() == plumbing.HashReference: + hashes[ref.Name()] = ref.Hash() + case ref.Name() == plumbing.HEAD && ref.Type() == plumbing.SymbolicReference: + defaultBranch = ref.Target() } } - return plumbing.ZeroHash, nil + if parentBranch != "" { + defaultBranch = plumbing.NewBranchReferenceName(parentBranch) + } + return remoteAdvertisement{ + branch: hashes[branch], parent: hashes[defaultBranch], parentBranch: defaultBranch.Short(), empty: empty, + }, nil } // localHead is the commit the worktree is on, or the zero hash when the branch is unborn. diff --git a/internal/git/refresh_test.go b/internal/git/refresh_test.go index 3e7bfe0f..9a726c9e 100644 --- a/internal/git/refresh_test.go +++ b/internal/git/refresh_test.go @@ -4,7 +4,9 @@ package git import ( "os" + "os/exec" "path/filepath" + "strings" "testing" "time" @@ -43,9 +45,15 @@ func newRefreshHarness(t *testing.T, slug string) *refreshHarness { // newRefreshHarnessOn builds the harness on a remote that may or may not already have the branch. func newRefreshHarnessOn(t *testing.T, slug string, seeded bool) *refreshHarness { + t.Helper() + return newRefreshHarnessForBranch(t, slug, seeded, "main") +} + +// newRefreshHarnessForBranch builds the harness for a worker on another branch than main. +func newRefreshHarnessForBranch(t *testing.T, slug string, seeded bool, branch string) *refreshHarness { t.Helper() h := &refreshHarness{ - ledgerFixture: newLedgerFixture(t, slug, seeded), + ledgerFixture: newLedgerFixtureOnBranch(t, slug, seeded, branch), target: itypes.NewResourceReference("checkout", "shop"), path: "team-a", } @@ -239,6 +247,46 @@ func TestRefresh_AnAbsentBranchCostsOneConnection(t *testing.T) { "no revision IS the observation: a branch does not exist without a commit") } +// TestRefresh_AnAbsentBranchFollowsItsParent: while the write branch does not exist, the checkout +// holds its parent, and a parent somebody moved is followed from the same advertisement, so the +// folder the scan re-reads is the one the next commit builds on. It writes nothing. +func TestRefresh_AnAbsentBranchFollowsItsParent(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-absent-parent-moved", true, "feature") + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + h.contend("OUTSIDE.md", "from-another-writer\n") + moved := revParseMain(t, h.repoDir) + + h.refresh(time.Nanosecond) + + assert.Equal(t, int64(1), fetchCount(t, reader, h.worker, fetchReasonRefresh), + "the parent moved, so the checkout has to be brought onto it") + repo, err := gogit.PlainOpen(h.worker.repoPath()) + require.NoError(t, err) + assert.Equal(t, moved, localHead(repo).String(), "the checkout is on the parent's new tip") + require.NotEmpty(t, h.reported) + assert.Empty(t, h.reported[len(h.reported)-1].Commit, "the write branch is still not on the remote") + assert.NotEmpty(t, h.scanVerdicts, "and the folder was re-read from it") + out, err := exec.Command("git", "-C", h.repoDir, "branch", "--list", "feature").Output() + require.NoError(t, err) + assert.Empty(t, strings.TrimSpace(string(out)), "and following it created no branch") +} + +// TestRefresh_AnAbsentBranchWithAnUnmovedParentCostsOneConnection: following the parent is free +// when it has not moved, because the advertisement that answers for the write branch carries it. +func TestRefresh_AnAbsentBranchWithAnUnmovedParentCostsOneConnection(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-absent-parent-unmoved", true, "feature") + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + + assert.Equal(t, int64(1), h.refresh(time.Nanosecond), "one advertisement answers for both branches") + assert.Zero(t, fetchCount(t, reader, h.worker, fetchReasonRefresh)) +} + // TestRefresh_ObservesTheRemoteWithNoCheckoutYet. A GitTarget that has been declared but has not // published has a worker and no clone, and it still deserves an honest status.remote. // @@ -569,3 +617,85 @@ func TestRefresh_WithNoPathScansNothing(t *testing.T) { assert.Empty(t, layouts) } + +// TestRefresh_ANewBranchDoesNotDependOnARecentRefresh: a refresh that completed just before the +// parent moved leaves the checkout one commit behind, and the next write must still create the +// write branch on the parent's current tip. Publication checks the parent itself; a refresh tick is +// never what makes it fresh. +func TestRefresh_ANewBranchDoesNotDependOnARecentRefresh(t *testing.T) { + h := newRefreshHarnessForBranch(t, "refresh-then-parent-moves", true, "feature") + h.worker.mapper = configMapMapper() + h.createLedgerTarget("live", nil) + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + + h.refresh(time.Nanosecond) + h.contend("LATEST.md", "moved right after the refresh\n") + moved := revParseMain(t, h.repoDir) + + loop := newBranchWorkerEventLoop(h.worker, 0) + defer loop.stopTimers() + loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("cm1", "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) + require.Empty(t, loop.pendingWrites, "the write was published") + + out, err := exec.Command("git", "-C", h.repoDir, "rev-parse", "refs/heads/feature^").Output() + require.NoError(t, err) + assert.Equal(t, moved, strings.TrimSpace(string(out)), "the new branch starts on the parent's current tip") +} + +// TestRefresh_FollowsTheConfiguredParent: with spec.parentBranch set, the refresher follows that +// branch rather than the remote's default, and a parent changed on a live target is re-read on the +// next refresh. +func TestRefresh_FollowsTheConfiguredParent(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-configured-parent", true, "feature") + release := simulateClientCommitOnDisk(t, h.repoDir, "release", "RELEASE.md", "release\n") + require.NoError(t, h.worker.ensureRepositoryInitialized(h.worker.ctx)) + + require.True(t, h.worker.SetParentBranch("release"), "changed on a live target") + h.refresh(time.Nanosecond) + + assert.Equal(t, int64(1), fetchCount(t, reader, h.worker, fetchReasonRefresh)) + repo, err := gogit.PlainOpen(h.worker.repoPath()) + require.NoError(t, err) + assert.Equal(t, release, localHead(repo), "the checkout is on the configured parent") + assert.NotEmpty(t, h.scanVerdicts, "and the folder was re-read from it") +} + +// TestRefresh_ReportsAMissingConfiguredParent: one advertisement shows neither the write branch nor +// its configured parent, which is reported and fetches nothing. +func TestRefresh_ReportsAMissingConfiguredParent(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + + h := newRefreshHarnessForBranch(t, "refresh-missing-parent", true, "feature") + h.worker.SetParentBranch("release") + h.reported = nil + + assert.Equal(t, int64(1), h.refresh(time.Nanosecond)) + assert.Zero(t, fetchCount(t, reader, h.worker, fetchReasonRefresh)) + require.NotEmpty(t, h.reported) + last := h.reported[len(h.reported)-1] + assert.Equal(t, ParentMissing, last.ParentState) + assert.Equal(t, "release", last.ParentBranch) + assert.Empty(t, last.Commit) +} + +// TestRefresh_ObservesTheParentOfAnAbsentBranch: the advertisement that answers for the write +// branch also names its parent, so the refresher reports both at one connection. +func TestRefresh_ObservesTheParentOfAnAbsentBranch(t *testing.T) { + h := newRefreshHarnessForBranch(t, "refresh-parent-observed", true, "feature") + h.reported = nil + + assert.Equal(t, int64(1), h.refresh(time.Nanosecond)) + require.NotEmpty(t, h.reported) + last := h.reported[len(h.reported)-1] + assert.Empty(t, last.Commit) + assert.Equal(t, ParentFound, last.ParentState) + assert.Equal(t, "main", last.ParentBranch, "an omitted parent is the branch the remote's HEAD names") + assert.Equal(t, revParseMain(t, h.repoDir), last.ParentCommit) +} diff --git a/internal/git/remote_observation.go b/internal/git/remote_observation.go index 465f3b6c..cef91b8f 100644 --- a/internal/git/remote_observation.go +++ b/internal/git/remote_observation.go @@ -50,7 +50,35 @@ type RemoteObservation struct { // It is zero for a worker that has no identity of its own: the CLI, and tests that never reach // a remote. A comparison against an unknown identity proves nothing, so it is not made. Repo RepoIdentity + // ParentState is what the remote offers an absent branch to be created from, observed + // together with its absence. Empty while the branch exists. + ParentState ParentState + // ParentBranch and ParentCommit name that parent and where it is. ParentCommit is empty + // unless ParentState is ParentFound; ParentBranch is empty for ParentUnborn. For an omitted + // parent that is ParentMissing, ParentBranch is the branch the remote's HEAD names, or empty + // when it names none. + ParentBranch string + ParentCommit string + // ParentRequested is spec.parentBranch as the observing operation read it: empty for the + // remote's default branch. It is set on ParentMissing, so a reader can tell whether the + // observation is about the parent configured now. + ParentRequested string } +// ParentState is what a branch the remote does not carry would be created from. +type ParentState string + +const ( + // ParentFound means the parent branch exists, and the branch's first commit builds on its tip. + ParentFound ParentState = "Found" + // ParentMissing means the parent is not on the remote either, so the branch cannot be created + // and nothing is written until one of them exists: the configured parent branch is absent, or, + // with none configured, the repository is not empty but its default branch does not resolve. + ParentMissing ParentState = "Missing" + // ParentUnborn means the repository is empty, with no hash refs at all (tags included), and + // no parent was configured, so the first commit starts the branch with no history. + ParentUnborn ParentState = "Unborn" +) + // Age is how long ago the observation was made, against now. func (o RemoteObservation) Age(now time.Time) time.Duration { return now.Sub(o.At) } diff --git a/internal/git/remote_report_test.go b/internal/git/remote_report_test.go index b17ac9f4..d8ffddeb 100644 --- a/internal/git/remote_report_test.go +++ b/internal/git/remote_report_test.go @@ -56,12 +56,12 @@ func TestRemoteForBranch_SurvivesTheWorkerItWasProvedBy(t *testing.T) { before := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} after := RepoIdentity{ProviderUID: "uid-2", URL: "https://example.invalid/second.git"} - require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) + require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) worker, ok := m.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) worker.recordRemoteObservation("aaaa", ObservedByPush) - require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, m.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) observed, known := m.RemoteForBranch(key) require.True(t, known, diff --git a/internal/git/replay_failure_test.go b/internal/git/replay_failure_test.go index 45941630..574a7ace 100644 --- a/internal/git/replay_failure_test.go +++ b/internal/git/replay_failure_test.go @@ -334,7 +334,7 @@ func TestReplayFailure_AResetThatDiesHalfwayIsAlsoHeld(t *testing.T) { original := syncToRemoteFn syncToRemoteFn = func( ctx context.Context, repo *gogit.Repository, - branch plumbing.ReferenceName, auth []gitclient.Option, + branch plumbing.ReferenceName, _ string, auth []gitclient.Option, ) (*PullReport, error) { if _, err := SmartFetch(ctx, repo, branch, auth); err != nil { return nil, err diff --git a/internal/git/resync_flush.go b/internal/git/resync_flush.go index 52dff404..56f49f39 100644 --- a/internal/git/resync_flush.go +++ b/internal/git/resync_flush.go @@ -94,6 +94,10 @@ func resyncHealKey(req *ResyncRequest) healKey { // on that flag, so on a target holding work a bare invalidation reached nothing at all and the // snapshot judged a tree nobody had read. That is the exact hazard this function exists to close. func (l *branchWorkerEventLoop) prepareBaseForResync(req *ResyncRequest) error { + // A parent known to be missing is not fetched again before its probe is due. + if l.w.awaitingParentProbe() { + return errAwaitingParentProbe + } // The invalidation is skipped when RefreshRemote already fetched a moment ago, which would // otherwise make a forced recheck pay twice. if req.RefreshRemote { @@ -159,6 +163,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { closedWindow = l.finalizeOpenWindowWithReason(windowFinalizeReasonResyncBeforeApply) } if err := l.prepareBaseForResync(req); err != nil { + l.noteParentUnavailable(err, resyncScope(req)) req.reply(ResyncResult{Err: err}) return } @@ -186,6 +191,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { refusalObservationForDesired(req.Desired, refused), req.refusalCollection()) } l.w.Log.Error(err, "Resync commit failed; dropping request", "resources", len(req.Desired)) + l.noteParentUnavailable(err, resyncScope(req)) req.reply(ResyncResult{Err: err}) return } @@ -206,6 +212,7 @@ func (l *branchWorkerEventLoop) applyResync(req *ResyncRequest) { l.pendingWrites = append(l.pendingWrites, *pendingWrite) l.pendingWritesBytes += pendingWrite.ByteSize } + l.noteResyncApplied(req.GitTargetNamespace, req.GitTargetName, req.refusalCollection(), committed) // Schedule a push whenever this request CLOSED a live window — that window's // commit is now in pendingWrites and must reach the remote — or the resync itself // committed. Any finalize that closes a window must schedule its push, or the diff --git a/internal/git/testdata/git-roundtrip-ledger.golden b/internal/git/testdata/git-roundtrip-ledger.golden index b84be853..1084d44f 100644 --- a/internal/git/testdata/git-roundtrip-ledger.golden +++ b/internal/git/testdata/git-roundtrip-ledger.golden @@ -30,3 +30,7 @@ operation conn fetch-open 10. a resync (snapshot) 4 2 -/<1K 0 1 -/<1K 1 <1K/<1K 11. publication onto a branch the remote does not have 2 0 0 1 -/<1K 1 <1K/<1K 12. an idle target held across several commit windows 0 0 0 0 0 +13. standby, a live write that changes nothing 1 0 0 1 -/<1K 0 +14. standby, the first write after the parent moved 6 2 -/<1K 1 <1K/<1K 2 -/<1K 1 <4K/<1K +15. parent probe while the parent is missing 1 1 -/<1K 0 0 0 +16. standby, the write branch created during our upload 8 3 -/<4K 1 <1K/<1K 2 -/<1K 2 <4K/<1K diff --git a/internal/git/types.go b/internal/git/types.go index ca4444a5..6b404a7b 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -142,6 +142,9 @@ type PullReport struct { ExistsOnRemote bool // Branch exists on remote HEAD BranchInfo IncomingChanges bool // SHA changed, requiring resource-level reconcile + // ParentBranch is the branch checked out instead, when the target branch is absent: the parent + // it would be created from. HEAD.Sha is then the parent's commit. Empty otherwise. + ParentBranch string } // BranchKey uniquely identifies a (GitProvider, Branch) combination. @@ -405,6 +408,9 @@ type WorkItem struct { // Refresh asks the worker to re-prove where its branch is on the remote. It is the only // work item that never writes anything: see RefreshRequest. Refresh *RefreshRequest + // Withdraw cancels a CommitRequest the worker has not acted on yet. It rides the same FIFO as + // the attach, so it is always handled after every attach sent before it. + Withdraw *AttachCommitRequest } // ResyncScope restricts a resync's mark-and-sweep to the slice of the mirror the desired diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 566b6af7..a466fa31 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -95,7 +95,11 @@ type WorkerManager struct { // the scrape goroutine. Take it BEFORE mu, never the other way round. lifecycleMu sync.Mutex workers map[BranchKey]*BranchWorker - ctx context.Context + // commitRequests records which worker holds each CommitRequest. It outlives the map above on + // purpose: a retired worker leaves the map before it stops, and keeps answering for the + // requests it held. See commitRequestOwners. + commitRequests *commitRequestOwners + ctx context.Context // mapper is the GVK->GVR resolver injected into every worker so store scans build a // resource-identity inventory. It is set once at startup (SetMapper) before any // worker is created; a nil mapper keeps workers structure-only. It is the LOCAL cluster's @@ -168,6 +172,7 @@ func NewWorkerManager( limits: limits.withDefaults(), sensitiveResources: sensitiveResources, workers: make(map[BranchKey]*BranchWorker), + commitRequests: &commitRequestOwners{}, remotes: make(map[BranchKey]RemoteObservation), replacements: make(map[BranchKey]struct{}), renderFidelityGate: NewRenderFidelityGate(), @@ -283,12 +288,19 @@ func (m *WorkerManager) RemoteForBranch(key BranchKey) (RemoteObservation, bool) // and the caller can fail. See ReplacementPending; the delivery is the GitTarget reconcile's // worker wiring gate. // +// parentBranch is the GitTarget's spec.parentBranch. It is immutable on a GitTarget, so a live +// worker sees a different one only when every target on its branch was replaced before the worker +// sweep ran. That is not a replacement: the worker keeps its clone, and SetParentBranch raises the +// parent generation, so nothing planned under the old parent is pushed and retained writes are +// rebuilt on the new one. +// // Worker creation/start is protected by the manager lock. func (m *WorkerManager) EnsureWorker( _ context.Context, providerName, providerNamespace string, branch string, repo RepoIdentity, + parentBranch string, ) error { // Held across the whole check-and-create so a replacement cannot start while the worker it // replaces is still stopping; they would share a clone. @@ -307,6 +319,10 @@ func (m *WorkerManager) EnsureWorker( if exists { if existing.repo == repo { + if existing.SetParentBranch(parentBranch) { + m.Log.Info("Parent branch changed; the worker re-reads the remote on its next cycle", + "key", key.String(), "parentBranch", parentBranch) + } return nil } // Both identities, at default verbosity: a comparison that is not stable across steady @@ -338,6 +354,7 @@ func (m *WorkerManager) EnsureWorker( // Inject the resolver before Start: the field is read only by the event-loop // goroutine Start spawns, so setting it here (under m.mu, before that goroutine // exists) is race-free. + worker.SetParentBranch(parentBranch) worker.mapper = m.mapper worker.clusterMapper = m.clusterMapper worker.sshHostKeys = m.sshHostKeys @@ -347,6 +364,7 @@ func (m *WorkerManager) EnsureWorker( worker.remoteReporter = func(observed RemoteObservation) { m.recordRemoteObservation(key, observed) } worker.scanAcceptance = m.scanAcceptance worker.renderFidelityGate = m.renderFidelityGate + worker.crOwners = m.commitRequests if err := worker.Start(m.ctx); err != nil { return fmt.Errorf("failed to start worker for %s: %w", key.String(), err) @@ -456,6 +474,14 @@ func (m *WorkerManager) GetWorkerForTarget( return worker, exists } +// CommitRequestOwner returns the worker that holds a CommitRequest: the one that accepted its +// attach and has not given it back. It is found whatever has happened to the request's GitTarget +// since, including a worker retired and still finishing its last push, so a caller that finds no +// owner knows that no worker can still commit the request. +func (m *WorkerManager) CommitRequestOwner(namespace, name, uid string) (*BranchWorker, bool) { + return m.commitRequests.owner(commitRequestID{Namespace: namespace, Name: name, UID: uid}) +} + // afterOrphanSelection runs between choosing the workers to retire and retiring them. It is nil in // production and exists because that gap is the ONLY place the lock discipline in ReconcileWorkers // can be observed: with lifecycleMu held across both steps a concurrent EnsureWorker blocks here, diff --git a/internal/git/worker_manager_test.go b/internal/git/worker_manager_test.go index 5fe9c1e7..3ce329b9 100644 --- a/internal/git/worker_manager_test.go +++ b/internal/git/worker_manager_test.go @@ -68,7 +68,7 @@ func mustEnsureWorker( repo RepoIdentity, ) bool { t.Helper() - require.NoError(t, m.EnsureWorker(ctx, providerName, providerNamespace, branch, repo)) + require.NoError(t, m.EnsureWorker(ctx, providerName, providerNamespace, branch, repo, "")) key := BranchKey{RepoNamespace: providerNamespace, RepoName: providerName, Branch: branch} pending := m.ReplacementPending(key) m.AcknowledgeReplacement(key) @@ -144,7 +144,7 @@ func TestEnsureWorker_CreatesAWorkerWithTheIdentityItWasAskedFor(t *testing.T) { createTargetForRegister(ctx, t, client, "target1", "repo1", "main", "clusters/prod") // Register first target - err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo) + err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "") if err != nil { t.Fatalf("Failed to register target: %v", err) } @@ -237,12 +237,12 @@ func TestWorkerManagerDifferentBranches(t *testing.T) { createTargetForRegister(ctx, t, client, "target-dev", "repo1", "develop", "base/") // Register targets for same repo, different branches - err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo) + err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "") if err != nil { t.Fatalf("Failed to register target-main: %v", err) } - err = manager.EnsureWorker(ctx, "repo1", "gitops-system", "develop", repo) + err = manager.EnsureWorker(ctx, "repo1", "gitops-system", "develop", repo, "") if err != nil { t.Fatalf("Failed to register target-dev: %v", err) } @@ -362,7 +362,7 @@ func TestWorkerManagerConcurrentRegistration(t *testing.T) { done := make(chan bool, 10) for i := range 10 { go func(index int) { - err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo) + err := manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "") if err != nil { t.Errorf("Failed to ensure worker %d: %v", index, err) } @@ -529,7 +529,7 @@ func TestWorkerLifecycle_BuildsAndReclaimsItsOwnState(t *testing.T) { root := workerStateRoot repo := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", repo, "")) worker, ok := manager.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) require.True(t, strings.HasPrefix(worker.repoPath(), root+string(filepath.Separator)), @@ -573,7 +573,7 @@ func TestWorkerLifecycle_AReplacementDoesNotReclaimItsPredecessorsCheckout(t *te before := RepoIdentity{ProviderUID: "uid-1", URL: sameURL} after := RepoIdentity{ProviderUID: "uid-2", URL: sameURL} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) old, ok := manager.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) require.NoError(t, os.MkdirAll(old.repoPath(), 0o750)) @@ -585,7 +585,7 @@ func TestWorkerLifecycle_AReplacementDoesNotReclaimItsPredecessorsCheckout(t *te sentinel := filepath.Join(successor, "HEAD") require.NoError(t, os.WriteFile(sentinel, []byte("ref: refs/heads/main\n"), 0o600)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) replacement, ok := manager.GetWorkerForTarget("repo1", "gitops-system", "main") require.True(t, ok) require.NotSame(t, old, replacement) diff --git a/internal/git/worker_repo_identity_test.go b/internal/git/worker_repo_identity_test.go index bc23e70c..6fd70ab0 100644 --- a/internal/git/worker_repo_identity_test.go +++ b/internal/git/worker_repo_identity_test.go @@ -352,21 +352,21 @@ func TestEnsureWorker_AnUndeliveredRecoveryIsStillPendingOnTheNextPass(t *testin before := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} after := RepoIdentity{ProviderUID: "uid-2", URL: "https://example.invalid/second.git"} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) require.False(t, manager.ReplacementPending(key)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) require.True(t, manager.ReplacementPending(key)) // The delivery fails, so nothing acknowledges it. Two more reconciles of this target, and of // a sibling on the same branch, still find the recovery waiting. - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) assert.True(t, manager.ReplacementPending(key), "a recovery nobody delivered must survive the calls that come after it") manager.AcknowledgeReplacement(key) assert.False(t, manager.ReplacementPending(key)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) assert.False(t, manager.ReplacementPending(key), "and a delivered one must not be asked for again on every steady tick") } @@ -379,8 +379,8 @@ func TestReconcileWorkers_DropsARecoveryForABranchNothingNeeds(t *testing.T) { before := RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/first.git"} after := RepoIdentity{ProviderUID: "uid-2", URL: "https://example.invalid/second.git"} - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before)) - require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after)) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", before, "")) + require.NoError(t, manager.EnsureWorker(ctx, "repo1", "gitops-system", "main", after, "")) require.True(t, manager.ReplacementPending(key)) // No GitTarget names this branch, so the sweep retires its worker. diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index a2cd42c6..542baccd 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -59,7 +59,7 @@ func NewEventRouter( } // ServiceCommitRequest is the controller's attach-then-poll seam (§6.4.3): it -// resolves the GitTarget's branch worker, registers the CommitRequest attach +// resolves the worker that holds the request, or else the GitTarget's branch worker, registers the CommitRequest attach // idempotently on that worker's FIFO event queue (attach it to the author's window // with its message and timers), and returns the request's current // outcome. resolved=false means the worker has not finished — the controller @@ -77,24 +77,14 @@ func (r *EventRouter) ServiceCommitRequest( ctx context.Context, attach git.AttachCommitRequest, ) (git.FinalizeResult, bool, error) { - var gitTarget configv1alpha3.GitTarget - if err := r.Client.Get(ctx, client.ObjectKey{ - Name: attach.GitTargetName, - Namespace: attach.GitTargetNamespace, - }, &gitTarget); err != nil { - return git.FinalizeResult{}, false, fmt.Errorf("get GitTarget %s/%s: %w", - attach.GitTargetNamespace, attach.GitTargetName, err) + worker, branch, err := r.commitRequestWorker(ctx, attach) + if err != nil { + return git.FinalizeResult{}, false, err } - - worker, exists := r.WorkerManager.GetWorkerForTarget( - gitTarget.Spec.GitProviderRef.Name, - gitTarget.Namespace, // provider is in the same namespace as the target - gitTarget.Spec.Branch, - ) - if !exists { + if worker == nil { r.Log.V(1).Info("ServiceCommitRequest: no branch worker for the GitTarget yet; will retry", "gitTarget", attach.GitTargetNamespace+"/"+attach.GitTargetName) - return git.FinalizeResult{Branch: gitTarget.Spec.Branch, Phase: git.PhaseWaitingForWorker}, false, nil + return git.FinalizeResult{Branch: branch, Phase: git.PhaseWaitingForWorker}, false, nil } // Idempotent register (the worker keys by request identity and keeps the first @@ -109,6 +99,89 @@ func (r *EventRouter) ServiceCommitRequest( return result, resolved, nil } +// WithdrawCommitRequest is the controller's fail-closed seam: it cancels a request the worker has +// not acted on, and reports the worker's answer, so the controller never fails a request the worker +// could still commit. +// +// - The worker that holds the request answers for it, even when its GitTarget is gone or the +// worker is being retired. The withdraw rides its FIFO behind every attach already sent, and +// is a no-op for a request it holds. Until it is handled, resolved=false and Phase says where +// the request stands; a held phase means the controller keeps waiting. +// - Nothing holds it, and the GitTarget is gone or names no worker: nothing can commit the +// request, so it resolves at once as withdrawn. The controller sends no attach after it starts +// withdrawing, so a worker that starts later never sees it. +func (r *EventRouter) WithdrawCommitRequest( + ctx context.Context, + attach git.AttachCommitRequest, +) (git.FinalizeResult, bool, error) { + worker, branch, err := r.commitRequestWorker(ctx, attach) + if errors.Is(err, errGitTargetGone) { + // No worker holds the request, and its GitTarget can never start one. + return git.FinalizeResult{Err: git.ErrCommitRequestWithdrawn}, true, nil + } + if err != nil { + return git.FinalizeResult{}, false, err + } + if worker == nil { + return git.FinalizeResult{ + Branch: branch, + Phase: git.PhaseWaitingForWorker, + Err: git.ErrCommitRequestWithdrawn, + }, true, nil + } + + if result, resolved := worker.LookupCommitRequestOutcome(attach.Namespace, attach.Name, attach.UID); resolved { + return result, true, nil + } + phase := worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID) + if phase.Held() { + return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil + } + worker.EnqueueWithdraw(&attach) + // A worker whose loop has exited answers the withdraw at once. + if result, resolved := worker.LookupCommitRequestOutcome(attach.Namespace, attach.Name, attach.UID); resolved { + return result, true, nil + } + return git.FinalizeResult{Branch: branch, Phase: phase}, false, nil +} + +// commitRequestWorker finds the worker to ask about a CommitRequest, and the branch it serves. +// +// The worker that holds the request comes first, whatever its GitTarget says now: a shared worker +// outlives a deleted target, and a retired one finishes its last push after the manager stopped +// listing it. Only a request no worker holds falls back to the GitTarget's current worker, which is +// nil while that worker has not started. A GitTarget that is gone is errGitTargetGone. +func (r *EventRouter) commitRequestWorker( + ctx context.Context, + attach git.AttachCommitRequest, +) (*git.BranchWorker, string, error) { + if owner, ok := r.WorkerManager.CommitRequestOwner(attach.Namespace, attach.Name, attach.UID); ok { + return owner, owner.Branch, nil + } + + var gitTarget configv1alpha3.GitTarget + if err := r.Client.Get(ctx, client.ObjectKey{ + Name: attach.GitTargetName, + Namespace: attach.GitTargetNamespace, + }, &gitTarget); err != nil { + if apierrors.IsNotFound(err) { + return nil, "", fmt.Errorf("%w: %s/%s", errGitTargetGone, attach.GitTargetNamespace, attach.GitTargetName) + } + return nil, "", fmt.Errorf("get GitTarget %s/%s: %w", + attach.GitTargetNamespace, attach.GitTargetName, err) + } + + worker, exists := r.WorkerManager.GetWorkerForTarget( + gitTarget.Spec.GitProviderRef.Name, + gitTarget.Namespace, // provider is in the same namespace as the target + gitTarget.Spec.Branch, + ) + if !exists { + return nil, gitTarget.Spec.Branch, nil + } + return worker, gitTarget.Spec.Branch, nil +} + // recordBackgroundResyncFailure counts a fire-and-forget resync whose apply failed or // timed out at the worker, so the failure is observable even though delivery was already // marked on enqueue. No-op until the counter is registered. diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index db2c3946..9a54e55a 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -58,7 +58,7 @@ func TestServiceCommitRequest_GitTargetNotFound(t *testing.T) { _, resolved, err := router.ServiceCommitRequest(context.Background(), saveAttach("missing", "team-a")) require.Error(t, err) assert.False(t, resolved) - assert.Contains(t, err.Error(), "get GitTarget") + require.ErrorIs(t, err, errGitTargetGone) } func TestGitTargetEventStreamRegistry(t *testing.T) { @@ -353,8 +353,8 @@ func TestServiceCommitRequest_RegisteredWorkerResolvesNoOpenWindow(t *testing.T) ensureErr := workerManager.EnsureWorker( ctx, "team-a-provider", "team-a", "main", - git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, - ) + git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, "") + require.NoError(t, ensureErr) router := NewEventRouter(workerManager, nil, client, logr.Discard()) @@ -375,6 +375,142 @@ func TestServiceCommitRequest_RegisteredWorkerResolvesNoOpenWindow(t *testing.T) assert.Equal(t, "main", result.Branch) } +// TestWithdrawCommitRequest_NothingHoldsItWithoutAWorker: a deleted GitTarget, or one whose worker +// never started, has nothing that could commit the request, so the withdraw resolves at once and +// says why when it can. +func TestWithdrawCommitRequest_NothingHoldsItWithoutAWorker(t *testing.T) { + scheme := eventRouterScheme(t) + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "team-a-provider"}, + Branch: "main", + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gitTarget).Build() + workerManager := git.NewWorkerManager( + client, logr.Discard(), git.BranchWorkerLimits{}, types.SensitiveResourcePolicy{}) + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + + result, resolved, err := router.WithdrawCommitRequest(context.Background(), saveAttach("team-a-config", "team-a")) + require.NoError(t, err) + require.True(t, resolved) + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + assert.Equal(t, git.PhaseWaitingForWorker, result.Phase) + assert.Equal(t, "main", result.Branch) + + result, resolved, err = router.WithdrawCommitRequest(context.Background(), saveAttach("deleted", "team-b")) + require.NoError(t, err) + require.True(t, resolved) + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + assert.Empty(t, result.Branch, "a deleted GitTarget has no identity to report") +} + +// TestWithdrawCommitRequest_ALiveWorkerCancelsAWaitingRequest: the withdraw rides the FIFO behind the +// attach, cancels the request while it only waits for a window, and an attach sent afterwards +// cannot bring it back. +func TestWithdrawCommitRequest_ALiveWorkerCancelsAWaitingRequest(t *testing.T) { + scheme := eventRouterScheme(t) + provider := &configv1alpha3.GitProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-provider", Namespace: "team-a"}, + Spec: configv1alpha3.GitProviderSpec{URL: "file:///tmp/does-not-need-to-exist"}, + } + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "team-a-provider"}, + Branch: "main", + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(provider, gitTarget).Build() + workerManager := git.NewWorkerManager( + client, logr.Discard(), git.BranchWorkerLimits{}, types.SensitiveResourcePolicy{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = workerManager.Start(ctx) }() + time.Sleep(100 * time.Millisecond) // allow the manager to record its context + require.NoError(t, workerManager.EnsureWorker( + ctx, "team-a-provider", "team-a", "main", git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, "")) + + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + attach := saveAttach("team-a-config", "team-a") + attach.AttachTimeout = time.Hour // it would wait an hour for a window + + _, resolved, err := router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err) + require.False(t, resolved) + + var result git.FinalizeResult + require.Eventually(t, func() bool { + var err error + result, resolved, err = router.WithdrawCommitRequest(context.Background(), attach) + require.NoError(t, err) + return resolved + }, 5*time.Second, 50*time.Millisecond, "the worker must answer the withdraw") + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + + result, resolved, err = router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err) + require.True(t, resolved, "a re-sent attach finds the withdrawn outcome") + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) +} + +// TestWithdrawCommitRequest_ADeletedGitTargetsWorkerStillAnswers: a worker is shared by every +// GitTarget on its branch, so it outlives a deleted one. The withdraw must reach that worker, and +// must never report success while the request is still registered there. +func TestWithdrawCommitRequest_ADeletedGitTargetsWorkerStillAnswers(t *testing.T) { + scheme := eventRouterScheme(t) + provider := &configv1alpha3.GitProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-provider", Namespace: "team-a"}, + Spec: configv1alpha3.GitProviderSpec{URL: "file:///tmp/does-not-need-to-exist"}, + } + gitTarget := &configv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "team-a-config", Namespace: "team-a"}, + Spec: configv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "team-a-provider"}, + Branch: "main", + }, + } + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(provider, gitTarget).Build() + workerManager := git.NewWorkerManager( + client, logr.Discard(), git.BranchWorkerLimits{}, types.SensitiveResourcePolicy{}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = workerManager.Start(ctx) }() + time.Sleep(100 * time.Millisecond) // allow the manager to record its context + require.NoError(t, workerManager.EnsureWorker( + ctx, "team-a-provider", "team-a", "main", git.RepoIdentity{URL: "file:///tmp/does-not-need-to-exist"}, "")) + worker, ok := workerManager.GetWorkerForTarget("team-a-provider", "team-a", "main") + require.True(t, ok) + + router := NewEventRouter(workerManager, nil, client, logr.Discard()) + attach := saveAttach("team-a-config", "team-a") + attach.AttachTimeout = time.Hour + _, _, err := router.ServiceCommitRequest(context.Background(), attach) + require.NoError(t, err) + require.Eventually(t, func() bool { + return worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID) == git.PhaseWaitingForWindow + }, 5*time.Second, 10*time.Millisecond) + + require.NoError(t, client.Delete(context.Background(), gitTarget)) + + var result git.FinalizeResult + require.Eventually(t, func() bool { + var resolved bool + result, resolved, err = router.WithdrawCommitRequest(context.Background(), attach) + require.NoError(t, err) + if resolved { + require.Empty(t, worker.LookupCommitRequestPhase(attach.Namespace, attach.Name, attach.UID), + "the withdraw reported success while the worker still held the registration") + } + return resolved + }, 5*time.Second, 50*time.Millisecond) + require.ErrorIs(t, result.Err, git.ErrCommitRequestWithdrawn) + assert.Equal(t, "main", result.Branch, "the worker answered, not the missing GitTarget") +} + // TestDrainScopedResync_QueueFullIsDrainedNotOrphaned guards the ordering invariant behind // Failure A/B: EnqueueResync answers a dropped request on its reply channel, so the drain MUST be // started even when the request never entered the FIFO. diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 3300d14c..81d74f2a 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -252,6 +252,26 @@ func createGitTargetWithOptions( encryptionSecretName string, generateWhenMissing bool, commit gitTargetCommitOptions, +) { + applyGitTarget(name, namespace, providerName, path, branch, "", encryptionSecretName, generateWhenMissing, commit) +} + +// createGitTargetWithParentBranch is createGitTarget with spec.parentBranch; empty omits it. +func createGitTargetWithParentBranch(name, namespace, providerName, path, branch, parentBranch string) { + applyGitTarget(name, namespace, providerName, path, branch, parentBranch, + e2eEncryptionRefName, false, gitTargetCommitOptions{Window: e2eCommitWindow}) +} + +func applyGitTarget( + name, + namespace, + providerName, + path, + branch, + parentBranch, + encryptionSecretName string, + generateWhenMissing bool, + commit gitTargetCommitOptions, ) { By(fmt.Sprintf("creating GitTarget '%s' in ns '%s' for GitProvider '%s' with path '%s'", name, namespace, providerName, path)) @@ -261,6 +281,7 @@ func createGitTargetWithOptions( Namespace string ProviderName string Branch string + ParentBranch string Path string EncryptionSecretName string GenerateWhenMissing bool @@ -273,6 +294,7 @@ func createGitTargetWithOptions( Namespace: namespace, ProviderName: providerName, Branch: branch, + ParentBranch: parentBranch, Path: path, EncryptionSecretName: encryptionSecretName, GenerateWhenMissing: generateWhenMissing, diff --git a/test/e2e/new_branch_parent_e2e_test.go b/test/e2e/new_branch_parent_e2e_test.go new file mode 100644 index 00000000..1f5ece94 --- /dev/null +++ b/test/e2e/new_branch_parent_e2e_test.go @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A GitTarget whose write branch does not exist yet stays on standby, and its first commit lands +// on the parent branch's CURRENT tip. +// +// The write branch is created from the remote's default branch (its parent) only when a write has +// something to commit. Until then nothing is pushed, including for a live event the parent +// already holds. When a write does commit, publication checks the parent against the push +// session's advertisement, so a worker that checked out the parent long ago still creates the +// branch on the parent's tip rather than on its own stale checkout. +// +// The stale checkout is the whole point, so the spec builds it deliberately: the target's own +// initial snapshot clones main at A and trusts it before B is pushed. A cold worker would fetch on +// its first write and land on B without exercising the check at all. +// +// The spec asserts ancestry and content, not which fetch found B: with periodic refresh on, a +// refresh can legitimately find B before the publication does. That publication alone finds a +// moved parent, with no refresher at all, is proved deterministically by the worker tests +// TestBranchWorker_LiveWriteOnAbsentBranchStartsFromTheParentsCurrentTip and +// TestBranchWorker_NewBranchIsNotPublishedOnAnUncheckedParent. +// +// It runs twice: once with spec.parentBranch omitted (the remote's default branch, main), and once +// naming `release`, which the GitProvider does not allow writing to because it is only read. A +// third context names a parent that does not exist yet, and recovers once it is pushed. +var _ = Describe("New write branch starts from its parent", Label("manager"), func() { + Context("with spec.parentBranch omitted", Ordered, func() { + describeNewWriteBranchSpec("", "main", "parent-branch") + }) + Context("with spec.parentBranch naming release", Ordered, func() { + describeNewWriteBranchSpec("release", "release", "parent-release") + }) + Context("with spec.parentBranch naming a branch that does not exist yet", Ordered, func() { + describeMissingParentRecoverySpec() + }) +}) + +const parentBranchSelectorLabel = "e2e.configbutler.ai/parent-branch" + +// applyParentConfigMap applies a ConfigMap the parent-branch specs' WatchRule selects. +func applyParentConfigMap(namespace, name string) { + GinkgoHelper() + manifest := fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s + labels: + %s: "yes" +data: + key: %s +`, name, namespace, parentBranchSelectorLabel, name) + out, err := kubectlRunWithStdin(namespace, manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply ConfigMap %q: %s", name, out) +} + +// applyParentSpecProvider declares the GitProvider the parent-branch specs write through, allowing +// main and the write branch only: a parent other than main is read, never written. +func applyParentSpecProvider(namespace, providerName string, repo *RepoArtifacts, writeBranch string) { + GinkgoHelper() + provider := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: GitProvider +metadata: + name: %s + namespace: %s +spec: + url: %s + allowedBranches: ["main", %q] + secretRef: + name: %s +`, providerName, namespace, repo.RepoURLHTTP, writeBranch, repo.GitSecretHTTP) + out, err := kubectlRunWithStdin(namespace, provider, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply GitProvider: %s", out) + verifyResourceStatus("gitprovider", providerName, namespace, "True", "Succeeded", "") +} + +// applyParentSpecWatchRule declares the WatchRule that selects the specs' labelled ConfigMaps. +func applyParentSpecWatchRule(namespace, ruleName, destName string) { + GinkgoHelper() + rule := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: WatchRule +metadata: + name: %s + namespace: %s +spec: + gitTargetRef: + name: %s + rules: + - apiGroups: [""] + resources: ["configmaps"] + objectSelector: + matchLabels: + %s: "yes" +`, ruleName, namespace, destName, parentBranchSelectorLabel) + out, err := kubectlRunWithStdin(namespace, rule, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "apply WatchRule: %s", out) +} + +// describeMissingParentRecoverySpec: a configured parent that does not exist refuses, writes +// nothing and creates no branch. Once the parent is pushed, the target recovers with no further +// cluster edit: the branch worker finds the parent on its own probe, asks for a fresh snapshot, +// and the edit made while the parent was missing lands on the parent's tip. +func describeMissingParentRecoverySpec() { + const ( + providerName = "missing-parent-provider" + destName = "missing-parent-dest" + ruleName = "missing-parent-rule" + gitPath = "apps/missing-parent" + ) + var ( + testNs string + repo *RepoArtifacts + writeBranch string + ) + + BeforeAll(func() { + testNs = testNamespaceFor("manager-missing-parent") + _, _ = kubectlRun("create", "namespace", testNs) + writeBranch = fmt.Sprintf("reverser-e2e-%d", GinkgoRandomSeed()) + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-missing-parent-%d", GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + applyParentSpecProvider(testNs, providerName, repo, writeBranch) + }) + + AfterAll(func() { + cleanupWatchRule(ruleName, testNs) + cleanupGitTarget(destName, testNs) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", providerName, "--ignore-not-found=true") + cleanupNamespace(testNs) + }) + + It("refuses while the parent is missing, then recovers on its own once it is pushed", func() { + commitFilesToBranchFromOutside(repo, testNs, "main", "e2e: seed main", map[string]string{ + "README.md": "main is not the parent here\n", + }) + createGitTargetWithParentBranch(destName, testNs, providerName, gitPath, writeBranch, "release") + applyParentSpecWatchRule(testNs, ruleName, destName) + + By("an edit made while release does not exist is not written") + applyParentConfigMap(testNs, "made-while-missing") + Eventually(func(g Gomega) { + g.Expect(readyReasonOf(g, destName, testNs)).To(Equal("ParentBranchNotFound")) + }, 3*time.Minute, 3*time.Second).Should(Succeed()) + Expect(remoteBranchHeadOf(Default, repo.CheckoutDir, writeBranch)).To(BeEmpty(), "no orphan branch") + + By("release is pushed from outside; nothing in the cluster changes after this") + releaseTip := commitFilesToBranchFromOutside(repo, testNs, "release", "e2e: release appears", + map[string]string{"RELEASE.md": "release\n"}) + + By("the write branch is created on release's tip, carrying the edit made while it was missing") + editPath := path.Join(gitPath, testNs, "configmaps", "made-while-missing.yaml") + Eventually(func(g Gomega) { + tip := remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch) + g.Expect(tip).NotTo(BeEmpty(), "the recovery creates the write branch") + _, fetchErr := gitRun(repo.CheckoutDir, "fetch", "origin", writeBranch) + g.Expect(fetchErr).NotTo(HaveOccurred()) + first, revErr := gitRun(repo.CheckoutDir, "rev-list", "--max-parents=1", "--reverse", + "origin/release.."+tip) + g.Expect(revErr).NotTo(HaveOccurred()) + commits := strings.Fields(first) + g.Expect(commits).NotTo(BeEmpty()) + parentOfFirst, revErr := gitRun(repo.CheckoutDir, "rev-parse", commits[0]+"^") + g.Expect(revErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(parentOfFirst)).To(Equal(releaseTip), + "the branch's first commit sits on release's tip") + _, showErr := gitRun(repo.CheckoutDir, "cat-file", "-e", tip+":"+editPath) + g.Expect(showErr).NotTo(HaveOccurred(), "the edit made while release was missing is on the branch") + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(readyReasonOf(g, destName, testNs)).NotTo( + BeElementOf("ParentBranchNotFound", "RecoveringParentBranch")) + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + }) +} + +func describeNewWriteBranchSpec(parentBranch, parent, slug string) { + const ( + providerName = "parent-provider" + destName = "parent-dest" + ruleName = "parent-rule" + gitPath = "apps/parent" + ) + var ( + testNs string + repo *RepoArtifacts + writeBranch string + ) + + applyConfigMap := func(name string) { + GinkgoHelper() + applyParentConfigMap(testNs, name) + } + + // mirroredDocument is the document the controller writes for applyConfigMap(name). + mirroredDocument := func(name string) string { + return fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + labels: + %s: "yes" + name: %s + namespace: %s +data: + key: %s +`, parentBranchSelectorLabel, name, testNs, name) + } + + branchSeries := func(metric, extra string) string { + return fmt.Sprintf(`sum(%s{provider_namespace=%q,branch=%q%s}) or vector(0)`, + metric, testNs, writeBranch, extra) + } + pushes := func() float64 { + GinkgoHelper() + v, err := queryPrometheus(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`)) + Expect(err).NotTo(HaveOccurred()) + return v + } + + BeforeAll(func() { + ensurePrometheusClient() + testNs = testNamespaceFor("manager-" + slug) + _, _ = kubectlRun("create", "namespace", testNs) + writeBranch = fmt.Sprintf("reverser-e2e-%d", GinkgoRandomSeed()) + + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-%s-%d", slug, GinkgoRandomSeed())) + _, err := kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + + applyParentSpecProvider(testNs, providerName, repo, writeBranch) + }) + + AfterAll(func() { + cleanupWatchRule(ruleName, testNs) + cleanupGitTarget(destName, testNs) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", providerName, "--ignore-not-found=true") + cleanupNamespace(testNs) + }) + + It("stays on standby, then creates the write branch on the parent's current tip", func() { + By("seeding " + parent + " with commit A, whose folder already holds a document the cluster will match") + if parent != "main" { + commitFilesToBranchFromOutside(repo, testNs, "main", "e2e: seed main", map[string]string{ + "README.md": "main is not the parent here\n", + }) + } + hashA := commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: seed A", map[string]string{ + "README.md": "seed\n", + path.Join(gitPath, testNs, "configmaps", "standby-present.yaml"): mirroredDocument("standby-present"), + }) + + By("creating a GitTarget on an absent write branch, and a WatchRule that selects nothing yet") + createGitTargetWithParentBranch(destName, testNs, providerName, gitPath, writeBranch, parentBranch) + applyParentSpecWatchRule(testNs, ruleName, destName) + verifyResourceStatus("watchrule", ruleName, testNs, "True", "Succeeded", "") + waitForWatchRuleStreamsRunning(ruleName, testNs) + + // The initial snapshot is an in-sync no-op that still clones: this is the worker checking + // out A and trusting it, which is the stale state the rest of the spec depends on. + waitForMetricWithTimeout(branchSeries("gitopsreverser_git_fetches_total", ""), + func(v float64) bool { return v > 0 }, + "the worker for the write branch checked out the parent", 2*time.Minute) + + By("status.remote names the parent the write branch would be created from") + Eventually(func(g Gomega) { + g.Expect(gitTargetRemoteField(g, destName, testNs, "commit")).To(BeEmpty(), "the write branch is absent") + g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.state")).To(Equal("Found")) + g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.branch")).To(Equal(parent), + "with spec.parentBranch omitted this is the default branch the remote resolved") + g.Expect(gitTargetRemoteField(g, destName, testNs, "parent.commit")).To(Equal(hashA)) + }, 3*time.Minute, 5*time.Second).Should(Succeed()) + + By("an idle, in-sync target creates no branch") + Consistently(func(g Gomega) { + g.Expect(remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch)).To(BeEmpty(), + "idle means no branch and no commit") + }, 15*time.Second, 3*time.Second).Should(Succeed()) + + By("a live event the parent already holds publishes nothing") + pushesBefore := pushes() + applyConfigMap("standby-present") + // The no-diff window is still pushed, because only the remote can confirm "already + // present". Wait for that push cycle to finish before judging the branch. + waitForMetricWithTimeout(branchSeries("gitopsreverser_git_pushes_total", `,outcome="pushed"`), + func(v float64) bool { return v > pushesBefore }, + "the no-diff window reached the remote", 2*time.Minute) + Expect(remoteBranchHeadOf(Default, repo.CheckoutDir, writeBranch)).To(BeEmpty(), + "a write that commits nothing must not create the write branch") + + By(parent + " moves to B from outside, touching the target's folder") + hashB := commitFilesToBranchFromOutside(repo, testNs, parent, "e2e: B moves the parent", map[string]string{ + path.Join(gitPath, "NOTES.md"): "added on the parent at B\n", + }) + + By("a live edit arrives with no resync in between") + applyConfigMap("standby-edit") + + By("the write branch is created on B and carries the edit") + editPath := path.Join(gitPath, testNs, "configmaps", "standby-edit.yaml") + Eventually(func(g Gomega) { + tip := remoteBranchHeadOf(g, repo.CheckoutDir, writeBranch) + g.Expect(tip).NotTo(BeEmpty(), "the edit creates the write branch") + _, fetchErr := gitRun(repo.CheckoutDir, "fetch", "origin", writeBranch) + g.Expect(fetchErr).NotTo(HaveOccurred()) + parent, revErr := gitRun(repo.CheckoutDir, "rev-parse", tip+"^") + g.Expect(revErr).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(parent)).To(Equal(hashB), + "the first commit must sit on the parent's current tip") + for _, file := range []string{path.Join(gitPath, "NOTES.md"), editPath} { + _, showErr := gitRun(repo.CheckoutDir, "cat-file", "-e", tip+":"+file) + g.Expect(showErr).NotTo(HaveOccurred(), "%s must be on the new branch", file) + } + }, 2*time.Minute, 3*time.Second).Should(Succeed()) + }) +} + +// commitFilesToBranchFromOutside commits files to a branch and pushes it directly, the way another +// writer would, and returns the new tip. A branch the remote does not carry yet starts from main, +// or from nothing in an empty repository. It retries because losing a push race with the +// controller says nothing about the behavior under test. +func commitFilesToBranchFromOutside( + repo *RepoArtifacts, namespace, branch, message string, files map[string]string, +) string { + GinkgoHelper() + configureRepoOriginWithCredentials(repo, namespace) + + Eventually(func() error { + return attemptCommitFilesToBranch(repo, branch, message, files) + }, seedPushTimeout, seedPushInterval).Should(Succeed(), "the outside writer kept losing the push race") + + head, err := gitRun(repo.CheckoutDir, "rev-parse", "HEAD") + Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(head) +} + +// attemptCommitFilesToBranch is one attempt, from the CURRENT remote tip. It returns errors rather +// than asserting, so Eventually can rebuild on the tip that beat it. +func attemptCommitFilesToBranch(repo *RepoArtifacts, branch, message string, files map[string]string) error { + runGit := func(args ...string) error { + if out, err := gitRun(repo.CheckoutDir, args...); err != nil { + return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, out) + } + return nil + } + if err := checkoutRemoteTipOf(repo, branch, runGit); err != nil { + return err + } + for name, content := range files { + full := filepath.Join(repo.CheckoutDir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil { + return err + } + if err := os.WriteFile(full, []byte(content), 0o600); err != nil { + return err + } + if err := runGit("add", name); err != nil { + return err + } + } + if err := runGit("commit", "-m", message); err != nil { + return err + } + return runGit("push", "origin", "HEAD:refs/heads/"+branch) +} + +// checkoutRemoteTipOf checks out branch at its remote tip; a branch the remote does not carry +// starts from main, and from an orphan when main is absent too. +func checkoutRemoteTipOf(repo *RepoArtifacts, branch string, runGit func(...string) error) error { + for _, start := range []string{branch, "main"} { + if _, err := gitRun(repo.CheckoutDir, "fetch", "origin", start); err != nil { + continue + } + if err := runGit("checkout", "-B", branch, "origin/"+start); err != nil { + return err + } + return runGit("reset", "--hard", "origin/"+start) + } + if err := runGit("checkout", "--orphan", branch); err != nil { + return err + } + _, _ = gitRun(repo.CheckoutDir, "rm", "-rf", ".") + return nil +} diff --git a/test/e2e/repo_assertions_test.go b/test/e2e/repo_assertions_test.go index f720cc2f..11e09e7d 100644 --- a/test/e2e/repo_assertions_test.go +++ b/test/e2e/repo_assertions_test.go @@ -45,8 +45,15 @@ func pullLatestRepoState(g Gomega, checkoutDir string) { // commit, and UNCHANGED (== the prior SHA) for an already-established branch. func remoteBranchHead(g Gomega, checkoutDir string) string { GinkgoHelper() + return remoteBranchHeadOf(g, checkoutDir, "main") +} + +// remoteBranchHeadOf is remoteBranchHead for any branch: the SHA at its tip on the remote, or "" +// when the remote does not carry it. +func remoteBranchHeadOf(g Gomega, checkoutDir, branch string) string { + GinkgoHelper() - output, err := gitRun(checkoutDir, "ls-remote", "--heads", "origin", "main") + output, err := gitRun(checkoutDir, "ls-remote", "--heads", "origin", branch) g.Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("git ls-remote must succeed even when the branch is absent. Output: %s", output)) @@ -54,7 +61,7 @@ func remoteBranchHead(g Gomega, checkoutDir string) string { if len(fields) == 0 { return "" // branch does not exist yet — nothing committed } - return fields[0] // ls-remote prints "\trefs/heads/main" + return fields[0] // ls-remote prints "\trefs/heads/" } func recentCommitDiagnostics(checkoutDir, pathspec string) string { diff --git a/test/e2e/templates/gittarget.tmpl b/test/e2e/templates/gittarget.tmpl index 3511362c..3a5fccc9 100644 --- a/test/e2e/templates/gittarget.tmpl +++ b/test/e2e/templates/gittarget.tmpl @@ -7,6 +7,9 @@ spec: gitProviderRef: name: {{ .ProviderName }} branch: {{ .Branch }} +{{- if .ParentBranch }} + parentBranch: {{ .ParentBranch }} +{{- end }} path: {{ .Path }} commit: window: