diff --git a/.coverage-baseline b/.coverage-baseline index 073f80d75..18a2c6385 100644 --- a/.coverage-baseline +++ b/.coverage-baseline @@ -1 +1 @@ -80.0 +80.2 diff --git a/.docs-lint-scope b/.docs-lint-scope index 5e1a3ec30..ef4b57fd0 100644 --- a/.docs-lint-scope +++ b/.docs-lint-scope @@ -25,11 +25,12 @@ README.md docs/architecture.md docs/configuration.md -docs/flux.md -docs/argocd.md +docs/bi-directional-flux.md +docs/bi-directional-argocd.md docs/spec/attribution.md docs/components.md docs/quickstart.md docs/demo/commit-window.md docs/facts/kubernetes-watch-options.md docs/spec/watch-event-ordering-and-attribution-grace.md +docs/bi-directional.md diff --git a/README.md b/README.md index 831a7d64a..ce40ebc98 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,11 @@ several source clusters, repositories, branches, and folders. ## Principles -- **Creates Git commits.** The only output is Git commits that reflect the state of your cluster. +- **Creates Git commits.** The output is Git commits that reflect the state of your cluster. Deploying is Flux's and Argo CD's job: run one alongside to bring changes from Git back into the - cluster. See [bi-directional usage](docs/bi-directional.md). + cluster. Only a folder that both GitOps Reverser writes and Flux or Argo CD applies needs more: an + apply trigger, so GitOps Reverser decides when it is applied. See + [bi-directional usage](docs/bi-directional.md). - **API-first, not API-only.** Most changes are assumed to arrive through the API, so that is what publishing is tuned for. Your changes end up in your remote Git repo within seconds. Other authors can push to the same branch, and that is handled without conflicts. If an API edit and a @@ -158,14 +160,15 @@ made after the changes or before them. It runs on top of the quickstart above. ## Try it with your existing repo Start with a [scratch branch](docs/configuration.md#seeing-what-a-target-will-do-before-it-does-it) -containing your existing manifests, with no reconciler deploying that branch. Select a small resource +containing your existing manifests, with nothing deploying that branch. Select a small resource scope and one destination folder. Inspect the initial commits before making a live edit, then check which source files changed. -The operator pushes directly to the configured branch. If you later write to a branch that Flux or -Argo CD deploys, read the [bidirectional guide](docs/bi-directional.md) first. Live edits can be -reverted by the reconciler, and [replaying captured state](docs/api-first-publication.md) can overwrite -concurrent Git edits to the same object. Decide per folder which side is authoritative. +The operator pushes directly to the configured branch. Flux or Argo CD deploying other folders is +unaffected. If one of them deploys the folder you write, read the +[bi-directional guide](docs/bi-directional.md) first. Without an apply trigger, live edits can be +reverted by Flux or Argo CD, and [replaying captured state](docs/api-first-publication.md) can +overwrite concurrent Git edits to the same object. Decide per folder which side is authoritative. ## Before you adopt it diff --git a/api/v1alpha3/applytrigger_types.go b/api/v1alpha3/applytrigger_types.go new file mode 100644 index 000000000..146ea9a53 --- /dev/null +++ b/api/v1alpha3/applytrigger_types.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" +) + +// What every apply trigger kind shares: the reference a GitTarget names one by, why an apply is +// asked for, and the operation a trigger's status records. The kinds themselves are in +// fluxtrigger_types.go and argocdtrigger_types.go. docs/definitions.md fixes the words: the +// APPLIER is the Flux Kustomization or Argo CD Application, and the TRIGGER is this operator's +// resource that holds it. + +// The apply trigger kinds GitTarget.spec.applyTriggerRef can name. +const ( + // ApplyTriggerKindFlux is a FluxTrigger, which holds a Flux Kustomization. + ApplyTriggerKindFlux = "FluxTrigger" + // ApplyTriggerKindArgoCD is an ArgoCDTrigger, which holds an Argo CD Application. + ApplyTriggerKindArgoCD = "ArgoCDTrigger" +) + +// The kinds of applier a trigger holds. +const ( + // ApplierKindKustomization is a Flux Kustomization (FluxTrigger.spec.kustomizationRef). + ApplierKindKustomization = "Kustomization" + // ApplierKindApplication is an Argo CD Application (ArgoCDTrigger.spec.applicationRef). + ApplierKindApplication = "Application" +) + +// Why a kind and a name, and no namespace: the kind keeps each trigger its own schema, the way +// Flux's sourceRef names a kind, and the namespace is the GitTarget's own because the trigger's +// ClusterProvider admits the namespace it is in (spec.accessFrom). A GitTarget that could name a +// trigger elsewhere would drive an applier that its own namespace was never admitted to. + +// ApplyTriggerReference names an apply trigger in the GitTarget's own namespace. +type ApplyTriggerReference struct { + // Kind is the trigger's kind: FluxTrigger or ArgoCDTrigger. + // +required + // +kubebuilder:validation:Enum=FluxTrigger;ArgoCDTrigger + Kind string `json:"kind"` + + // Name is the trigger's name. + // +required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + +// ApplyReason is why GitOps Reverser asked an applier to apply. +// +kubebuilder:validation:Enum=ForeignCommit;Refusal;Repair;CatchUp +type ApplyReason string + +const ( + // ApplyReasonForeignCommit is a commit GitOps Reverser did not push. + ApplyReasonForeignCommit ApplyReason = "ForeignCommit" + // ApplyReasonRefusal is a refused edit on a target with spec.onRefusal: Reapply. + ApplyReasonRefusal ApplyReason = "Refusal" + // ApplyReasonRepair puts back an API edit that an earlier apply reverted on the cluster after + // the edit was made. Git kept the edit; the cluster runs the older value until the repair. + ApplyReasonRepair ApplyReason = "Repair" + // ApplyReasonCatchUp keeps the applier current with GitOps Reverser's own pushes. + ApplyReasonCatchUp ApplyReason = "CatchUp" +) + +// ApplyOperationPhase is where an apply operation stands. +// +kubebuilder:validation:Enum=Requested;Running;Settling;Unresolved;Succeeded;Failed +type ApplyOperationPhase string + +const ( + // ApplyOperationRequested is an operation GitOps Reverser has asked the applier for. + ApplyOperationRequested ApplyOperationPhase = "Requested" + // ApplyOperationRunning is an operation the applier is seen working on. + ApplyOperationRunning ApplyOperationPhase = "Running" + // ApplyOperationSettling is a Flux operation that has applied, and whose Kustomization is not + // yet seen suspended again. + ApplyOperationSettling ApplyOperationPhase = "Settling" + // ApplyOperationUnresolved is an operation that has not completed in the time it was given. It + // is still followed, and no timer ends it. + ApplyOperationUnresolved ApplyOperationPhase = "Unresolved" + // ApplyOperationSucceeded is an operation that completed and applied. + ApplyOperationSucceeded ApplyOperationPhase = "Succeeded" + // ApplyOperationFailed is an operation that completed without applying the commit. It may have + // applied part of it. + ApplyOperationFailed ApplyOperationPhase = "Failed" +) + +// Completed reports whether the phase ends its operation. +func (p ApplyOperationPhase) Completed() bool { + return p == ApplyOperationSucceeded || p == ApplyOperationFailed +} + +// ApplyOperation is an apply GitOps Reverser started on the trigger's applier: the one in flight, +// or the last one, once it completed. The branch worker writes it on every transition. +type ApplyOperation struct { + // Phase is where the operation stands. Requested, Running, Settling and Unresolved are in + // flight: while one is, an edit to an object the applier applies can still be reverted by it, + // and the applier is asked nothing else. Succeeded and Failed have completed. + // +required + Phase ApplyOperationPhase `json:"phase"` + + // Reason is why the apply was asked for. With several GitTargets on the trigger, it is the + // strongest of their reasons. + // +optional + Reason ApplyReason `json:"reason,omitempty"` + + // Commit is the commit the applier was asked to apply. Empty when the remote did not carry the + // branch. + // +optional + Commit string `json:"commit,omitempty"` + + // Token identifies the request in the applier's own status: the reconcile.fluxcd.io/requestedAt + // value on a Flux object, or the "gitops-reverser request" entry in an Argo CD operation's info. + // +optional + Token string `json:"token,omitempty"` + + // SuspendToken is the reconcile.fluxcd.io/requestedAt value the suspend after the apply carried. + // Once the Kustomization reports it handled, nothing started before the suspend is still + // running. Flux only. + // +optional + SuspendToken string `json:"suspendToken,omitempty"` + + // AppliedRevision is the revision the applier reports it applied once the operation completed. + // It can differ from commit when the branch moved between the check and the apply. + // +optional + AppliedRevision string `json:"appliedRevision,omitempty"` + + // RequestedAt is when the applier was asked. + // +required + RequestedAt metav1.Time `json:"requestedAt"` + + // FinishedAt is when the operation was seen to complete. + // +optional + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` + + // Message says why a Failed operation failed, or why an Unresolved one is overdue. + // +optional + Message string `json:"message,omitempty"` +} + +// DefaultApplierQuietPeriod is FluxTrigger.spec.quietPeriod when it is omitted. +const DefaultApplierQuietPeriod = 30 * time.Second + +// ApplyTrigger is what every trigger kind answers, so the controllers, the branch workers and the +// dispatcher handle a FluxTrigger and an ArgoCDTrigger alike. +// +kubebuilder:object:generate=false +type ApplyTrigger interface { + metav1.Object + runtime.Object + + // TriggerKind is the trigger's kind (ApplyTriggerKind*). + TriggerKind() string + // ClusterProvider is the name of the ClusterProvider whose cluster holds the applier. + ClusterProvider() string + // ApplierKind is the kind of applier the trigger holds (ApplierKind*). + ApplierKind() string + // Applier is the object the trigger holds, its namespace defaulted to the trigger's own. + Applier() types.NamespacedName + // EffectiveQuietPeriod is how long the branch must be quiet before a CatchUp. + EffectiveQuietPeriod() time.Duration + // SourceBinding is the repository and branch the applier applies, as the trigger last read + // them, and false when it has not read them. + SourceBinding() (repositoryURL, branch string, known bool) + // SharedSource is the source object the applier reads, which other appliers may read too, and + // false when the applier has none it shares this way or the trigger has not read it. Appliers + // sharing one have one operation at a time. + SharedSource() (source types.NamespacedName, known bool) + // Prefetches reports whether a push may be fetched into the shared source ahead of the apply: + // every other reader of it is held, so a new artifact applies nothing on its own. + Prefetches() bool + // Operation is status.operation, and SetOperation replaces it and status.repairPending. + Operation() *ApplyOperation + RepairPending() bool + SetOperation(operation *ApplyOperation, repairPending bool) + // StatusConditions returns the condition set, and SetObservedGeneration stamps the generation. + StatusConditions() *[]metav1.Condition + SetObservedGeneration(generation int64) +} + +// NewApplyTrigger returns an empty trigger of kind, or nil for a kind this release does not know. +func NewApplyTrigger(kind string) ApplyTrigger { + switch kind { + case ApplyTriggerKindFlux: + return &FluxTrigger{} + case ApplyTriggerKindArgoCD: + return &ArgoCDTrigger{} + default: + return nil + } +} + +// defaultedNamespace is namespace, or fallback when it is empty. +func defaultedNamespace(namespace, fallback string) string { + if namespace == "" { + return fallback + } + return namespace +} + +// clusterProviderName is ref's name, or DefaultClusterProviderName when ref names none, the way +// GitTarget.SourceCluster resolves an omitted clusterProviderRef. +func clusterProviderName(ref *meta.LocalObjectReference) string { + if ref == nil || ref.Name == "" { + return DefaultClusterProviderName + } + return ref.Name +} diff --git a/api/v1alpha3/argocdtrigger_types.go b/api/v1alpha3/argocdtrigger_types.go new file mode 100644 index 000000000..15f202f67 --- /dev/null +++ b/api/v1alpha3/argocdtrigger_types.go @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + "strings" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Why no quietPeriod: it times a CatchUp, and an Application is never asked to catch up. With +// automated sync off it applies nothing on its own, so it holds no stale state to correct. + +// ArgoCDTriggerSpec names the Argo CD Application the trigger holds, and the cluster it is in. +type ArgoCDTriggerSpec struct { + // Immutable for the reason FluxTriggerSpec's references are: the trigger holds what it names. + + // ClusterProviderRef names the ClusterProvider whose cluster holds the Application, such as a + // management cluster running Argo CD for the source cluster. A provider without + // spec.kubeConfig is the operator's own cluster. The provider must admit this trigger's + // namespace (spec.accessFrom). Its identity needs get and update on the Application. Omitted, + // it is the ClusterProvider named "default", as for a GitTarget. Immutable. + // +optional + // +kubebuilder:default={name: "default"} + // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.clusterProviderRef.name must not be empty" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.clusterProviderRef is immutable; delete and recreate the ArgoCDTrigger to hold an Application elsewhere" + ClusterProviderRef *meta.LocalObjectReference `json:"clusterProviderRef,omitempty"` + + // ApplicationRef is the Argo CD Application that applies the folders of the GitTargets naming + // this trigger. It must have a single spec.source whose targetRevision names a branch, and + // automated sync off: either no spec.syncPolicy.automated, or automated.enabled: false, which + // keeps automated.prune for the syncs GitOps Reverser starts. GitOps Reverser starts each sync + // by setting the Application's operation, as Argo CD's automated sync does, pinned to the commit + // it checked, and never replaces an operation that is running. Its namespace defaults to the + // trigger's namespace. Immutable. + // +required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.applicationRef is immutable; delete and recreate the ArgoCDTrigger to hold another Application" + // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.applicationRef.name must not be empty" + ApplicationRef meta.NamespacedObjectReference `json:"applicationRef"` +} + +// ArgoCDTriggerStatus is what the trigger knows about its Application, and the sync it started. +type ArgoCDTriggerStatus struct { + // ObservedGeneration is the latest generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Conditions are Configured (an eligible GitTarget names the trigger; the Application exists, + // syncs a single source, has automated sync off, may be updated, and no other ArgoCDTrigger + // holds it) and the kstatus Ready/Reconciling/Stalled trio. A trigger acts only while an + // eligible GitTarget references it: one in its namespace that names it, is admitted by its + // ClusterProviders, and writes the repository and branch the Application syncs. A trigger no + // GitTarget names reads nothing in the Application's cluster and is Configured=Unknown, + // Ready=True, both NotReferenced; one named only by GitTargets that are not eligible is + // Configured NotBound or a refusal. Of the ArgoCDTriggers in line for one Application in one cluster, the one + // created first holds it; the others are Configured=False (HeldByAnotherTrigger) until it is + // deleted or no longer referenced, and the next in line takes it over. + // +optional + // +listType=map + // +listMapKey=type + // +patchMergeKey=type + // +patchStrategy=merge + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + // Source is the Application's spec.source, as last read, published while an eligible GitTarget + // references the trigger. A GitTarget naming this trigger must write the repository it syncs, on + // the branch its targetRevision names. + // +optional + Source *ArgoCDTriggerSource `json:"source,omitempty"` + + // Operation is the sync GitOps Reverser started on the Application: the one in flight, or the + // last one. Absent until the first. + // +optional + Operation *ApplyOperation `json:"operation,omitempty"` + + // RepairPending is true while a Repair sync is pending for the Application: a sync reverted an + // API edit made after it was asked for, so the cluster runs an older value than Git until the + // next sync. + // +optional + RepairPending bool `json:"repairPending,omitempty"` +} + +// ArgoCDTriggerSource is the source an Application syncs. +type ArgoCDTriggerSource struct { + // RepoURL is the Application's spec.source.repoURL. + // +optional + RepoURL string `json:"repoURL,omitempty"` + + // TargetRevision is the Application's spec.source.targetRevision, as written: a branch name, or + // refs/heads/ and the name. Empty is Argo CD's HEAD. + // +optional + TargetRevision string `json:"targetRevision,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Application",type=string,JSONPath=`.spec.applicationRef.name` +// +kubebuilder:printcolumn:name="Operation",type=string,JSONPath=`.status.operation.phase` +// +kubebuilder:printcolumn:name="Commit",type=string,JSONPath=`.status.operation.commit`,priority=1 +// +kubebuilder:printcolumn:name="Cluster",type=string,JSONPath=`.spec.clusterProviderRef.name`,priority=1 +// +kubebuilder:printcolumn:name="RepairPending",type=boolean,JSONPath=`.status.repairPending`,priority=1 +// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// ArgoCDTrigger holds an Argo CD Application, with automated sync off, for GitOps Reverser, which +// starts every sync it runs: only on a branch with nothing live missing from the commit, after a +// commit GitOps Reverser did not push, after a refused edit, and after a reverted edit. GitTargets +// name it with spec.applyTriggerRef. +type ArgoCDTrigger struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` + + // spec defines the Application the trigger holds + // +required + Spec ArgoCDTriggerSpec `json:"spec"` + + // status defines the observed state of ArgoCDTrigger + // +optional + Status ArgoCDTriggerStatus `json:"status,omitempty,omitzero"` +} + +// +kubebuilder:object:root=true + +// ArgoCDTriggerList contains a list of ArgoCDTrigger. +type ArgoCDTriggerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []ArgoCDTrigger `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ArgoCDTrigger{}, &ArgoCDTriggerList{}) +} + +var _ ApplyTrigger = (*ArgoCDTrigger)(nil) + +// TriggerKind is ArgoCDTrigger. +func (t *ArgoCDTrigger) TriggerKind() string { return ApplyTriggerKindArgoCD } + +// ClusterProvider is spec.clusterProviderRef.name, or "default" when it is unset, as the schema +// defaults it: a literal built in a test or by an older client still names a concrete provider. +func (t *ArgoCDTrigger) ClusterProvider() string { + return clusterProviderName(t.Spec.ClusterProviderRef) +} + +// ApplierKind is Application. +func (t *ArgoCDTrigger) ApplierKind() string { return ApplierKindApplication } + +// Applier is spec.applicationRef, its namespace defaulted to the trigger's own. +func (t *ArgoCDTrigger) Applier() types.NamespacedName { + return types.NamespacedName{ + Namespace: defaultedNamespace(t.Spec.ApplicationRef.Namespace, t.Namespace), + Name: t.Spec.ApplicationRef.Name, + } +} + +// EffectiveQuietPeriod is the default: it only times a CatchUp, which an Application is never asked. +func (t *ArgoCDTrigger) EffectiveQuietPeriod() time.Duration { return DefaultApplierQuietPeriod } + +// SourceBinding is status.source's repoURL, and the branch its targetRevision names ("HEAD" for an +// empty one, which follows no branch GitOps Reverser writes). +func (t *ArgoCDTrigger) SourceBinding() (string, string, bool) { + if t.Status.Source == nil { + return "", "", false + } + revision := strings.TrimPrefix(t.Status.Source.TargetRevision, "refs/heads/") + if revision == "" { + revision = "HEAD" + } + return t.Status.Source.RepoURL, revision, true +} + +// SharedSource is none: every sync pins its own revision, so Applications reading one repository +// do not disturb each other's operations. +func (t *ArgoCDTrigger) SharedSource() (types.NamespacedName, bool) { + return types.NamespacedName{}, false +} + +// Prefetches is false: an Application has no source object to fetch ahead. +func (t *ArgoCDTrigger) Prefetches() bool { return false } + +// Operation is status.operation. +func (t *ArgoCDTrigger) Operation() *ApplyOperation { return t.Status.Operation } + +// RepairPending is status.repairPending. +func (t *ArgoCDTrigger) RepairPending() bool { return t.Status.RepairPending } + +// SetOperation replaces status.operation and status.repairPending. +func (t *ArgoCDTrigger) SetOperation(operation *ApplyOperation, repairPending bool) { + t.Status.Operation, t.Status.RepairPending = operation, repairPending +} diff --git a/api/v1alpha3/audit_route_test.go b/api/v1alpha3/audit_route_test.go index a19b7df07..a03af9bdd 100644 --- a/api/v1alpha3/audit_route_test.go +++ b/api/v1alpha3/audit_route_test.go @@ -62,8 +62,8 @@ func TestClusterProvider_AuditRoute_SharedBySeveralProviders(t *testing.T) { delegating := ClusterProvider{ ObjectMeta: metav1.ObjectMeta{Name: "srcns-delegating"}, Spec: ClusterProviderSpec{ - AllowAnySourceNamespace: true, - Attribution: &ClusterProviderAttribution{AuditRoute: "default"}, + AllowCrossNamespace: &ClusterProviderCrossNamespace{Sources: true}, + Attribution: &ClusterProviderAttribution{AuditRoute: "default"}, }, } diff --git a/api/v1alpha3/clusterprovider_types.go b/api/v1alpha3/clusterprovider_types.go index 431868040..31a5bed1c 100644 --- a/api/v1alpha3/clusterprovider_types.go +++ b/api/v1alpha3/clusterprovider_types.go @@ -35,6 +35,13 @@ const DefaultClusterProviderName = "default" // spec.qps and spec.burst moved to spec.client. A pruned field would apply cleanly and drop the // throttle without a word, so the old spellings stay in the schema only to be refused by name. // +kubebuilder:validation:XValidation:rule="!has(self.qps) && !has(self.burst)",message="spec.qps and spec.burst moved to spec.client.qps and spec.client.burst" +// +// spec.allowAnySourceNamespace was removed for spec.allowCrossNamespace.sources. A pruned field +// would apply cleanly and stop the cross-namespace mirroring without a word, so it stays in the +// schema only so that true is refused by name. The rule sits on spec, not on the field: any spec +// change re-runs it, so CRD validation ratcheting cannot wave a stored true through on the next +// edit. false stays accepted, because the old default stored it on every provider. +// +kubebuilder:validation:XValidation:rule="!has(self.allowAnySourceNamespace) || !self.allowAnySourceNamespace",message="allowAnySourceNamespace was removed; set allowCrossNamespace.sources" type ClusterProviderSpec struct { // KubeConfig names the SOURCE CLUSTER this provider represents and the credentials to reach it // (Flux's meta.KubeConfigReference, embedded verbatim). OMITTED means the operator's own @@ -60,26 +67,15 @@ type ClusterProviderSpec struct { // +optional AccessFrom *NamespaceMatcher `json:"accessFrom,omitempty"` - // The dangerous case is an IN-CLUSTER provider (kubeConfig omitted), where same-name coupling - // WAS the boundary: setting this bypasses live namespace RBAC, letting the owner of an admitted - // GitTarget mirror another namespace's objects through the operator's own cluster-wide - // credential into a destination they control. Legitimate to grant deliberately, never by - // default or as a side effect of another field, which is why it defaults to false. + // AllowCrossNamespace grants what a namespace this provider admits may reach outside itself, + // through this provider's credential. Omitted, it may reach nothing outside itself. + // +optional + AllowCrossNamespace *ClusterProviderCrossNamespace `json:"allowCrossNamespace,omitempty"` - // AllowAnySourceNamespace delegates SOURCE-namespace selection to the GitTargets this provider - // admits. While false (the default) a WatchRule mirroring through this provider may watch only - // its OWN namespace. - // - // It grants no access by itself: the source credential's own RBAC remains the hard maximum, and - // a request it permits still fails 403 if the credential cannot read that namespace. What it - // delegates is the AUTHORITY to choose, so set it only when the owners of admitted GitTargets - // are trusted to pick a subset of what that credential may read. Every cross-namespace request - // needs it, including a rules[].sourceNamespace of "*", which reaches every namespace the - // credential can read. It does not apply to ClusterWatchRule, which selects no namespaces at - // all. + // RemovedAllowAnySourceNamespace is the removed spelling of allowCrossNamespace.sources. It is + // never read; setting it to true is refused. // +optional - // +kubebuilder:default=false - AllowAnySourceNamespace bool `json:"allowAnySourceNamespace,omitempty"` + RemovedAllowAnySourceNamespace *bool `json:"allowAnySourceNamespace,omitempty"` // Client overrides the operator's outgoing kube-client throttles for this cluster. Omitted, // the built-in defaults apply (20 QPS, burst 30). Ignored when kubeConfig is omitted (the @@ -103,6 +99,33 @@ type ClusterProviderSpec struct { Attribution *ClusterProviderAttribution `json:"attribution,omitempty"` } +// Each field is a power of its own, granted on purpose: sources lets a namespace READ other +// namespaces' objects into its own Git, appliers lets it suspend and apply other namespaces' +// deployments. On an IN-CLUSTER provider (kubeConfig omitted) same-name coupling WAS the boundary, +// so either one bypasses live namespace RBAC: the owner of an admitted object acts on another +// namespace through the operator's own credential. It is a delegation, not owner consent: it does +// not prove that the namespace owns the objects it reaches, or the repository it writes. + +// ClusterProviderCrossNamespace is what a namespace a ClusterProvider admits may reach outside +// itself. The trust is in every admitted namespace, for everything in reach of the provider's +// identity; it does not prove that the namespace owns what it reaches. +type ClusterProviderCrossNamespace struct { + // Sources lets a WatchRule mirroring through this provider watch namespaces other than its own, + // including a rules[].sourceNamespace of "*", which reaches every namespace the credential can + // read. False (the default): a WatchRule watches only its own namespace. It grants no access by + // itself: the source credential's RBAC remains the hard maximum. It does not apply to + // ClusterWatchRule, which selects no namespaces at all. + // +optional + Sources bool `json:"sources,omitempty"` + + // Appliers lets a FluxTrigger or ArgoCDTrigger reaching its applier through this provider name a + // Kustomization or Application in another namespace than its own. False (the default): only + // the trigger's own namespace, which in a remote cluster is the namespace of the same name; a + // trigger naming another is Configured=False (ApplierNamespaceNotAllowed). + // +optional + Appliers bool `json:"appliers,omitempty"` +} + // ClusterProviderClient holds the per-cluster kube-client throttles. type ClusterProviderClient struct { // QPS overrides the operator's outgoing kube-client query-per-second throttle for this @@ -275,11 +298,16 @@ func (p *ClusterProvider) AllowsNamespace(nsName string, nsLabels map[string]str return p.Spec.AccessFrom.Matches(nsName, nsLabels) } -// AllowsAnySourceNamespace reports whether this provider delegates source-namespace selection to -// the GitTargets it admits. See the field's documentation: false (the default) means a WatchRule -// mirroring through this provider may watch only its own namespace. -func (p *ClusterProvider) AllowsAnySourceNamespace() bool { - return p.Spec.AllowAnySourceNamespace +// AllowsCrossNamespaceSources reports whether a WatchRule mirroring through this provider may +// watch namespaces other than its own (spec.allowCrossNamespace.sources). +func (p *ClusterProvider) AllowsCrossNamespaceSources() bool { + return p.Spec.AllowCrossNamespace != nil && p.Spec.AllowCrossNamespace.Sources +} + +// AllowsCrossNamespaceAppliers reports whether a trigger reaching its applier through this provider +// may name one in another namespace than its own (spec.allowCrossNamespace.appliers). +func (p *ClusterProvider) AllowsCrossNamespaceAppliers() bool { + return p.Spec.AllowCrossNamespace != nil && p.Spec.AllowCrossNamespace.Appliers } func init() { diff --git a/api/v1alpha3/examples_test.go b/api/v1alpha3/examples_test.go index db71a4519..a90ef29b6 100644 --- a/api/v1alpha3/examples_test.go +++ b/api/v1alpha3/examples_test.go @@ -134,6 +134,10 @@ func newExampleObject(kind string) any { return &ClusterWatchRule{} case "CommitRequest": return &CommitRequest{} + case ApplyTriggerKindFlux: + return &FluxTrigger{} + case ApplyTriggerKindArgoCD: + return &ArgoCDTrigger{} } return nil } diff --git a/api/v1alpha3/fluxtrigger_types.go b/api/v1alpha3/fluxtrigger_types.go new file mode 100644 index 000000000..d4bec7ad0 --- /dev/null +++ b/api/v1alpha3/fluxtrigger_types.go @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha3 + +import ( + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// FluxTriggerFinalizer is the finalizer a FluxTrigger carries while it holds its Kustomization +// suspended: from before its first adoption write until it has handed the Kustomization back or +// over. Deleting the trigger, or the last eligible GitTarget naming it, suspends the Kustomization +// once more, waits until nothing started before that suspend is running, and then hands it back to +// Flux: spec.suspend is released, so the Kustomization is resumed. A trigger that never held its +// Kustomization has none. +const FluxTriggerFinalizer = "configbutler.ai/flux-trigger" + +// FluxTriggerSpec names the Flux Kustomization the trigger holds, and the cluster it is in. +type FluxTriggerSpec struct { + // Why immutable: the trigger holds the Kustomization it names (it suspends it while idle and + // hands it back when deleted), so pointing it elsewhere would leave the first one held by + // nothing. Delete and recreate the trigger instead; its finalizer hands the old one back. + + // ClusterProviderRef names the ClusterProvider whose cluster holds the Kustomization, such as a + // management cluster running Flux for the source cluster. A provider without spec.kubeConfig is + // the operator's own cluster. The provider must admit this trigger's namespace + // (spec.accessFrom). Its identity needs get and patch on the Kustomization and its + // GitRepository. Omitted, + // it is the ClusterProvider named "default", as for a GitTarget. Immutable. + // +optional + // +kubebuilder:default={name: "default"} + // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.clusterProviderRef.name must not be empty" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.clusterProviderRef is immutable; delete and recreate the FluxTrigger to hold a Kustomization elsewhere" + ClusterProviderRef *meta.LocalObjectReference `json:"clusterProviderRef,omitempty"` + + // KustomizationRef is the Flux Kustomization that applies the folders of the GitTargets naming + // this trigger. Its spec.sourceRef must be a GitRepository that follows a branch. Its namespace + // defaults to the trigger's namespace. Immutable. + // +required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec.kustomizationRef is immutable; delete and recreate the FluxTrigger to hold another Kustomization" + // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.kustomizationRef.name must not be empty" + KustomizationRef meta.NamespacedObjectReference `json:"kustomizationRef"` + + // QuietPeriod is how long the branch must hold no open commit window and no pending writes, + // after GitOps Reverser's own pushes, before the Kustomization is asked to catch up with them. + // Every new write restarts the wait. A Go duration of at most "1h". + // +optional + // +kubebuilder:default="30s" + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$" + // +kubebuilder:validation:XValidation:rule="duration(self) <= duration('1h')",message="spec.quietPeriod must be a Go duration of at most 1h" + QuietPeriod *metav1.Duration `json:"quietPeriod,omitempty"` +} + +// FluxTriggerStatus is what the trigger knows about its Kustomization, and the apply it started. +type FluxTriggerStatus struct { + // ObservedGeneration is the latest generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Conditions are Configured (an eligible GitTarget names the trigger; the Kustomization exists, + // applies a GitRepository that follows a branch, may be patched, and no other FluxTrigger holds + // it), Held (Suspended: suspended under GitOps Reverser with the last suspend handled; Applying: + // resumed for an apply GitOps Reverser started; Suspending: a suspend is not handled yet; + // Releasing: being handed back; NotReferenced; NotConfigured), and the kstatus + // Ready/Reconciling/Stalled trio. A trigger acts only while an eligible GitTarget references it: + // one in its namespace that names it, is admitted by its ClusterProviders, and writes the + // repository and branch the GitRepository fetches. A trigger no GitTarget names reads nothing + // in the Kustomization's cluster and is Configured=Unknown, Ready=True, both NotReferenced; one + // named only by GitTargets that are not eligible is Configured NotBound or a refusal. Of the + // FluxTriggers in line for one Kustomization in one cluster, one holds it: the one with an apply + // in flight, then the one holding it already, then the one created first. The others are + // Configured=False (HeldByAnotherTrigger) until it lets go, and the next in line takes it over. + // +optional + // +listType=map + // +listMapKey=type + // +patchMergeKey=type + // +patchStrategy=merge + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + // Source is the GitRepository the Kustomization applies, as last read, published while an + // eligible GitTarget references the trigger. A GitTarget naming this trigger must write the + // repository and branch it fetches. + // +optional + Source *FluxTriggerSource `json:"source,omitempty"` + + // Operation is the apply GitOps Reverser started on the Kustomization: the one in flight, or + // the last one. Absent until the first. + // +optional + Operation *ApplyOperation `json:"operation,omitempty"` + + // RepairPending is true while a Repair apply is pending for the Kustomization: an apply reverted + // an API edit made after it was asked for, so the cluster runs an older value than Git until the + // next apply. + // +optional + RepairPending bool `json:"repairPending,omitempty"` +} + +// FluxTriggerSource is the GitRepository a Kustomization applies. +type FluxTriggerSource struct { + // GitRepository is the Kustomization's spec.sourceRef, its namespace defaulted. + // +required + GitRepository meta.NamespacedObjectReference `json:"gitRepository"` + + // URL is the GitRepository's spec.url. + // +optional + URL string `json:"url,omitempty"` + + // Branch is the GitRepository's spec.ref.branch. + // +optional + Branch string `json:"branch,omitempty"` + + // Why information only (decision 2 of 2026-10-17 in docs/design/apply-trigger.md): one + // GitRepository for several Kustomizations is a normal setup, and a consumer no trigger names + // applies a folder that has no promise of protection anyway. + + // OtherConsumers are the other objects in the cluster that apply this GitRepository and that no + // FluxTrigger holds: Kustomizations, and HelmReleases whose chart comes from it. Each applies a + // new artifact the moment it arrives. Information only: it never affects Ready. + // +optional + // +listType=atomic + OtherConsumers []meta.NamespacedObjectKindReference `json:"otherConsumers,omitempty"` + + // Prefetch is true while GitOps Reverser fetches the GitRepository as soon as it learns of a push, + // before the apply: every consumer of it was listed and is held by a FluxTrigger, so a new + // artifact applies nothing on its own. False when otherConsumers is not empty, or the consumers + // could not be listed (the identity needs list on kustomizations, helmreleases and helmcharts). + // +optional + Prefetch bool `json:"prefetch,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Held",type=string,JSONPath=`.status.conditions[?(@.type=="Held")].reason` +// +kubebuilder:printcolumn:name="Kustomization",type=string,JSONPath=`.spec.kustomizationRef.name` +// +kubebuilder:printcolumn:name="Operation",type=string,JSONPath=`.status.operation.phase` +// +kubebuilder:printcolumn:name="Commit",type=string,JSONPath=`.status.operation.commit`,priority=1 +// +kubebuilder:printcolumn:name="Cluster",type=string,JSONPath=`.spec.clusterProviderRef.name`,priority=1 +// +kubebuilder:printcolumn:name="RepairPending",type=boolean,JSONPath=`.status.repairPending`,priority=1 +// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// FluxTrigger holds a Flux Kustomization for GitOps Reverser, which starts every apply it makes: +// only on a branch with nothing live missing from the commit, after a commit GitOps Reverser did +// not push, after a refused edit, after a reverted edit, and to catch up with its own pushes. +// GitTargets name it with spec.applyTriggerRef. +type FluxTrigger struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` + + // spec defines the Kustomization the trigger holds + // +required + Spec FluxTriggerSpec `json:"spec"` + + // status defines the observed state of FluxTrigger + // +optional + Status FluxTriggerStatus `json:"status,omitempty,omitzero"` +} + +// +kubebuilder:object:root=true + +// FluxTriggerList contains a list of FluxTrigger. +type FluxTriggerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []FluxTrigger `json:"items"` +} + +func init() { + SchemeBuilder.Register(&FluxTrigger{}, &FluxTriggerList{}) +} + +var _ ApplyTrigger = (*FluxTrigger)(nil) + +// TriggerKind is FluxTrigger. +func (t *FluxTrigger) TriggerKind() string { return ApplyTriggerKindFlux } + +// ClusterProvider is spec.clusterProviderRef.name, or "default" when it is unset, as the schema +// defaults it: a literal built in a test or by an older client still names a concrete provider. +func (t *FluxTrigger) ClusterProvider() string { return clusterProviderName(t.Spec.ClusterProviderRef) } + +// ApplierKind is Kustomization. +func (t *FluxTrigger) ApplierKind() string { return ApplierKindKustomization } + +// Applier is spec.kustomizationRef, its namespace defaulted to the trigger's own. +func (t *FluxTrigger) Applier() types.NamespacedName { + return types.NamespacedName{ + Namespace: defaultedNamespace(t.Spec.KustomizationRef.Namespace, t.Namespace), + Name: t.Spec.KustomizationRef.Name, + } +} + +// EffectiveQuietPeriod is spec.quietPeriod, resolving an omitted field to the default. +func (t *FluxTrigger) EffectiveQuietPeriod() time.Duration { + if t.Spec.QuietPeriod == nil { + return DefaultApplierQuietPeriod + } + return t.Spec.QuietPeriod.Duration +} + +// SourceBinding is status.source's url and branch. +func (t *FluxTrigger) SourceBinding() (string, string, bool) { + if t.Status.Source == nil { + return "", "", false + } + return t.Status.Source.URL, t.Status.Source.Branch, true +} + +// SharedSource is the GitRepository in status.source. +func (t *FluxTrigger) SharedSource() (types.NamespacedName, bool) { + if t.Status.Source == nil { + return types.NamespacedName{}, false + } + source := t.Status.Source.GitRepository + return types.NamespacedName{Namespace: source.Namespace, Name: source.Name}, true +} + +// Prefetches is status.source.prefetch. +func (t *FluxTrigger) Prefetches() bool { return t.Status.Source != nil && t.Status.Source.Prefetch } + +// Operation is status.operation. +func (t *FluxTrigger) Operation() *ApplyOperation { return t.Status.Operation } + +// RepairPending is status.repairPending. +func (t *FluxTrigger) RepairPending() bool { return t.Status.RepairPending } + +// SetOperation replaces status.operation and status.repairPending. +func (t *FluxTrigger) SetOperation(operation *ApplyOperation, repairPending bool) { + t.Status.Operation, t.Status.RepairPending = operation, repairPending +} diff --git a/api/v1alpha3/gittarget_types.go b/api/v1alpha3/gittarget_types.go index fb6e8c462..181482722 100644 --- a/api/v1alpha3/gittarget_types.go +++ b/api/v1alpha3/gittarget_types.go @@ -3,8 +3,6 @@ package v1alpha3 import ( - "time" - meta "github.com/fluxcd/pkg/apis/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -37,7 +35,7 @@ import ( // gitProviderRef/branch/path above — and defaults to a ClusterProvider named "default", so it is // always populated (never nil) and always jumpable. // +kubebuilder:validation:XValidation:rule="self.clusterProviderRef == oldSelf.clusterProviderRef",message="spec.clusterProviderRef is immutable; delete and recreate the GitTarget to change the cluster it mirrors" -// +kubebuilder:validation:XValidation:rule="!has(self.onRefusal) || self.onRefusal != 'Reapply' || has(self.applier)",message="spec.onRefusal: Reapply needs spec.applier, the object asked to reapply" +// +kubebuilder:validation:XValidation:rule="!has(self.onRefusal) || self.onRefusal != 'Reapply' || has(self.applyTriggerRef)",message="spec.onRefusal: Reapply needs spec.applyTriggerRef, the trigger asked to reapply" type GitTargetSpec struct { // GitProviderRef names the GitProvider that backs this target, in this GitTarget's own // namespace. Many GitTargets may name the same GitProvider. @@ -92,17 +90,17 @@ type GitTargetSpec struct { // be guessing on Flux's behalf, and the guess would drive a write. A reference the user declares // claims nothing. See docs/design/apply-trigger.md. - // Applier names the object that applies this folder back to its source cluster. With it set, - // GitOps Reverser decides when that object applies: it asks for an apply only when the branch - // holds no open commit window and no pending writes, after a commit it did not push, after a - // refused edit (spec.onRefusal: Reapply), and, for a Kustomization, once the branch has been - // quiet for quietPeriod, to keep it current with its own pushes. Switch off the applier's own - // triggers (short Flux intervals, Git-host webhooks into Flux, Argo CD automated sync), or they - // go on applying without that check. The reference is looked up in the cluster - // spec.applier.clusterProviderRef names, by default the source cluster, with the identity that - // ClusterProvider uses. + // ApplyTriggerRef names the apply trigger, in this GitTarget's namespace, that holds the applier + // of this folder: a FluxTrigger (a Flux Kustomization) or an ArgoCDTrigger (an Argo CD + // Application). With it set, GitOps Reverser decides when that applier applies: only when the + // branch holds no open commit window and no pending writes, after a commit it did not push, + // after a refused edit (spec.onRefusal: Reapply), after an apply reverted a newer API edit, and, + // for a Kustomization, once the branch has been quiet for the trigger's quietPeriod, to keep it + // current with its own pushes. The trigger's applier must apply this target's repository and + // branch; the ApplyTriggerReady condition says whether it does. Several GitTargets may name one + // trigger, and it is asked once for all of them. // +optional - Applier *GitTargetApplier `json:"applier,omitempty"` + ApplyTriggerRef *ApplyTriggerReference `json:"applyTriggerRef,omitempty"` // OnRefusal decides what happens when a live edit is refused because it has no legal // destination in the Git folder. A refusal produces no commit, so nothing reverts the edit @@ -110,12 +108,13 @@ type GitTargetSpec struct { // // "Ignore", the default, records the refusal in conditions and events and does nothing else. // - // "Reapply" asks this target's applier (spec.applier, which it requires) to apply the current - // tip once the branch holds no open window and no pending writes, which reverts the refused - // edit and nothing that is still waiting to be published. Only this target's applier is asked; - // if its GitRepository has to fetch to reach the tip, every Kustomization on that GitRepository - // applies the new revision, as for any push. Give the folder its own GitRepository to keep the - // apply to it alone. + // "Reapply" asks this target's applier (through spec.applyTriggerRef, which it requires) to + // apply the current tip once the branch holds no open window and no pending writes, which + // reverts the refused edit and nothing that is still waiting to be published. Only this + // target's applier is asked. If its GitRepository has to fetch to reach the tip, the + // Kustomizations a FluxTrigger holds stay suspended and ignore the new artifact; only the + // consumers no trigger holds, listed in the trigger's status.source.otherConsumers, apply it, as + // they would for any push. // // It fires only for an edit to a document the folder ALREADY holds, in a write that removes no // document anywhere: re-applying would otherwise prune, or put back, an object somebody else is @@ -444,11 +443,6 @@ type GitTargetStatus struct { // one this target points at. // +optional Remote *GitTargetRemoteStatus `json:"remote,omitempty"` - - // Applier is the last apply request made to spec.applier: the commit, when, and why. Absent - // until the first request. - // +optional - Applier *GitTargetApplierStatus `json:"applier,omitempty"` } // GitTargetRemoteStatus is the answer to "where is my branch, and when did we last prove it". @@ -635,7 +629,7 @@ type GitTargetRetentionStatus struct { // +kubebuilder:printcolumn:name="LayoutResolved",type=string,JSONPath=`.status.conditions[?(@.type=="LayoutResolved")].reason`,priority=1 // +kubebuilder:printcolumn:name="GitPathAccepted",type=string,JSONPath=`.status.conditions[?(@.type=="GitPathAccepted")].status`,priority=1 // +kubebuilder:printcolumn:name="RenderMatchesLive",type=string,JSONPath=`.status.conditions[?(@.type=="RenderMatchesLive")].status`,priority=1 -// +kubebuilder:printcolumn:name="ApplierConfigured",type=string,JSONPath=`.status.conditions[?(@.type=="ApplierConfigured")].status`,priority=1 +// +kubebuilder:printcolumn:name="ApplyTriggerReady",type=string,JSONPath=`.status.conditions[?(@.type=="ApplyTriggerReady")].status`,priority=1 // +kubebuilder:printcolumn:name="StreamsRunning",type=string,JSONPath=`.status.conditions[?(@.type=="StreamsRunning")].status`,priority=1 // +kubebuilder:printcolumn:name="SourceReachable",type=string,JSONPath=`.status.conditions[?(@.type=="SourceClusterReachable")].reason`,priority=1 // +kubebuilder:printcolumn:name="Encryption",type=string,JSONPath=`.spec.encryption.provider`,priority=1 @@ -676,15 +670,6 @@ func (g *GitTarget) SourceCluster() string { return g.Spec.ClusterProviderRef.Name } -// ApplierCluster is the ClusterProvider whose cluster holds spec.applier: its clusterProviderRef, -// or the source cluster (SourceCluster) when that is omitted. Call it only with spec.applier set. -func (g *GitTarget) ApplierCluster() string { - if ref := g.Spec.Applier.ClusterProviderRef; ref != nil && ref.Name != "" { - return ref.Name - } - return g.SourceCluster() -} - // IsLocalSource reports whether this GitTarget references the "default" ClusterProvider, which the // watch data plane maps to its local cluster context. It is a NAME test, not a claim about the // physical cluster: a "default" provider may carry a kubeConfig. It only supplies the pre-discovery @@ -718,143 +703,6 @@ const ( RefusalActionReapply RefusalAction = "Reapply" ) -// Why quietPeriod is refused rather than ignored with an applicationRef: it times the CatchUp, and -// Argo CD is never asked to catch up (with automated sync off it applies nothing on its own, so it -// holds no stale state to correct). Accepting it would be a knob that does nothing. That is also -// why the field carries no schema default: a default would be present on every object and fail the -// rule for every Application. - -// GitTargetApplier names the object that applies a GitTarget's folder to its source cluster. -// +kubebuilder:validation:XValidation:rule="has(self.kustomizationRef) != has(self.applicationRef)",message="spec.applier names exactly one of kustomizationRef and applicationRef" -// +kubebuilder:validation:XValidation:rule="!has(self.applicationRef) || !has(self.quietPeriod)",message="spec.applier.quietPeriod applies only to a kustomizationRef: Argo CD is never asked to catch up" -type GitTargetApplier struct { - // The same reference spec.clusterProviderRef is, for the same reason: a ClusterProvider is how - // GitOps Reverser reaches any cluster, with its credential and its accessFrom gate. Flux run - // from a management cluster is the usual way to deploy to a remote one, so the applier's - // cluster cannot be assumed to be the source cluster. Unlike the source, it may change: moving - // an applier does not change what the folder means. - - // ClusterProviderRef names the ClusterProvider whose cluster holds the applier, such as a - // management cluster running Flux or Argo CD for the source cluster. Omitted, it is the - // GitTarget's own spec.clusterProviderRef. The provider must admit this GitTarget's namespace - // (spec.accessFrom). Its identity needs get and patch on a Kustomization and its GitRepository, - // or get and update on an Application. A provider without spec.kubeConfig is the operator's own - // cluster. - // +optional - // +kubebuilder:validation:XValidation:rule="self.name != ''",message="spec.applier.clusterProviderRef.name must not be empty" - ClusterProviderRef *meta.LocalObjectReference `json:"clusterProviderRef,omitempty"` - - // KustomizationRef is a Flux Kustomization whose spec.sourceRef is a GitRepository on this - // target's repository and branch. Its namespace defaults to the GitTarget's namespace. GitOps - // Reverser asks it to apply the way `flux reconcile kustomization --with-source` does: it - // stamps reconcile.fluxcd.io/requestedAt on the GitRepository, and on the Kustomization too - // when the GitRepository's artifact already holds the commit. Exactly one of kustomizationRef - // and applicationRef is set. - // +optional - KustomizationRef *meta.NamespacedObjectReference `json:"kustomizationRef,omitempty"` - - // ApplicationRef is an Argo CD Application whose spec.source is this target's repository, with - // targetRevision this target's branch (the branch name, or refs/heads/ and the name). Its - // namespace defaults to the GitTarget's namespace. Automated sync must be off: the Application - // either has no spec.syncPolicy.automated, or sets automated.enabled: false, which keeps - // automated.prune for the syncs GitOps Reverser starts. One with automated sync on is - // ApplierConfigured=False (AutomatedSyncEnabled) and is never synced. GitOps Reverser starts - // each sync by setting the Application's operation, as Argo CD's automated sync does, pinned to - // the commit it checked, and never replaces an operation that is running. A multi-source - // Application is not supported. Exactly one of kustomizationRef and applicationRef is set. - // +optional - ApplicationRef *meta.NamespacedObjectReference `json:"applicationRef,omitempty"` - - // QuietPeriod is how long the branch must hold no open commit window and no pending writes, - // after GitOps Reverser's own pushes, before a Kustomization is asked to catch up with them. - // Flux applies on its own timers and on a controller restart from whatever artifact it holds, - // so an artifact left behind those pushes would revert edits that are already in Git. Every new - // write restarts the wait. A Go duration of at most "1h". Omitted, it is "30s". Only for a - // kustomizationRef: it is rejected with an applicationRef. - // +optional - // +kubebuilder:validation:Type=string - // +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$" - // +kubebuilder:validation:XValidation:rule="duration(self) <= duration('1h')",message="spec.applier.quietPeriod must be a Go duration of at most 1h" - QuietPeriod *metav1.Duration `json:"quietPeriod,omitempty"` -} - -// The kinds of applier spec.applier can name. -const ( - // ApplierKindKustomization is a Flux Kustomization (spec.applier.kustomizationRef). - ApplierKindKustomization = "Kustomization" - // ApplierKindApplication is an Argo CD Application (spec.applier.applicationRef). - ApplierKindApplication = "Application" -) - -// Kind is the kind of object the applier names: ApplierKindApplication for an applicationRef, -// otherwise ApplierKindKustomization. -func (a *GitTargetApplier) Kind() string { - if a.ApplicationRef != nil { - return ApplierKindApplication - } - return ApplierKindKustomization -} - -// Reference is the object the applier names, whichever kind it is. The zero reference when it -// names none, which the schema rejects. -func (a *GitTargetApplier) Reference() meta.NamespacedObjectReference { - switch { - case a.ApplicationRef != nil: - return *a.ApplicationRef - case a.KustomizationRef != nil: - return *a.KustomizationRef - default: - return meta.NamespacedObjectReference{} - } -} - -// CatchesUp reports whether the applier is asked to catch up with GitOps Reverser's own pushes. -// Only a Kustomization is: an Argo CD Application with automated sync off applies nothing on its -// own, so it never holds a stale state to correct. -func (a *GitTargetApplier) CatchesUp() bool { - return a.Kind() == ApplierKindKustomization -} - -// DefaultApplierQuietPeriod is spec.applier.quietPeriod when it is omitted. -const DefaultApplierQuietPeriod = 30 * time.Second - -// EffectiveQuietPeriod is spec.applier.quietPeriod, resolving an omitted field to the default. -func (a *GitTargetApplier) EffectiveQuietPeriod() time.Duration { - if a == nil || a.QuietPeriod == nil { - return DefaultApplierQuietPeriod - } - return a.QuietPeriod.Duration -} - -// ApplyReason is why GitOps Reverser asked an applier to apply. -// +kubebuilder:validation:Enum=ForeignCommit;Refusal;CatchUp -type ApplyReason string - -const ( - // ApplyReasonForeignCommit is a commit GitOps Reverser did not push. - ApplyReasonForeignCommit ApplyReason = "ForeignCommit" - // ApplyReasonRefusal is a refused edit on a target with spec.onRefusal: Reapply. - ApplyReasonRefusal ApplyReason = "Refusal" - // ApplyReasonCatchUp keeps the applier current with GitOps Reverser's own pushes. - ApplyReasonCatchUp ApplyReason = "CatchUp" -) - -// GitTargetApplierStatus is the last apply request GitOps Reverser made for this target. -type GitTargetApplierStatus struct { - // RequestedCommit is the commit the applier was asked to apply. - // +optional - RequestedCommit string `json:"requestedCommit,omitempty"` - - // RequestedAt is when the request was made. It says the applier was asked, never that it - // applied: read the applier's own status for that. - // +optional - RequestedAt *metav1.Time `json:"requestedAt,omitempty"` - - // Reason is why it was asked. - // +optional - Reason ApplyReason `json:"reason,omitempty"` -} - // EffectiveOnRefusal returns the refusal action, resolving an omitted field to the default. The // CRD default makes this redundant for objects that went through the API server, and not for a // literal built in a test or by an older client. A value this release does not know, such as the diff --git a/api/v1alpha3/helpers_test.go b/api/v1alpha3/helpers_test.go index 9f7d10fc6..e8e6c2f6e 100644 --- a/api/v1alpha3/helpers_test.go +++ b/api/v1alpha3/helpers_test.go @@ -3,6 +3,8 @@ package v1alpha3 import ( + "time" + "testing" meta "github.com/fluxcd/pkg/apis/meta" @@ -322,20 +324,120 @@ func TestIsLocalSource_IsANameTest(t *testing.T) { } } -// TestApplierCluster_DefaultsToTheSourceCluster: an omitted spec.applier.clusterProviderRef is the -// GitTarget's own source cluster, never the "default" provider, so a remote-source target keeps its -// meaning when the field is added to the API. -func TestApplierCluster_DefaultsToTheSourceCluster(t *testing.T) { - target := &GitTarget{Spec: GitTargetSpec{ - ClusterProviderRef: &meta.LocalObjectReference{Name: "prod-eu-1"}, - Applier: &GitTargetApplier{}, - }} - assert.Equal(t, "prod-eu-1", target.ApplierCluster()) - - target.Spec.Applier.ClusterProviderRef = &meta.LocalObjectReference{Name: "hub"} - assert.Equal(t, "hub", target.ApplierCluster()) - - target.Spec.ClusterProviderRef = nil - target.Spec.Applier.ClusterProviderRef = nil - assert.Equal(t, DefaultClusterProviderName, target.ApplierCluster()) +// TestApplyTrigger_AppliersDefaultToTheTriggersNamespace: a reference with no namespace names an +// applier in the trigger's own namespace, as Flux defaults a sourceRef. +func TestApplyTrigger_AppliersDefaultToTheTriggersNamespace(t *testing.T) { + flux := &FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a"}, + Spec: FluxTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: "hub"}, + KustomizationRef: meta.NamespacedObjectReference{Name: "apps"}, + }, + } + assert.Equal(t, "team-a/apps", flux.Applier().String()) + assert.Equal(t, "hub", flux.ClusterProvider()) + assert.Equal(t, DefaultApplierQuietPeriod, flux.EffectiveQuietPeriod()) + + argo := &ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-a"}, + Spec: ArgoCDTriggerSpec{ + ApplicationRef: meta.NamespacedObjectReference{Namespace: "argocd", Name: "apps"}, + }, + } + assert.Equal(t, "argocd/apps", argo.Applier().String()) +} + +// TestArgoCDTrigger_SourceBindingNamesTheBranch: targetRevision may spell a branch as refs/heads/, +// and an empty one is Argo CD's HEAD, which follows no branch GitOps Reverser writes. +func TestArgoCDTrigger_SourceBindingNamesTheBranch(t *testing.T) { + trigger := &ArgoCDTrigger{} + _, _, known := trigger.SourceBinding() + assert.False(t, known) + + for revision, branch := range map[string]string{"main": "main", "refs/heads/main": "main", "": "HEAD"} { + trigger.Status.Source = &ArgoCDTriggerSource{RepoURL: "https://example.com/r.git", TargetRevision: revision} + url, got, known := trigger.SourceBinding() + assert.True(t, known) + assert.Equal(t, "https://example.com/r.git", url) + assert.Equal(t, branch, got, "targetRevision %q", revision) + } +} + +// TestNewApplyTrigger_KnowsBothKinds: a kind this release does not know is nil, never a guess. +// TestApplyTrigger_SharedSource: a FluxTrigger shares the GitRepository it last read, and prefetches +// only when its status says so; an ArgoCDTrigger shares nothing and never prefetches. +func TestApplyTrigger_SharedSource(t *testing.T) { + flux := &FluxTrigger{} + _, known := flux.SharedSource() + assert.False(t, known, "a trigger that has not read its source shares none yet") + assert.False(t, flux.Prefetches()) + + flux.Status.Source = &FluxTriggerSource{ + GitRepository: meta.NamespacedObjectReference{Namespace: "flux-system", Name: "platform"}, + Prefetch: true, + } + source, known := flux.SharedSource() + assert.True(t, known) + assert.Equal(t, "flux-system/platform", source.String()) + assert.True(t, flux.Prefetches()) + + argo := &ArgoCDTrigger{} + _, known = argo.SharedSource() + assert.False(t, known, "every sync pins its own revision") + assert.False(t, argo.Prefetches()) +} + +func TestNewApplyTrigger_KnowsBothKinds(t *testing.T) { + assert.IsType(t, &FluxTrigger{}, NewApplyTrigger(ApplyTriggerKindFlux)) + assert.IsType(t, &ArgoCDTrigger{}, NewApplyTrigger(ApplyTriggerKindArgoCD)) + assert.Nil(t, NewApplyTrigger("WebhookTrigger")) +} + +// TestApplyTrigger_StatusAccessors: both kinds answer the shared interface from their own status, +// and SetOperation replaces the operation and repairPending together. +func TestApplyTrigger_StatusAccessors(t *testing.T) { + operation := &ApplyOperation{Phase: ApplyOperationRunning, Commit: "abc"} + quiet := &metav1.Duration{Duration: time.Minute} + flux := &FluxTrigger{Spec: FluxTriggerSpec{QuietPeriod: quiet}} + argo := &ArgoCDTrigger{} + for _, trigger := range []ApplyTrigger{flux, argo} { + assert.Nil(t, trigger.Operation()) + assert.False(t, trigger.RepairPending()) + trigger.SetOperation(operation, true) + assert.Equal(t, operation, trigger.Operation()) + assert.True(t, trigger.RepairPending()) + trigger.SetObservedGeneration(3) + assert.NotNil(t, trigger.StatusConditions()) + } + assert.Equal(t, int64(3), flux.Status.ObservedGeneration) + assert.Equal(t, int64(3), argo.Status.ObservedGeneration) + + assert.Equal(t, ApplyTriggerKindFlux, flux.TriggerKind()) + assert.Equal(t, ApplierKindKustomization, flux.ApplierKind()) + assert.Equal(t, time.Minute, flux.EffectiveQuietPeriod()) + assert.Equal(t, ApplyTriggerKindArgoCD, argo.TriggerKind()) + assert.Equal(t, ApplierKindApplication, argo.ApplierKind()) + assert.Equal(t, DefaultApplierQuietPeriod, argo.EffectiveQuietPeriod()) + assert.Equal(t, DefaultClusterProviderName, argo.ClusterProvider(), + "an omitted clusterProviderRef is the default provider, as for a GitTarget") + argo.Spec.ClusterProviderRef = &meta.LocalObjectReference{Name: "hub"} + assert.Equal(t, "hub", argo.ClusterProvider()) + + _, _, known := flux.SourceBinding() + assert.False(t, known) + flux.Status.Source = &FluxTriggerSource{URL: "https://example.com/r.git", Branch: "main"} + url, branch, known := flux.SourceBinding() + assert.True(t, known) + assert.Equal(t, "https://example.com/r.git", url) + assert.Equal(t, "main", branch) +} + +// TestApplyOperationPhase_Completed: only Succeeded and Failed end an operation. +func TestApplyOperationPhase_Completed(t *testing.T) { + for phase, completed := range map[ApplyOperationPhase]bool{ + ApplyOperationRequested: false, ApplyOperationRunning: false, ApplyOperationSettling: false, + ApplyOperationUnresolved: false, ApplyOperationSucceeded: true, ApplyOperationFailed: true, + } { + assert.Equal(t, completed, phase.Completed(), phase) + } } diff --git a/api/v1alpha3/namespace_matcher_test.go b/api/v1alpha3/namespace_matcher_test.go index 529cc291a..aefad78ec 100644 --- a/api/v1alpha3/namespace_matcher_test.go +++ b/api/v1alpha3/namespace_matcher_test.go @@ -128,6 +128,11 @@ func TestResourceRule_EffectiveSourceNamespace(t *testing.T) { // otherwise. func TestClusterProvider_DelegationFlagDefaultsClosed(t *testing.T) { provider := &ClusterProvider{} - assert.False(t, provider.AllowsAnySourceNamespace(), + assert.False(t, provider.AllowsCrossNamespaceSources(), "source-namespace delegation must never be on by default") + assert.False(t, provider.AllowsCrossNamespaceAppliers(), + "nor may a trigger reach another namespace's applier by default") + provider.Spec.AllowCrossNamespace = &ClusterProviderCrossNamespace{Appliers: true} + assert.False(t, provider.AllowsCrossNamespaceSources(), "each power is granted on its own") + assert.True(t, provider.AllowsCrossNamespaceAppliers()) } diff --git a/api/v1alpha3/status_accessors.go b/api/v1alpha3/status_accessors.go index 28e018970..bb3d04cba 100644 --- a/api/v1alpha3/status_accessors.go +++ b/api/v1alpha3/status_accessors.go @@ -54,3 +54,19 @@ func (r *ClusterWatchRule) SetObservedGeneration(generation int64) { // StatusConditions returns the condition set to write through. func (r *ClusterWatchRule) StatusConditions() *[]metav1.Condition { return &r.Status.Conditions } + +// SetObservedGeneration records the generation this object's status describes. +func (t *FluxTrigger) SetObservedGeneration(generation int64) { + t.Status.ObservedGeneration = generation +} + +// StatusConditions returns the condition set to write through. +func (t *FluxTrigger) StatusConditions() *[]metav1.Condition { return &t.Status.Conditions } + +// SetObservedGeneration records the generation this object's status describes. +func (t *ArgoCDTrigger) SetObservedGeneration(generation int64) { + t.Status.ObservedGeneration = generation +} + +// StatusConditions returns the condition set to write through. +func (t *ArgoCDTrigger) StatusConditions() *[]metav1.Condition { return &t.Status.Conditions } diff --git a/api/v1alpha3/watchrule_types.go b/api/v1alpha3/watchrule_types.go index 146eff09a..a40cc1b6a 100644 --- a/api/v1alpha3/watchrule_types.go +++ b/api/v1alpha3/watchrule_types.go @@ -12,7 +12,8 @@ import ( // WatchRuleSpec defines the desired state of WatchRule. // WatchRule selects NAMESPACED resources on its GitTarget's source cluster. Each rules[] item // carries its own source namespace: omitted for this WatchRule's own namespace, an explicit name, -// or "*" for every namespace the GitTarget admits. +// or "*" for every namespace the source credential can read (refused unless the ClusterProvider +// sets allowCrossNamespace.sources). type WatchRuleSpec struct { // GitTargetRef names the GitTarget this rule feeds, in this WatchRule's own namespace. // +required @@ -110,7 +111,7 @@ type ResourceRule struct { // "*" is one cluster-wide list and watch rather than a set of per-namespace ones, so its bound // is the credential's own RBAC and nothing else. Naming any namespace other than this rule's // own, "*" included, requires the GitTarget's ClusterProvider to admit the target's namespace - // AND to set spec.allowAnySourceNamespace; while that flag is false, "*" is refused outright. + // AND to set spec.allowCrossNamespace.sources; while that is false, "*" is refused outright. // // This changes only which namespace is WATCHED, never where objects are written: Git placement // follows each mirrored object's own namespace. @@ -126,7 +127,7 @@ type ResourceRule struct { // It kept its spelling and changed its meaning: it used to resolve live through // GitTarget.spec.allowedSourceNamespaces into a concrete set. That field is gone, so the token is // bounded by source-cluster RBAC alone and is refused while the ClusterProvider does not set -// spec.allowAnySourceNamespace. See docs/design/source-scope-simplification.md, which is the +// spec.allowCrossNamespace.sources. See docs/design/source-scope-simplification.md, which is the // definition of record. const SourceNamespaceWildcard = "*" diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index 0b0c51137..62b2724ff 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -9,7 +9,7 @@ package v1alpha3 import ( "github.com/fluxcd/pkg/apis/meta" "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -48,6 +48,168 @@ func (in *AgeRecipientsSpec) DeepCopy() *AgeRecipientsSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplyOperation) DeepCopyInto(out *ApplyOperation) { + *out = *in + in.RequestedAt.DeepCopyInto(&out.RequestedAt) + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplyOperation. +func (in *ApplyOperation) DeepCopy() *ApplyOperation { + if in == nil { + return nil + } + out := new(ApplyOperation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplyTriggerReference) DeepCopyInto(out *ApplyTriggerReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplyTriggerReference. +func (in *ApplyTriggerReference) DeepCopy() *ApplyTriggerReference { + if in == nil { + return nil + } + out := new(ApplyTriggerReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArgoCDTrigger) DeepCopyInto(out *ArgoCDTrigger) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDTrigger. +func (in *ArgoCDTrigger) DeepCopy() *ArgoCDTrigger { + if in == nil { + return nil + } + out := new(ArgoCDTrigger) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ArgoCDTrigger) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArgoCDTriggerList) DeepCopyInto(out *ArgoCDTriggerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ArgoCDTrigger, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDTriggerList. +func (in *ArgoCDTriggerList) DeepCopy() *ArgoCDTriggerList { + if in == nil { + return nil + } + out := new(ArgoCDTriggerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ArgoCDTriggerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArgoCDTriggerSource) DeepCopyInto(out *ArgoCDTriggerSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDTriggerSource. +func (in *ArgoCDTriggerSource) DeepCopy() *ArgoCDTriggerSource { + if in == nil { + return nil + } + out := new(ArgoCDTriggerSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArgoCDTriggerSpec) DeepCopyInto(out *ArgoCDTriggerSpec) { + *out = *in + if in.ClusterProviderRef != nil { + in, out := &in.ClusterProviderRef, &out.ClusterProviderRef + *out = new(meta.LocalObjectReference) + **out = **in + } + out.ApplicationRef = in.ApplicationRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDTriggerSpec. +func (in *ArgoCDTriggerSpec) DeepCopy() *ArgoCDTriggerSpec { + if in == nil { + return nil + } + out := new(ArgoCDTriggerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArgoCDTriggerStatus) DeepCopyInto(out *ArgoCDTriggerStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Source != nil { + in, out := &in.Source, &out.Source + *out = new(ArgoCDTriggerSource) + **out = **in + } + if in.Operation != nil { + in, out := &in.Operation, &out.Operation + *out = new(ApplyOperation) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDTriggerStatus. +func (in *ArgoCDTriggerStatus) DeepCopy() *ArgoCDTriggerStatus { + if in == nil { + return nil + } + out := new(ArgoCDTriggerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterProvider) DeepCopyInto(out *ClusterProvider) { *out = *in @@ -115,6 +277,21 @@ func (in *ClusterProviderClient) DeepCopy() *ClusterProviderClient { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterProviderCrossNamespace) DeepCopyInto(out *ClusterProviderCrossNamespace) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterProviderCrossNamespace. +func (in *ClusterProviderCrossNamespace) DeepCopy() *ClusterProviderCrossNamespace { + if in == nil { + return nil + } + out := new(ClusterProviderCrossNamespace) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterProviderList) DeepCopyInto(out *ClusterProviderList) { *out = *in @@ -160,6 +337,16 @@ func (in *ClusterProviderSpec) DeepCopyInto(out *ClusterProviderSpec) { *out = new(NamespaceMatcher) (*in).DeepCopyInto(*out) } + if in.AllowCrossNamespace != nil { + in, out := &in.AllowCrossNamespace, &out.AllowCrossNamespace + *out = new(ClusterProviderCrossNamespace) + **out = **in + } + if in.RemovedAllowAnySourceNamespace != nil { + in, out := &in.RemovedAllowAnySourceNamespace, &out.RemovedAllowAnySourceNamespace + *out = new(bool) + **out = **in + } if in.Client != nil { in, out := &in.Client, &out.Client *out = new(ClusterProviderClient) @@ -607,6 +794,144 @@ func (in *EncryptionSpec) DeepCopy() *EncryptionSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FluxTrigger) DeepCopyInto(out *FluxTrigger) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxTrigger. +func (in *FluxTrigger) DeepCopy() *FluxTrigger { + if in == nil { + return nil + } + out := new(FluxTrigger) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FluxTrigger) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FluxTriggerList) DeepCopyInto(out *FluxTriggerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FluxTrigger, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxTriggerList. +func (in *FluxTriggerList) DeepCopy() *FluxTriggerList { + if in == nil { + return nil + } + out := new(FluxTriggerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FluxTriggerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FluxTriggerSource) DeepCopyInto(out *FluxTriggerSource) { + *out = *in + out.GitRepository = in.GitRepository + if in.OtherConsumers != nil { + in, out := &in.OtherConsumers, &out.OtherConsumers + *out = make([]meta.NamespacedObjectKindReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxTriggerSource. +func (in *FluxTriggerSource) DeepCopy() *FluxTriggerSource { + if in == nil { + return nil + } + out := new(FluxTriggerSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FluxTriggerSpec) DeepCopyInto(out *FluxTriggerSpec) { + *out = *in + if in.ClusterProviderRef != nil { + in, out := &in.ClusterProviderRef, &out.ClusterProviderRef + *out = new(meta.LocalObjectReference) + **out = **in + } + out.KustomizationRef = in.KustomizationRef + if in.QuietPeriod != nil { + in, out := &in.QuietPeriod, &out.QuietPeriod + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxTriggerSpec. +func (in *FluxTriggerSpec) DeepCopy() *FluxTriggerSpec { + if in == nil { + return nil + } + out := new(FluxTriggerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FluxTriggerStatus) DeepCopyInto(out *FluxTriggerStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Source != nil { + in, out := &in.Source, &out.Source + *out = new(FluxTriggerSource) + (*in).DeepCopyInto(*out) + } + if in.Operation != nil { + in, out := &in.Operation, &out.Operation + *out = new(ApplyOperation) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FluxTriggerStatus. +func (in *FluxTriggerStatus) DeepCopy() *FluxTriggerStatus { + if in == nil { + return nil + } + out := new(FluxTriggerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitProvider) DeepCopyInto(out *GitProvider) { *out = *in @@ -795,60 +1120,6 @@ func (in *GitTarget) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GitTargetApplier) DeepCopyInto(out *GitTargetApplier) { - *out = *in - if in.ClusterProviderRef != nil { - in, out := &in.ClusterProviderRef, &out.ClusterProviderRef - *out = new(meta.LocalObjectReference) - **out = **in - } - if in.KustomizationRef != nil { - in, out := &in.KustomizationRef, &out.KustomizationRef - *out = new(meta.NamespacedObjectReference) - **out = **in - } - if in.ApplicationRef != nil { - in, out := &in.ApplicationRef, &out.ApplicationRef - *out = new(meta.NamespacedObjectReference) - **out = **in - } - if in.QuietPeriod != nil { - in, out := &in.QuietPeriod, &out.QuietPeriod - *out = new(v1.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetApplier. -func (in *GitTargetApplier) DeepCopy() *GitTargetApplier { - if in == nil { - return nil - } - out := new(GitTargetApplier) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GitTargetApplierStatus) DeepCopyInto(out *GitTargetApplierStatus) { - *out = *in - if in.RequestedAt != nil { - in, out := &in.RequestedAt, &out.RequestedAt - *out = (*in).DeepCopy() - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetApplierStatus. -func (in *GitTargetApplierStatus) DeepCopy() *GitTargetApplierStatus { - if in == nil { - return nil - } - out := new(GitTargetApplierStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitTargetCommitSpec) DeepCopyInto(out *GitTargetCommitSpec) { *out = *in @@ -1019,10 +1290,10 @@ func (in *GitTargetSpec) DeepCopyInto(out *GitTargetSpec) { *out = new(EncryptionSpec) (*in).DeepCopyInto(*out) } - if in.Applier != nil { - in, out := &in.Applier, &out.Applier - *out = new(GitTargetApplier) - (*in).DeepCopyInto(*out) + if in.ApplyTriggerRef != nil { + in, out := &in.ApplyTriggerRef, &out.ApplyTriggerRef + *out = new(ApplyTriggerReference) + **out = **in } if in.Placement != nil { in, out := &in.Placement, &out.Placement @@ -1091,11 +1362,6 @@ func (in *GitTargetStatus) DeepCopyInto(out *GitTargetStatus) { *out = new(GitTargetRemoteStatus) (*in).DeepCopyInto(*out) } - if in.Applier != nil { - in, out := &in.Applier, &out.Applier - *out = new(GitTargetApplierStatus) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitTargetStatus. diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index 73231299a..a3ba1243e 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -18,8 +18,8 @@ For the chart's optional starter `quickstart` block, see [`docs/configuration.md - ✅ **Git synchronization**: Push Kubernetes changes back to Git repositories - ✅ **Single-pod stability**: 1 replica by default while multi-pod support is in progress -- ✅ **Automatic CRD installation**: GitProvider, GitTarget, WatchRule, ClusterWatchRule, and - CommitRequest CRDs installed automatically +- ✅ **Automatic CRD installation**: ClusterProvider, GitProvider, GitTarget, WatchRule, + ClusterWatchRule, CommitRequest, FluxTrigger, and ArgoCDTrigger CRDs installed automatically - ✅ **Watch-based ingestion**: object state comes from the Kubernetes watch API - ✅ **Optional audit attribution**: name commit authors from kube-apiserver audit events over HTTPS - ✅ **Prometheus metrics**: Built-in monitoring support @@ -174,15 +174,17 @@ nodeSelector: | `servers.audit.timeouts.write` | Audit-server write timeout | `30s` | | `servers.audit.timeouts.idle` | Audit-server idle timeout | `60s` | | `servers.audit.tls.secretNameOverride` | Override Secret name for audit TLS cert/key | `-audit-server-cert` | +| `servers.receiver.enabled` | Serve the push receiver, over plain HTTP, for Git hosts' push webhooks to `GitProvider.status.webhookPath`, and add a `-receiver` Service; expose it with an Ingress or Gateway route of your own that terminates TLS. See [push receiver settings](../../docs/configuration.md#push-receiver-settings) | `false` | +| `servers.receiver.bindAddress` | host:port the push receiver binds to (`--receiver-bind-address`) | `:9292` | +| `servers.receiver.port` | Push receiver container port; must match the port in `bindAddress` | `9292` | | `controllerManager.additionalSensitiveResources` | Extra Secret-shaped resource types encrypted as `resource` or `group/resource` | `[]` | | `controllerManager.insecureAllowGitHTTP` | Permit credentials with `http://` GitProvider URLs for trusted in-cluster development Git servers | `false` | | `controllerManager.branchWorkerQueueDepth` | Event queue depth per branch worker. A full queue refuses the write (`git_queue_refusals_total`) instead of stalling the watch path; the watch delivers it again from its cursor, and only a cursor that expires first loses that write's own commit (and, under `prune.mode: OnEvent`, a delete's removal). Costs queue depth × payload of pod memory, which `branchBufferMaxSize` does **not** cover | `1000` | | `controllerManager.branchBufferMaxSize` | Maximum in-memory event buffer per branch worker. Bounds the open commit window plus the writes retained for replay until a push succeeds. While the remote cannot be reached, retained writes that reach it close the worker to new writes, saves and resyncs until they are published; the watch delivers a refused event again | `8Mi` | | `controllerManager.commitRequestTTL` | How long a `CommitRequest` is kept once it finishes. Written onto the request as the `configbutler.ai/delete-after` annotation; remove or edit that to keep one. `"0"` writes none | `48h` | | `git.refreshInterval` | How often an idle `GitTarget` re-proves where its branch is on the Git remote. `"0"` disables it | `10m` | -| `receiver.bindAddress` | `host:port` the push receiver listens on, over plain HTTP (`--receiver-bind-address`), for Git hosts' push webhooks to `GitProvider.status.webhookPath`. Empty or `"0"` serves no receiver. When set, the chart adds a `-receiver` Service; expose it with an Ingress or Gateway route of your own that terminates TLS. See [push receiver settings](../../docs/configuration.md#push-receiver-settings) | `""` | -| `applierAccess.flux` | Grant the operator `get` and `patch` on Flux `Kustomizations` and `GitRepositories`, and `create` on `SelfSubjectAccessReviews`, so it can ask Flux to apply a commit somebody else pushed | `false` | -| `applierAccess.argocd` | Grant the operator `get` and `update` on Argo CD `Applications`, and `create` on `SelfSubjectAccessReviews`, so it can start the syncs of an `Application` named by `spec.applier.applicationRef` | `false` | +| `applierAccess.flux` | Grant the operator `get`, `list` and `patch` on Flux `Kustomizations`, `get` and `patch` on `GitRepositories`, `list` on `HelmReleases` and `HelmCharts` (to find a source's other consumers), and `create` on `SelfSubjectAccessReviews`, so it can hold the `Kustomization` a `FluxTrigger` names suspended and resume it for each apply | `false` | +| `applierAccess.argocd` | Grant the operator `get` and `update` on Argo CD `Applications`, and `create` on `SelfSubjectAccessReviews`, so it can start the syncs of an `Application` an `ArgoCDTrigger` holds | `false` | | `auditService.type` | Service type for the dedicated audit Service | `NodePort` | | `auditService.nodePort` | Fixed NodePort for the audit Service when `auditService.type=NodePort` | `30444` | | `auditService.clusterIP` | Optional fixed ClusterIP for the dedicated audit Service | `""` | @@ -213,6 +215,8 @@ nodeSelector: | `clusterProvider.default.kubeConfig.secretRef.name` | Secret (release namespace) holding a kubeconfig for the rendered `default` provider. Empty means the operator's **own in-cluster** cluster; a name points `default` at a **remote** cluster instead. The name is a convention | `""` | | `clusterProvider.default.kubeConfig.secretRef.key` | Key within that Secret. Empty reads `value` then `value.yaml` | `""` | | `clusterProvider.default.accessFrom` | Deny-by-default policy (`names` and/or `selector`) for which **control-cluster** namespaces may reference this provider from a `GitTarget`. The default empty selector admits every namespace | `{selector: {}}` | +| `clusterProvider.default.allowCrossNamespace.sources` | Let a `WatchRule` in an admitted namespace watch other namespaces, `"*"` included. A delegation to every admitted namespace, not proof that it owns what it reads | `false` | +| `clusterProvider.default.allowCrossNamespace.appliers` | Let a `FluxTrigger` or `ArgoCDTrigger` in an admitted namespace suspend and apply a `Kustomization` or `Application` in another namespace, such as `flux-system`. A delegation to every admitted namespace, not proof that it owns the applier or the repository | `false` | | `servers.admission.enabled` | Install the validate-operator-types admission webhook that captures CommitRequest authors (a form of author attribution). Enabled by default; a no-op until `queue.redis.addr` is set | `true` | | `rbac.create` | Create the manager ClusterRole and its binding | `true` | | `rbac.watchTypes.mode` | Which types a `WatchRule` may read. `any` grants cluster-wide read on everything, including every Secret in the cluster. `selected` grants read on `rbac.watchTypes.selected` only, so the reverser cannot list or watch Secrets (it keeps `get` on named Secrets it is pointed at). See [`docs/rbac.md`](../../docs/rbac.md) | `any` | @@ -223,7 +227,7 @@ nodeSelector: | `servers.metrics.tls.secretNameOverride` | Override Secret name for metrics TLS cert/key | `-metrics-server-cert` | | `service.clusterIP` | Optional fixed ClusterIP for single controller Service | `""` | | `service.ports.audit` | Service port for audit ingress when `attribution.enabled=true` | `9444` | -| `service.ports.receiver` | Port of the `-receiver` Service when `receiver.bindAddress` is set | `80` | +| `service.ports.receiver` | Port of the `-receiver` Service when `servers.receiver.enabled=true` | `80` | | `service.ports.metrics` | Service port for metrics | `8080` | | `servers.healthProbe.bindAddress` | Liveness/readiness probe bind address (`--health-probe-bind-address`) | `:8081` | | `servers.admission.port` | Admission webhook container/Service port | `9443` | @@ -279,21 +283,32 @@ This chart automatically manages the following CRDs: - **`watchrules.configbutler.ai`** - Namespaced watch rules - **`clusterwatchrules.configbutler.ai`** - Cluster-scoped watch rules - **`commitrequests.configbutler.ai`** - One-shot "save now" signals that finalize the open commit window +- **`fluxtriggers.configbutler.ai`** - Apply triggers holding a Flux `Kustomization` for bi-directional GitOps +- **`argocdtriggers.configbutler.ai`** - Apply triggers holding an Argo CD `Application` for bi-directional GitOps ### CRD Lifecycle | Operation | Behavior | |-----------|----------| | `helm install` | ✅ CRDs installed automatically | -| `helm upgrade` | ✅ CRDs upgraded automatically | +| `helm upgrade` | ⚠️ CRDs NOT upgraded (Helm never upgrades `crds/`): apply the release's `crds.yaml` first, as below | | `helm uninstall` | ⚠️ CRDs NOT deleted (prevents data loss) | +Before `helm upgrade`, apply the new release's CRDs, so new kinds and changed schemas are served +before the controller that needs them starts: + +```bash +kubectl apply -f https://github.com/ConfigButler/gitops-reverser/releases/download//crds.yaml +``` + To manually remove CRDs after uninstallation: ```bash -kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai commitrequests.configbutler.ai +kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai commitrequests.configbutler.ai fluxtriggers.configbutler.ai argocdtriggers.configbutler.ai clusterproviders.configbutler.ai ``` +Delete the triggers while the operator still runs, before this ([Uninstallation](#uninstallation)). + > ⚠️ **Warning**: Deleting CRDs will also delete all custom resources of those types! ## Verification @@ -358,6 +373,20 @@ If upgrading from earlier chart versions: ## Uninstallation +**Hand Flux its `Kustomization`s back first, while the operator still runs.** A `FluxTrigger` that +holds its `Kustomization` carries the finalizer `configbutler.ai/flux-trigger`, and only the running +operator removes it, once it has handed the `Kustomization` back. One left after `helm uninstall` +keeps its namespace and the `fluxtriggers` CRD from being deleted, and its `Kustomization` stays +suspended. Delete your `GitTarget`s, or the triggers, before uninstalling, and wait until they are +gone: + +```bash +kubectl delete fluxtriggers,argocdtriggers --all --all-namespaces --wait +``` + +A trigger that never held its `Kustomization`, such as one no `GitTarget` names, carries no +finalizer. An `ArgoCDTrigger` never carries one. + ```bash # Uninstall chart helm uninstall gitops-reverser --namespace gitops-reverser @@ -366,7 +395,7 @@ helm uninstall gitops-reverser --namespace gitops-reverser kubectl delete namespace gitops-reverser # Delete CRDs (optional, but removes all custom resources) -kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai commitrequests.configbutler.ai +kubectl delete crd gitproviders.configbutler.ai gittargets.configbutler.ai watchrules.configbutler.ai clusterwatchrules.configbutler.ai commitrequests.configbutler.ai fluxtriggers.configbutler.ai argocdtriggers.configbutler.ai clusterproviders.configbutler.ai ``` ## Troubleshooting diff --git a/charts/gitops-reverser/templates/_helpers.tpl b/charts/gitops-reverser/templates/_helpers.tpl index 71576c9b3..3e2f666bf 100644 --- a/charts/gitops-reverser/templates/_helpers.tpl +++ b/charts/gitops-reverser/templates/_helpers.tpl @@ -108,23 +108,6 @@ as the Certificate name and in the webhook's cert-manager.io/inject-ca-from anno {{- printf "%s-admission-server-cert" (include "gitops-reverser.fullname" .) -}} {{- end }} -{{/* -Non-empty when the push receiver listener is on: receiver.bindAddress is set and is not the "0" -the binary reads as off. Used as `if include ...`, so it renders "true" or nothing. -*/}} -{{- define "gitops-reverser.receiverEnabled" -}} -{{- $addr := .Values.receiver.bindAddress | default "" | toString | trim -}} -{{- if and $addr (ne $addr "0") -}}true{{- end -}} -{{- end }} - -{{/* -The port in receiver.bindAddress ("host:port"; an empty host or an IPv6 "[::]" host included), so -the containerPort and the Service's targetPort cannot drift from what the binary binds. -*/}} -{{- define "gitops-reverser.receiverPort" -}} -{{- splitList ":" (.Values.receiver.bindAddress | toString | trim) | last | int -}} -{{- end }} - {{/* Name of the push receiver Service. */}} diff --git a/charts/gitops-reverser/templates/applier-rbac.yaml b/charts/gitops-reverser/templates/applier-rbac.yaml index 683b1704c..dcd76bb2a 100644 --- a/charts/gitops-reverser/templates/applier-rbac.yaml +++ b/charts/gitops-reverser/templates/applier-rbac.yaml @@ -1,8 +1,9 @@ {{- if and .Values.rbac.create .Values.applierAccess.flux }} -# What GitOps Reverser needs to ask Flux to apply a commit after somebody else pushed to a branch a -# GitTarget writes: read and patch the Kustomization and its GitRepository (an apply request is a -# reconcile request on them), and create SelfSubjectAccessReviews so the operator can check it holds -# that access before relying on it. Separate from the manager role because it is optional and +# What GitOps Reverser needs to drive a Kustomization a FluxTrigger holds: read and patch the +# Kustomization and its GitRepository (it suspends and resumes the one and asks the other to fetch), +# list the GitRepository's consumers (Kustomizations, and HelmReleases with the HelmCharts built +# from it; without it the source is never prefetched), and create SelfSubjectAccessReviews so the operator can check it holds that +# access before relying on it. Separate from the manager role because it is optional and # names another project's API: an install without Flux grants nothing on it. See # docs/design/apply-trigger.md. apiVersion: rbac.authorization.k8s.io/v1 @@ -18,7 +19,14 @@ rules: - kustomizations verbs: - get + - list - patch +- apiGroups: + - helm.toolkit.fluxcd.io + resources: + - helmreleases + verbs: + - list - apiGroups: - source.toolkit.fluxcd.io resources: @@ -26,6 +34,12 @@ rules: verbs: - get - patch +- apiGroups: + - source.toolkit.fluxcd.io + resources: + - helmcharts + verbs: + - list - apiGroups: - authorization.k8s.io resources: @@ -54,7 +68,7 @@ subjects: # Application and update it with a sync operation (how Argo CD's own automated sync starts one), and # create SelfSubjectAccessReviews so the operator can check it holds that access before relying on # it. Separate from the manager role because it is optional and names another project's API: an -# install without Argo CD grants nothing on it. See docs/argocd.md. +# install without Argo CD grants nothing on it. See docs/bi-directional-argocd.md. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/charts/gitops-reverser/templates/clusterprovider-default.yaml b/charts/gitops-reverser/templates/clusterprovider-default.yaml index 3ded63821..ea36b1b09 100644 --- a/charts/gitops-reverser/templates/clusterprovider-default.yaml +++ b/charts/gitops-reverser/templates/clusterprovider-default.yaml @@ -25,4 +25,7 @@ spec: {{- end }} accessFrom: {{- toYaml .Values.clusterProvider.default.accessFrom | nindent 4 }} + allowCrossNamespace: + sources: {{ .Values.clusterProvider.default.allowCrossNamespace.sources }} + appliers: {{ .Values.clusterProvider.default.allowCrossNamespace.appliers }} {{- end }} diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index cf3fd7b0a..1b5558b04 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -109,8 +109,8 @@ spec: - --branch-buffer-max-size={{ .Values.controllerManager.branchBufferMaxSize }} - --git-refresh-interval={{ .Values.git.refreshInterval }} - --commit-request-ttl={{ .Values.controllerManager.commitRequestTTL }} - {{- if include "gitops-reverser.receiverEnabled" . }} - - {{ printf "--receiver-bind-address=%s" (.Values.receiver.bindAddress | toString | trim) | quote }} + {{- if .Values.servers.receiver.enabled }} + - --receiver-bind-address={{ .Values.servers.receiver.bindAddress }} {{- end }} {{- if .Values.logging.level }} - --zap-log-level={{ .Values.logging.level }} @@ -138,9 +138,9 @@ spec: containerPort: {{ .Values.servers.admission.port }} protocol: TCP {{- end }} - {{- if include "gitops-reverser.receiverEnabled" . }} + {{- if .Values.servers.receiver.enabled }} - name: receiver - containerPort: {{ include "gitops-reverser.receiverPort" . }} + containerPort: {{ .Values.servers.receiver.port }} protocol: TCP {{- end }} env: diff --git a/charts/gitops-reverser/templates/services.yaml b/charts/gitops-reverser/templates/services.yaml index 16a23e1b7..e6c20408c 100644 --- a/charts/gitops-reverser/templates/services.yaml +++ b/charts/gitops-reverser/templates/services.yaml @@ -55,7 +55,7 @@ spec: {{- include "gitops-reverser.selectorLabels" . | nindent 4 }} control-plane: gitops-reverser {{- end }} -{{- if include "gitops-reverser.receiverEnabled" . }} +{{- if .Values.servers.receiver.enabled }} --- # The push receiver: plain HTTP for Git hosts' push webhooks. ClusterIP only; expose it with an # Ingress or Gateway route of your own that terminates TLS, the way Flux's webhook-receiver is. diff --git a/charts/gitops-reverser/values.schema.json b/charts/gitops-reverser/values.schema.json index 3916f782a..979db5587 100644 --- a/charts/gitops-reverser/values.schema.json +++ b/charts/gitops-reverser/values.schema.json @@ -131,20 +131,6 @@ } }, - "receiver": { - "type": "object", - "additionalProperties": false, - "properties": { - "bindAddress": { - "anyOf": [ - { "type": "string", "pattern": "^(|0|[^\\s]*:[1-9][0-9]{0,4})$" }, - { "const": 0 } - ], - "description": "host:port the push receiver listens on, over plain HTTP (--receiver-bind-address). Empty or \"0\" serves no receiver. When set, the chart also renders a -receiver Service; expose it with an Ingress or Gateway route that terminates TLS." - } - } - }, - "applierAccess": { "type": "object", "additionalProperties": false, @@ -152,7 +138,7 @@ "properties": { "flux": { "type": "boolean", - "description": "Grant get/patch on Flux Kustomizations and GitRepositories, plus create on SelfSubjectAccessReviews." + "description": "Grant get/patch on Flux Kustomizations and GitRepositories, list on Kustomizations, HelmReleases and HelmCharts, plus create on SelfSubjectAccessReviews." }, "argocd": { "type": "boolean", @@ -186,6 +172,22 @@ "type": "boolean", "description": "Serve HTTP/2 on the TLS servers. Off by default: it mitigates the HTTP/2 Rapid-Reset CVE class." }, + "receiver": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Serve the push receiver, over plain HTTP, for Git hosts' push webhooks to GitProvider.status.webhookPath, and render a -receiver Service; expose it with an Ingress or Gateway route that terminates TLS." + }, + "bindAddress": { + "type": "string", + "pattern": "^[^\\s]*:[1-9][0-9]{0,4}$", + "description": "host:port the push receiver binds to (--receiver-bind-address)." + }, + "port": { "$ref": "#/$defs/port" } + } + }, "healthProbe": { "type": "object", "additionalProperties": false, @@ -403,6 +405,21 @@ "type": "object", "description": "Deny-by-default control-cluster namespace policy (names and/or selector) for which namespaces may reference this provider from a GitTarget.", "additionalProperties": true + }, + "allowCrossNamespace": { + "type": "object", + "additionalProperties": false, + "description": "What an admitted namespace may reach outside itself through this provider.", + "properties": { + "sources": { + "type": "boolean", + "description": "A WatchRule may watch other namespaces, \"*\" included." + }, + "appliers": { + "type": "boolean", + "description": "A FluxTrigger or ArgoCDTrigger may name a Kustomization or Application in another namespace." + } + } } } } diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 38e3568ae..3f04cadff 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -112,27 +112,15 @@ git: # "every tick". gitopsreverser_git_fetches_total{reason="refresh"} is what it costs. refreshInterval: 10m -# The push receiver: the endpoint a Git host's push webhook points at, so GitOps Reverser learns in -# seconds, instead of at the next refresh, that somebody else pushed. A GitProvider opts in with -# spec.receiver and publishes its path as status.webhookPath (/git-push/). -receiver: - # host:port the receiver listens on (passed as --receiver-bind-address), e.g. ":9292". Empty (the - # default) serves no receiver: a foreign push is then still found, by the next rejected push or by - # git.refreshInterval. When set, the chart adds a containerPort and a -receiver Service - # (port service.ports.receiver). The listener is plain HTTP, because a Git host cannot present the - # client certificate the audit listener requires: expose the Service to your Git host with an - # Ingress or Gateway route of your own that terminates TLS, the way Flux's webhook-receiver is - # exposed. Every request must still carry a valid signature for the route it names. - bindAddress: "" -# Optional RBAC that lets GitOps Reverser ask an applier in this cluster to apply a commit, after -# somebody else pushed to a branch a GitTarget writes. Off unless asked for, since most installs run -# one applier or none. +# Optional RBAC that lets GitOps Reverser hold the applier an apply trigger names in this cluster: +# a FluxTrigger's Kustomization, suspended and resumed for each apply, or an ArgoCDTrigger's +# Application, synced for each apply. Off unless asked for, since one-way mirroring needs neither. applierAccess: # Grant get/patch on kustomize.toolkit.fluxcd.io Kustomizations and source.toolkit.fluxcd.io - # GitRepositories, plus create on SelfSubjectAccessReviews (to check that access before relying - # on it). Cluster-wide, because a GitTarget's applier can live in any - # namespace. + # GitRepositories, list on Kustomizations, HelmReleases and HelmCharts (to find a + # GitRepository's other consumers), plus create on SelfSubjectAccessReviews (to check that access + # before relying on it). Cluster-wide, because a trigger's applier can live in any namespace. flux: false # Grant get/update on argoproj.io Applications (a sync is started by updating the Application's # operation, as Argo CD's own automated sync does), plus create on SelfSubjectAccessReviews. @@ -248,6 +236,22 @@ servers: # Leave empty to use the chart-generated Secret name. secretNameOverride: "" + # The push receiver: the endpoint a Git host's push webhook points at, so GitOps Reverser learns in + # seconds, instead of at the next refresh, that somebody else pushed. A GitProvider opts in with + # spec.receiver and publishes its path as status.webhookPath (/git-push/). + # Off by default: a foreign push is then still found, by the next rejected push or by + # git.refreshInterval. Enabled, the chart adds a containerPort and a -receiver Service + # (port service.ports.receiver). The listener is plain HTTP, because a Git host cannot present the + # client certificate the audit listener requires: expose the Service to your Git host with an + # Ingress or Gateway route of your own that terminates TLS, the way Flux's webhook-receiver is + # exposed. Every request must still carry a valid signature for the route it names. + receiver: + enabled: false + # host:port the receiver binds to (passed as --receiver-bind-address). + bindAddress: ":9292" + # Container/Service target port; must match the port in bindAddress above. + port: 9292 + # Redis/Valkey holds each GitTarget's watch resume cursors (warm restart) and, when attribution is # enabled, the audit attribution facts. Empty by default: the chart runs in configured-author mode with # no Redis (watches cold-replay on restart). Set addr to enable warm restarts; required when @@ -355,12 +359,20 @@ clusterProvider: # out of the box; tighten it to a names list or a label selector in a real install. accessFrom: selector: {} + # What a namespace accessFrom admits may reach OUTSIDE itself through this provider's identity. + # Each is a power of its own: sources lets a WatchRule read other namespaces' objects into its + # own Git ("*" included); appliers lets a FluxTrigger or ArgoCDTrigger suspend and apply a + # Kustomization or Application in another namespace. It is a delegation to every admitted + # namespace, not proof that a namespace owns what it reaches. + allowCrossNamespace: + sources: false + appliers: false # RBAC configuration rbac: create: true # Which types the reverser may read. It never writes to a watched type, so verbs are always - # get/list/watch. Shape is enforced by values.schema.json; see docs/rbac.md. + # get/list/watch (applierAccess, above, is a separate grant). Shape is enforced by values.schema.json; see docs/rbac.md. # # any cluster-wide read on everything, so a WatchRule can name any type — including # one installed later. It also means the reverser can list and read every Secret @@ -433,8 +445,8 @@ service: ports: audit: 9444 metrics: 8080 - # Port of the -receiver Service, rendered only when receiver.bindAddress is set. It - # forwards to the port in receiver.bindAddress. + # Port of the -receiver Service, rendered only when servers.receiver.enabled. Port 80, + # like Flux's webhook-receiver Service, since an Ingress or Gateway route sits in front of it. receiver: 80 auditService: diff --git a/cmd/main.go b/cmd/main.go index 4b4a5bf6e..bc8d9512d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -182,12 +182,18 @@ func main() { // and a folder somebody else broke in Git produces no Kubernetes event to notice it by. workerManager.SetScanAcceptanceReporter(watchMgr.ReportGitPathScan) // The branch workers decide when a GitTarget's applier may apply; the dispatcher delivers it, - // off their queues, to the Flux objects in each target's source cluster. + // off their queues, to the applier its apply trigger holds, in the trigger's cluster. The status + // writer records each operation on the trigger, also off their queues. applyDispatcher := applier.NewDispatcher( mgr.GetClient(), watchMgr.ApplierDynamicClient, mgr.GetEventRecorderFor("gittarget"), ctrl.Log.WithName("applier")) + applyDispatcher.APIReader = mgr.GetAPIReader() fatalIfErr(mgr.Add(applyDispatcher), "unable to add apply request dispatcher to manager") workerManager.SetApplyRequester(applyDispatcher) + triggerStatusWriter := applier.NewStatusWriter(mgr.GetAPIReader(), mgr.GetClient(), + ctrl.Log.WithName("applier").WithName("status")) + fatalIfErr(mgr.Add(triggerStatusWriter), "unable to add apply trigger status writer to manager") + workerManager.SetApplyTriggerStatusWriter(triggerStatusWriter) // WatchRule controller (with WatchManager reference for dynamic reconciliation) fatalIfErr((&controller.WatchRuleReconciler{ @@ -354,12 +360,20 @@ func main() { Recorder: mgr.GetEventRecorderFor("gittarget"), GitRefreshInterval: cfg.gitRefreshInterval, - ApplierClients: watchMgr.ApplierDynamicClient, - ApplyRequests: applyDispatcher, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GitTarget") os.Exit(1) } + for _, kind := range []string{configbutleraiv1alpha3.ApplyTriggerKindFlux, configbutleraiv1alpha3.ApplyTriggerKindArgoCD} { + fatalIfErr((&controller.ApplyTriggerReconciler{ + Client: mgr.GetClient(), + Kind: kind, + Recorder: mgr.GetEventRecorderFor(strings.ToLower(kind)), + ApplierClients: watchMgr.ApplierDynamicClient, + APIReader: mgr.GetAPIReader(), + Follower: applyDispatcher, + }).SetupWithManager(mgr), "unable to create controller", "controller", kind) + } // Command authorship is captured at admission by the validate-operator-types webhook and // lives in its own Redis corner (author:v1:command), independent of // --author-attribution (which governs mirrored-resource attribution). It is wired diff --git a/cmd/main_chart_receiver_test.go b/cmd/main_chart_receiver_test.go index 9fc4d2514..95c7b841b 100644 --- a/cmd/main_chart_receiver_test.go +++ b/cmd/main_chart_receiver_test.go @@ -95,12 +95,12 @@ func containerPort(c corev1.Container, name string) (int32, bool) { } // The receiver is off by default and the chart must say nothing about it: no flag (the binary's -// own default is off), no port, no Service. +// own default is off), no port, no Service. A bind address alone does not switch it on. func TestChartReceiver_OffByDefault(t *testing.T) { for name, setValues := range map[string][]string{ - "default": nil, - "explicit zero": {"receiver.bindAddress=0"}, - "explicit empty": {"receiver.bindAddress="}, + "default": nil, + "explicit off": {"servers.receiver.enabled=false"}, + "bind address but off": {"servers.receiver.bindAddress=0.0.0.0:9393"}, } { t.Run(name, func(t *testing.T) { manager := renderManager(t, setValues...) @@ -118,18 +118,22 @@ func TestChartReceiver_OffByDefault(t *testing.T) { } } -// Set, the address reaches the binary unquoted, and the containerPort and the Service's target -// are the port the binary binds. +// Enabled, the address reaches the binary unquoted, the containerPort is servers.receiver.port, and +// the Service targets it by name, as the other servers are wired. func TestChartReceiver_BindAddressReachesTheBinaryAndTheService(t *testing.T) { for name, tc := range map[string]struct { + setValues []string bindAddress string wantPort int32 }{ - "port only": {":9292", 9292}, - "host and port": {"0.0.0.0:9393", 9393}, + "defaults": {nil, ":9292", 9292}, + "host and port": { + []string{"servers.receiver.bindAddress=0.0.0.0:9393", "servers.receiver.port=9393"}, + "0.0.0.0:9393", 9393, + }, } { t.Run(name, func(t *testing.T) { - setValues := []string{"receiver.bindAddress=" + tc.bindAddress} + setValues := append([]string{"servers.receiver.enabled=true"}, tc.setValues...) manager := renderManager(t, setValues...) cfg, err := parseFlagsWithArgs(flag.NewFlagSet("manager", flag.ContinueOnError), manager.Args) @@ -153,13 +157,15 @@ func TestChartReceiver_BindAddressReachesTheBinaryAndTheService(t *testing.T) { func TestChartReceiver_MalformedBindAddressFailsTheRender(t *testing.T) { out, err := exec.Command("helm", "template", "gitops-reverser", "../charts/gitops-reverser", - "--show-only", "templates/deployment.yaml", "--set", "receiver.bindAddress=localhost").CombinedOutput() + "--show-only", "templates/deployment.yaml", "--set", "servers.receiver.enabled=true", + "--set", "servers.receiver.bindAddress=localhost").CombinedOutput() require.Error(t, err) - assert.Contains(t, string(out), "receiver") + assert.Contains(t, string(out), "/servers/receiver/bindAddress") } -// applierAccess.flux is off by default, and on it grants exactly what an apply request to Flux -// needs, to the operator's own ServiceAccount. +// applierAccess.flux is off by default, and on it grants exactly what holding a Flux Kustomization +// needs (suspend and resume it, fetch its source, list the source's consumers), to the operator's own +// ServiceAccount. func TestChartApplierAccess_Flux(t *testing.T) { assert.Empty(t, helmTemplateDocs(t, "templates/applier-rbac.yaml"), "off unless asked for") assert.Empty(t, helmTemplateDocs(t, "templates/applier-rbac.yaml", @@ -175,13 +181,23 @@ func TestChartApplierAccess_Flux(t *testing.T) { { APIGroups: []string{"kustomize.toolkit.fluxcd.io"}, Resources: []string{"kustomizations"}, - Verbs: []string{"get", "patch"}, + Verbs: []string{"get", "list", "patch"}, + }, + { + APIGroups: []string{"helm.toolkit.fluxcd.io"}, + Resources: []string{"helmreleases"}, + Verbs: []string{"list"}, }, { APIGroups: []string{"source.toolkit.fluxcd.io"}, Resources: []string{"gitrepositories"}, Verbs: []string{"get", "patch"}, }, + { + APIGroups: []string{"source.toolkit.fluxcd.io"}, + Resources: []string{"helmcharts"}, + Verbs: []string{"list"}, + }, { APIGroups: []string{"authorization.k8s.io"}, Resources: []string{"selfsubjectaccessreviews"}, diff --git a/cmd/receiver.go b/cmd/receiver.go index e24ab2fb1..0b1f95dbe 100644 --- a/cmd/receiver.go +++ b/cmd/receiver.go @@ -30,7 +30,7 @@ const ( // The receiver's server timeouts. A delivery is one small POST answered without waiting on the // remote (the branch worker reads the remote later, on its own loop), so these only have to - // outlast a slow client sending a body up to the 25 MiB cap, and keep an idle or trickling + // outlast a slow client sending a body up to the 2 MiB cap, and keep an idle or trickling // connection from holding a socket. They are not flags: nothing a Git host does needs them // tuned, and every knob is one more thing to get wrong. receiverReadHeaderTimeout = 10 * time.Second diff --git a/config/clusterprovider-default.yaml b/config/clusterprovider-default.yaml index 1a4f63caa..f1fc774ba 100644 --- a/config/clusterprovider-default.yaml +++ b/config/clusterprovider-default.yaml @@ -11,3 +11,7 @@ spec: # no kubeConfig => the operator's own in-cluster config (optional for any provider name) accessFrom: selector: {} + # Nothing outside the admitted namespaces, as the chart renders it. + allowCrossNamespace: + sources: false + appliers: false diff --git a/config/crd/bases/configbutler.ai_argocdtriggers.yaml b/config/crd/bases/configbutler.ai_argocdtriggers.yaml new file mode 100644 index 000000000..ebd9260bb --- /dev/null +++ b/config/crd/bases/configbutler.ai_argocdtriggers.yaml @@ -0,0 +1,300 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: argocdtriggers.configbutler.ai +spec: + group: configbutler.ai + names: + kind: ArgoCDTrigger + listKind: ArgoCDTriggerList + plural: argocdtriggers + singular: argocdtrigger + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + - jsonPath: .spec.applicationRef.name + name: Application + type: string + - jsonPath: .status.operation.phase + name: Operation + type: string + - jsonPath: .status.operation.commit + name: Commit + priority: 1 + type: string + - jsonPath: .spec.clusterProviderRef.name + name: Cluster + priority: 1 + type: string + - jsonPath: .status.repairPending + name: RepairPending + priority: 1 + type: boolean + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Message + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + ArgoCDTrigger holds an Argo CD Application, with automated sync off, for GitOps Reverser, which + starts every sync it runs: only on a branch with nothing live missing from the commit, after a + commit GitOps Reverser did not push, after a refused edit, and after a reverted edit. GitTargets + name it with spec.applyTriggerRef. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the Application the trigger holds + properties: + applicationRef: + description: |- + ApplicationRef is the Argo CD Application that applies the folders of the GitTargets naming + this trigger. It must have a single spec.source whose targetRevision names a branch, and + automated sync off: either no spec.syncPolicy.automated, or automated.enabled: false, which + keeps automated.prune for the syncs GitOps Reverser starts. GitOps Reverser starts each sync + by setting the Application's operation, as Argo CD's automated sync does, pinned to the commit + it checked, and never replaces an operation that is running. Its namespace defaults to the + trigger's namespace. Immutable. + properties: + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, when not specified it + acts as LocalObjectReference. + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: spec.applicationRef is immutable; delete and recreate the + ArgoCDTrigger to hold another Application + rule: self == oldSelf + - message: spec.applicationRef.name must not be empty + rule: self.name != '' + clusterProviderRef: + default: + name: default + description: |- + ClusterProviderRef names the ClusterProvider whose cluster holds the Application, such as a + management cluster running Argo CD for the source cluster. A provider without + spec.kubeConfig is the operator's own cluster. The provider must admit this trigger's + namespace (spec.accessFrom). Its identity needs get and update on the Application. Omitted, + it is the ClusterProvider named "default", as for a GitTarget. Immutable. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: spec.clusterProviderRef.name must not be empty + rule: self.name != '' + - message: spec.clusterProviderRef is immutable; delete and recreate + the ArgoCDTrigger to hold an Application elsewhere + rule: self == oldSelf + required: + - applicationRef + type: object + status: + description: status defines the observed state of ArgoCDTrigger + properties: + conditions: + description: |- + Conditions are Configured (an eligible GitTarget names the trigger; the Application exists, + syncs a single source, has automated sync off, may be updated, and no other ArgoCDTrigger + holds it) and the kstatus Ready/Reconciling/Stalled trio. A trigger acts only while an + eligible GitTarget references it: one in its namespace that names it, is admitted by its + ClusterProviders, and writes the repository and branch the Application syncs. A trigger no + GitTarget names reads nothing in the Application's cluster and is Configured=Unknown, + Ready=True, both NotReferenced; one named only by GitTargets that are not eligible is + Configured NotBound or a refusal. Of the ArgoCDTriggers in line for one Application in one cluster, the one + created first holds it; the others are Configured=False (HeldByAnotherTrigger) until it is + deleted or no longer referenced, and the next in line takes it over. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + operation: + description: |- + Operation is the sync GitOps Reverser started on the Application: the one in flight, or the + last one. Absent until the first. + properties: + appliedRevision: + description: |- + AppliedRevision is the revision the applier reports it applied once the operation completed. + It can differ from commit when the branch moved between the check and the apply. + type: string + commit: + description: |- + Commit is the commit the applier was asked to apply. Empty when the remote did not carry the + branch. + type: string + finishedAt: + description: FinishedAt is when the operation was seen to complete. + format: date-time + type: string + message: + description: Message says why a Failed operation failed, or why + an Unresolved one is overdue. + type: string + phase: + description: |- + Phase is where the operation stands. Requested, Running, Settling and Unresolved are in + flight: while one is, an edit to an object the applier applies can still be reverted by it, + and the applier is asked nothing else. Succeeded and Failed have completed. + enum: + - Requested + - Running + - Settling + - Unresolved + - Succeeded + - Failed + type: string + reason: + description: |- + Reason is why the apply was asked for. With several GitTargets on the trigger, it is the + strongest of their reasons. + enum: + - ForeignCommit + - Refusal + - Repair + - CatchUp + type: string + requestedAt: + description: RequestedAt is when the applier was asked. + format: date-time + type: string + suspendToken: + description: |- + SuspendToken is the reconcile.fluxcd.io/requestedAt value the suspend after the apply carried. + Once the Kustomization reports it handled, nothing started before the suspend is still + running. Flux only. + type: string + token: + description: |- + Token identifies the request in the applier's own status: the reconcile.fluxcd.io/requestedAt + value on a Flux object, or the "gitops-reverser request" entry in an Argo CD operation's info. + type: string + required: + - phase + - requestedAt + type: object + repairPending: + description: |- + RepairPending is true while a Repair sync is pending for the Application: a sync reverted an + API edit made after it was asked for, so the cluster runs an older value than Git until the + next sync. + type: boolean + source: + description: |- + Source is the Application's spec.source, as last read, published while an eligible GitTarget + references the trigger. A GitTarget naming this trigger must write the repository it syncs, on + the branch its targetRevision names. + properties: + repoURL: + description: RepoURL is the Application's spec.source.repoURL. + type: string + targetRevision: + description: |- + TargetRevision is the Application's spec.source.targetRevision, as written: a branch name, or + refs/heads/ and the name. Empty is Argo CD's HEAD. + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/configbutler.ai_clusterproviders.yaml b/config/crd/bases/configbutler.ai_clusterproviders.yaml index 6cfd73de7..676dd1272 100644 --- a/config/crd/bases/configbutler.ai_clusterproviders.yaml +++ b/config/crd/bases/configbutler.ai_clusterproviders.yaml @@ -141,20 +141,31 @@ spec: x-kubernetes-map-type: atomic type: object allowAnySourceNamespace: - default: false description: |- - AllowAnySourceNamespace delegates SOURCE-namespace selection to the GitTargets this provider - admits. While false (the default) a WatchRule mirroring through this provider may watch only - its OWN namespace. - - It grants no access by itself: the source credential's own RBAC remains the hard maximum, and - a request it permits still fails 403 if the credential cannot read that namespace. What it - delegates is the AUTHORITY to choose, so set it only when the owners of admitted GitTargets - are trusted to pick a subset of what that credential may read. Every cross-namespace request - needs it, including a rules[].sourceNamespace of "*", which reaches every namespace the - credential can read. It does not apply to ClusterWatchRule, which selects no namespaces at - all. + RemovedAllowAnySourceNamespace is the removed spelling of allowCrossNamespace.sources. It is + never read; setting it to true is refused. type: boolean + allowCrossNamespace: + description: |- + AllowCrossNamespace grants what a namespace this provider admits may reach outside itself, + through this provider's credential. Omitted, it may reach nothing outside itself. + properties: + appliers: + description: |- + Appliers lets a FluxTrigger or ArgoCDTrigger reaching its applier through this provider name a + Kustomization or Application in another namespace than its own. False (the default): only + the trigger's own namespace, which in a remote cluster is the namespace of the same name; a + trigger naming another is Configured=False (ApplierNamespaceNotAllowed). + type: boolean + sources: + description: |- + Sources lets a WatchRule mirroring through this provider watch namespaces other than its own, + including a rules[].sourceNamespace of "*", which reaches every namespace the credential can + read. False (the default): a WatchRule watches only its own namespace. It grants no access by + itself: the source credential's RBAC remains the hard maximum. It does not apply to + ClusterWatchRule, which selects no namespaces at all. + type: boolean + type: object attribution: description: |- Attribution groups this cluster's author-attribution settings. The block is spelled @@ -305,6 +316,8 @@ spec: > 0' - message: spec.qps and spec.burst moved to spec.client.qps and spec.client.burst rule: '!has(self.qps) && !has(self.burst)' + - message: allowAnySourceNamespace was removed; set allowCrossNamespace.sources + rule: '!has(self.allowAnySourceNamespace) || !self.allowAnySourceNamespace' status: description: status defines the observed state of ClusterProvider. properties: diff --git a/config/crd/bases/configbutler.ai_fluxtriggers.yaml b/config/crd/bases/configbutler.ai_fluxtriggers.yaml new file mode 100644 index 000000000..e3a996a2d --- /dev/null +++ b/config/crd/bases/configbutler.ai_fluxtriggers.yaml @@ -0,0 +1,365 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: fluxtriggers.configbutler.ai +spec: + group: configbutler.ai + names: + kind: FluxTrigger + listKind: FluxTriggerList + plural: fluxtriggers + singular: fluxtrigger + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=="Held")].reason + name: Held + type: string + - jsonPath: .spec.kustomizationRef.name + name: Kustomization + type: string + - jsonPath: .status.operation.phase + name: Operation + type: string + - jsonPath: .status.operation.commit + name: Commit + priority: 1 + type: string + - jsonPath: .spec.clusterProviderRef.name + name: Cluster + priority: 1 + type: string + - jsonPath: .status.repairPending + name: RepairPending + priority: 1 + type: boolean + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Message + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + description: |- + FluxTrigger holds a Flux Kustomization for GitOps Reverser, which starts every apply it makes: + only on a branch with nothing live missing from the commit, after a commit GitOps Reverser did + not push, after a refused edit, after a reverted edit, and to catch up with its own pushes. + GitTargets name it with spec.applyTriggerRef. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the Kustomization the trigger holds + properties: + clusterProviderRef: + default: + name: default + description: |- + ClusterProviderRef names the ClusterProvider whose cluster holds the Kustomization, such as a + management cluster running Flux for the source cluster. A provider without spec.kubeConfig is + the operator's own cluster. The provider must admit this trigger's namespace + (spec.accessFrom). Its identity needs get and patch on the Kustomization and its + GitRepository. Omitted, + it is the ClusterProvider named "default", as for a GitTarget. Immutable. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: spec.clusterProviderRef.name must not be empty + rule: self.name != '' + - message: spec.clusterProviderRef is immutable; delete and recreate + the FluxTrigger to hold a Kustomization elsewhere + rule: self == oldSelf + kustomizationRef: + description: |- + KustomizationRef is the Flux Kustomization that applies the folders of the GitTargets naming + this trigger. Its spec.sourceRef must be a GitRepository that follows a branch. Its namespace + defaults to the trigger's namespace. Immutable. + properties: + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, when not specified it + acts as LocalObjectReference. + type: string + required: + - name + type: object + x-kubernetes-validations: + - message: spec.kustomizationRef is immutable; delete and recreate + the FluxTrigger to hold another Kustomization + rule: self == oldSelf + - message: spec.kustomizationRef.name must not be empty + rule: self.name != '' + quietPeriod: + default: 30s + description: |- + QuietPeriod is how long the branch must hold no open commit window and no pending writes, + after GitOps Reverser's own pushes, before the Kustomization is asked to catch up with them. + Every new write restarts the wait. A Go duration of at most "1h". + pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$ + type: string + x-kubernetes-validations: + - message: spec.quietPeriod must be a Go duration of at most 1h + rule: duration(self) <= duration('1h') + required: + - kustomizationRef + type: object + status: + description: status defines the observed state of FluxTrigger + properties: + conditions: + description: |- + Conditions are Configured (an eligible GitTarget names the trigger; the Kustomization exists, + applies a GitRepository that follows a branch, may be patched, and no other FluxTrigger holds + it), Held (Suspended: suspended under GitOps Reverser with the last suspend handled; Applying: + resumed for an apply GitOps Reverser started; Suspending: a suspend is not handled yet; + Releasing: being handed back; NotReferenced; NotConfigured), and the kstatus + Ready/Reconciling/Stalled trio. A trigger acts only while an eligible GitTarget references it: + one in its namespace that names it, is admitted by its ClusterProviders, and writes the + repository and branch the GitRepository fetches. A trigger no GitTarget names reads nothing + in the Kustomization's cluster and is Configured=Unknown, Ready=True, both NotReferenced; one + named only by GitTargets that are not eligible is Configured NotBound or a refusal. Of the + FluxTriggers in line for one Kustomization in one cluster, one holds it: the one with an apply + in flight, then the one holding it already, then the one created first. The others are + Configured=False (HeldByAnotherTrigger) until it lets go, and the next in line takes it over. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + operation: + description: |- + Operation is the apply GitOps Reverser started on the Kustomization: the one in flight, or + the last one. Absent until the first. + properties: + appliedRevision: + description: |- + AppliedRevision is the revision the applier reports it applied once the operation completed. + It can differ from commit when the branch moved between the check and the apply. + type: string + commit: + description: |- + Commit is the commit the applier was asked to apply. Empty when the remote did not carry the + branch. + type: string + finishedAt: + description: FinishedAt is when the operation was seen to complete. + format: date-time + type: string + message: + description: Message says why a Failed operation failed, or why + an Unresolved one is overdue. + type: string + phase: + description: |- + Phase is where the operation stands. Requested, Running, Settling and Unresolved are in + flight: while one is, an edit to an object the applier applies can still be reverted by it, + and the applier is asked nothing else. Succeeded and Failed have completed. + enum: + - Requested + - Running + - Settling + - Unresolved + - Succeeded + - Failed + type: string + reason: + description: |- + Reason is why the apply was asked for. With several GitTargets on the trigger, it is the + strongest of their reasons. + enum: + - ForeignCommit + - Refusal + - Repair + - CatchUp + type: string + requestedAt: + description: RequestedAt is when the applier was asked. + format: date-time + type: string + suspendToken: + description: |- + SuspendToken is the reconcile.fluxcd.io/requestedAt value the suspend after the apply carried. + Once the Kustomization reports it handled, nothing started before the suspend is still + running. Flux only. + type: string + token: + description: |- + Token identifies the request in the applier's own status: the reconcile.fluxcd.io/requestedAt + value on a Flux object, or the "gitops-reverser request" entry in an Argo CD operation's info. + type: string + required: + - phase + - requestedAt + type: object + repairPending: + description: |- + RepairPending is true while a Repair apply is pending for the Kustomization: an apply reverted + an API edit made after it was asked for, so the cluster runs an older value than Git until the + next apply. + type: boolean + source: + description: |- + Source is the GitRepository the Kustomization applies, as last read, published while an + eligible GitTarget references the trigger. A GitTarget naming this trigger must write the + repository and branch it fetches. + properties: + branch: + description: Branch is the GitRepository's spec.ref.branch. + type: string + gitRepository: + description: GitRepository is the Kustomization's spec.sourceRef, + its namespace defaulted. + properties: + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, when not specified + it acts as LocalObjectReference. + type: string + required: + - name + type: object + otherConsumers: + description: |- + OtherConsumers are the other objects in the cluster that apply this GitRepository and that no + FluxTrigger holds: Kustomizations, and HelmReleases whose chart comes from it. Each applies a + new artifact the moment it arrives. Information only: it never affects Ready. + items: + description: |- + NamespacedObjectKindReference contains enough information to locate the typed referenced Kubernetes resource object + in any namespace. + properties: + apiVersion: + description: API version of the referent, if not specified + the Kubernetes preferred version will be used. + type: string + kind: + description: Kind of the referent. + type: string + name: + description: Name of the referent. + type: string + namespace: + description: Namespace of the referent, when not specified + it acts as LocalObjectReference. + type: string + required: + - kind + - name + type: object + type: array + x-kubernetes-list-type: atomic + prefetch: + description: |- + Prefetch is true while GitOps Reverser fetches the GitRepository as soon as it learns of a push, + before the apply: every consumer of it was listed and is held by a FluxTrigger, so a new + artifact applies nothing on its own. False when otherConsumers is not empty, or the consumers + could not be listed (the identity needs list on kustomizations, helmreleases and helmcharts). + type: boolean + url: + description: URL is the GitRepository's spec.url. + type: string + required: + - gitRepository + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/configbutler.ai_gittargets.yaml b/config/crd/bases/configbutler.ai_gittargets.yaml index 6233c1610..65be93b56 100644 --- a/config/crd/bases/configbutler.ai_gittargets.yaml +++ b/config/crd/bases/configbutler.ai_gittargets.yaml @@ -52,8 +52,8 @@ spec: name: RenderMatchesLive priority: 1 type: string - - jsonPath: .status.conditions[?(@.type=="ApplierConfigured")].status - name: ApplierConfigured + - jsonPath: .status.conditions[?(@.type=="ApplyTriggerReady")].status + name: ApplyTriggerReady priority: 1 type: string - jsonPath: .status.conditions[?(@.type=="StreamsRunning")].status @@ -112,99 +112,32 @@ spec: spec: description: spec defines the desired state of GitTarget properties: - applier: + applyTriggerRef: description: |- - Applier names the object that applies this folder back to its source cluster. With it set, - GitOps Reverser decides when that object applies: it asks for an apply only when the branch - holds no open commit window and no pending writes, after a commit it did not push, after a - refused edit (spec.onRefusal: Reapply), and, for a Kustomization, once the branch has been - quiet for quietPeriod, to keep it current with its own pushes. Switch off the applier's own - triggers (short Flux intervals, Git-host webhooks into Flux, Argo CD automated sync), or they - go on applying without that check. The reference is looked up in the cluster - spec.applier.clusterProviderRef names, by default the source cluster, with the identity that - ClusterProvider uses. + ApplyTriggerRef names the apply trigger, in this GitTarget's namespace, that holds the applier + of this folder: a FluxTrigger (a Flux Kustomization) or an ArgoCDTrigger (an Argo CD + Application). With it set, GitOps Reverser decides when that applier applies: only when the + branch holds no open commit window and no pending writes, after a commit it did not push, + after a refused edit (spec.onRefusal: Reapply), after an apply reverted a newer API edit, and, + for a Kustomization, once the branch has been quiet for the trigger's quietPeriod, to keep it + current with its own pushes. The trigger's applier must apply this target's repository and + branch; the ApplyTriggerReady condition says whether it does. Several GitTargets may name one + trigger, and it is asked once for all of them. properties: - applicationRef: - description: |- - ApplicationRef is an Argo CD Application whose spec.source is this target's repository, with - targetRevision this target's branch (the branch name, or refs/heads/ and the name). Its - namespace defaults to the GitTarget's namespace. Automated sync must be off: the Application - either has no spec.syncPolicy.automated, or sets automated.enabled: false, which keeps - automated.prune for the syncs GitOps Reverser starts. One with automated sync on is - ApplierConfigured=False (AutomatedSyncEnabled) and is never synced. GitOps Reverser starts - each sync by setting the Application's operation, as Argo CD's automated sync does, pinned to - the commit it checked, and never replaces an operation that is running. A multi-source - Application is not supported. Exactly one of kustomizationRef and applicationRef is set. - properties: - name: - description: Name of the referent. - type: string - namespace: - description: Namespace of the referent, when not specified - it acts as LocalObjectReference. - type: string - required: - - name - type: object - clusterProviderRef: - description: |- - ClusterProviderRef names the ClusterProvider whose cluster holds the applier, such as a - management cluster running Flux or Argo CD for the source cluster. Omitted, it is the - GitTarget's own spec.clusterProviderRef. The provider must admit this GitTarget's namespace - (spec.accessFrom). Its identity needs get and patch on a Kustomization and its GitRepository, - or get and update on an Application. A provider without spec.kubeConfig is the operator's own - cluster. - properties: - name: - description: Name of the referent. - type: string - required: - - name - type: object - x-kubernetes-validations: - - message: spec.applier.clusterProviderRef.name must not be empty - rule: self.name != '' - kustomizationRef: - description: |- - KustomizationRef is a Flux Kustomization whose spec.sourceRef is a GitRepository on this - target's repository and branch. Its namespace defaults to the GitTarget's namespace. GitOps - Reverser asks it to apply the way `flux reconcile kustomization --with-source` does: it - stamps reconcile.fluxcd.io/requestedAt on the GitRepository, and on the Kustomization too - when the GitRepository's artifact already holds the commit. Exactly one of kustomizationRef - and applicationRef is set. - properties: - name: - description: Name of the referent. - type: string - namespace: - description: Namespace of the referent, when not specified - it acts as LocalObjectReference. - type: string - required: - - name - type: object - quietPeriod: - description: |- - QuietPeriod is how long the branch must hold no open commit window and no pending writes, - after GitOps Reverser's own pushes, before a Kustomization is asked to catch up with them. - Flux applies on its own timers and on a controller restart from whatever artifact it holds, - so an artifact left behind those pushes would revert edits that are already in Git. Every new - write restarts the wait. A Go duration of at most "1h". Omitted, it is "30s". Only for a - kustomizationRef: it is rejected with an applicationRef. - pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|μs|ms|s|m|h))+$ + kind: + description: 'Kind is the trigger''s kind: FluxTrigger or ArgoCDTrigger.' + enum: + - FluxTrigger + - ArgoCDTrigger type: string - x-kubernetes-validations: - - message: spec.applier.quietPeriod must be a Go duration of at - most 1h - rule: duration(self) <= duration('1h') + name: + description: Name is the trigger's name. + minLength: 1 + type: string + required: + - kind + - name type: object - x-kubernetes-validations: - - message: spec.applier names exactly one of kustomizationRef and - applicationRef - rule: has(self.kustomizationRef) != has(self.applicationRef) - - message: 'spec.applier.quietPeriod applies only to a kustomizationRef: - Argo CD is never asked to catch up' - rule: '!has(self.applicationRef) || !has(self.quietPeriod)' branch: description: |- Branch to use for this target. @@ -411,12 +344,13 @@ spec: "Ignore", the default, records the refusal in conditions and events and does nothing else. - "Reapply" asks this target's applier (spec.applier, which it requires) to apply the current - tip once the branch holds no open window and no pending writes, which reverts the refused - edit and nothing that is still waiting to be published. Only this target's applier is asked; - if its GitRepository has to fetch to reach the tip, every Kustomization on that GitRepository - applies the new revision, as for any push. Give the folder its own GitRepository to keep the - apply to it alone. + "Reapply" asks this target's applier (through spec.applyTriggerRef, which it requires) to + apply the current tip once the branch holds no open window and no pending writes, which + reverts the refused edit and nothing that is still waiting to be published. Only this + target's applier is asked. If its GitRepository has to fetch to reach the tip, the + Kustomizations a FluxTrigger holds stay suspended and ignore the new artifact; only the + consumers no trigger holds, listed in the trigger's status.source.otherConsumers, apply it, as + they would for any push. It fires only for an edit to a document the folder ALREADY holds, in a write that removes no document anywhere: re-applying would otherwise prune, or put back, an object somebody else is @@ -621,35 +555,12 @@ spec: - message: spec.clusterProviderRef is immutable; delete and recreate the GitTarget to change the cluster it mirrors rule: self.clusterProviderRef == oldSelf.clusterProviderRef - - message: 'spec.onRefusal: Reapply needs spec.applier, the object asked - to reapply' - rule: '!has(self.onRefusal) || self.onRefusal != ''Reapply'' || has(self.applier)' + - message: 'spec.onRefusal: Reapply needs spec.applyTriggerRef, the trigger + asked to reapply' + rule: '!has(self.onRefusal) || self.onRefusal != ''Reapply'' || has(self.applyTriggerRef)' status: description: status defines the observed state of GitTarget properties: - applier: - description: |- - Applier is the last apply request made to spec.applier: the commit, when, and why. Absent - until the first request. - properties: - reason: - description: Reason is why it was asked. - enum: - - ForeignCommit - - Refusal - - CatchUp - type: string - requestedAt: - description: |- - RequestedAt is when the request was made. It says the applier was asked, never that it - applied: read the applier's own status for that. - format: date-time - type: string - requestedCommit: - description: RequestedCommit is the commit the applier was asked - to apply. - type: string - type: object conditions: description: Conditions represent the latest available observations of an object's state diff --git a/config/crd/bases/configbutler.ai_watchrules.yaml b/config/crd/bases/configbutler.ai_watchrules.yaml index af3ffa27d..a469bf49f 100644 --- a/config/crd/bases/configbutler.ai_watchrules.yaml +++ b/config/crd/bases/configbutler.ai_watchrules.yaml @@ -225,7 +225,7 @@ spec: "*" is one cluster-wide list and watch rather than a set of per-namespace ones, so its bound is the credential's own RBAC and nothing else. Naming any namespace other than this rule's own, "*" included, requires the GitTarget's ClusterProvider to admit the target's namespace - AND to set spec.allowAnySourceNamespace; while that flag is false, "*" is refused outright. + AND to set spec.allowCrossNamespace.sources; while that is false, "*" is refused outright. This changes only which namespace is WATCHED, never where objects are written: Git placement follows each mirrored object's own namespace. diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index f78a624b4..802252a22 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -1,9 +1,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - bases/configbutler.ai_argocdtriggers.yaml - bases/configbutler.ai_clusterproviders.yaml - bases/configbutler.ai_clusterwatchrules.yaml - bases/configbutler.ai_commitrequests.yaml + - bases/configbutler.ai_fluxtriggers.yaml - bases/configbutler.ai_gitproviders.yaml - bases/configbutler.ai_gittargets.yaml - bases/configbutler.ai_watchrules.yaml diff --git a/config/rbac/argocd-applier-role.yaml b/config/rbac/argocd-applier-role.yaml index e8d1512c4..91e920cd7 100644 --- a/config/rbac/argocd-applier-role.yaml +++ b/config/rbac/argocd-applier-role.yaml @@ -1,10 +1,10 @@ -# What GitOps Reverser needs to drive an Argo CD Application named in GitTarget.spec.applier: read +# What GitOps Reverser needs to drive an Argo CD Application an ArgoCDTrigger holds: read # the Application, set its operation to a sync (an update of the whole object, as Argo CD's own # SetAppOperation makes it), and check that it may. # # It is kept OUT of role.yaml because no GitTarget needs it until one names an Application, and an # install that does not use Argo CD should not hold update rights on Applications. Drop this pair -# when no GitTarget sets spec.applier.applicationRef (Helm: `applierAccess.argocd: false`, the +# when no ArgoCDTrigger exists (Helm: `applierAccess.argocd: false`, the # default there). For an Application in another cluster, grant the same to the identity of the # ClusterProvider's kubeconfig there. apiVersion: rbac.authorization.k8s.io/v1 diff --git a/config/rbac/flux-applier-role.yaml b/config/rbac/flux-applier-role.yaml index 48051a1d7..08502e1c5 100644 --- a/config/rbac/flux-applier-role.yaml +++ b/config/rbac/flux-applier-role.yaml @@ -1,10 +1,12 @@ -# What GitOps Reverser needs to drive a Flux Kustomization named in GitTarget.spec.applier: read -# the Kustomization and the GitRepository it applies from, stamp reconcile.fluxcd.io/requestedAt on -# either (the request `flux reconcile kustomization --with-source` makes), and check that it may. +# What GitOps Reverser needs to drive a Flux Kustomization a FluxTrigger holds: read the +# Kustomization and the GitRepository it applies from, suspend and resume the Kustomization and +# stamp reconcile.fluxcd.io/requestedAt on either, list the consumers of the GitRepository +# (Kustomizations, and HelmReleases with the HelmCharts built from it; without it the source is +# never prefetched), and check that it may. # # It is kept OUT of role.yaml because no GitTarget needs it until one names an applier, and an # install that does not use Flux should not hold patch rights on Flux objects. Drop this pair when -# no GitTarget sets spec.applier (Helm: `applierAccess.flux: false`, the default there). For a +# no FluxTrigger exists (Helm: `applierAccess.flux: false`, the default there). For a # remote source cluster, grant the same to the identity of the ClusterProvider's kubeconfig there. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -20,7 +22,14 @@ rules: - kustomizations verbs: - get + - list - patch + - apiGroups: + - helm.toolkit.fluxcd.io + resources: + - helmreleases + verbs: + - list - apiGroups: - source.toolkit.fluxcd.io resources: @@ -28,6 +37,12 @@ rules: verbs: - get - patch + - apiGroups: + - source.toolkit.fluxcd.io + resources: + - helmcharts + verbs: + - list - apiGroups: - authorization.k8s.io resources: diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 53a056e9c..daec1484e 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,10 +7,10 @@ resources: # See docs/rbac.md; the Helm chart renders the same pair from `rbac.watchTypes`. - watch-any-role.yaml - watch-any-rolebinding.yaml - # Patch rights on Flux Kustomizations and GitRepositories, for GitTarget.spec.applier. See the file. + # Patch rights on Flux Kustomizations and GitRepositories, for FluxTriggers. See the file. - flux-applier-role.yaml - flux-applier-rolebinding.yaml - # Update rights on Argo CD Applications, for GitTarget.spec.applier.applicationRef. See the file. + # Update rights on Argo CD Applications, for ArgoCDTriggers. See the file. - argocd-applier-role.yaml - argocd-applier-rolebinding.yaml - cluster-role-jane-aggregated.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 762e1642e..5ee8f4ee0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -46,19 +46,19 @@ rules: - apiGroups: - configbutler.ai resources: - - clusterproviders + - argocdtriggers verbs: - get - list - - patch - - update - watch - apiGroups: - configbutler.ai resources: + - argocdtriggers/status - clusterproviders/status - clusterwatchrules/status - commitrequests/status + - fluxtriggers/status - gitproviders/status - gittargets/status - watchrules/status @@ -66,6 +66,17 @@ rules: - get - patch - update +- apiGroups: + - configbutler.ai + resources: + - clusterproviders + - fluxtriggers + verbs: + - get + - list + - patch + - update + - watch - apiGroups: - configbutler.ai resources: @@ -94,6 +105,7 @@ rules: - apiGroups: - configbutler.ai resources: + - fluxtriggers/finalizers - gitproviders/finalizers verbs: - update diff --git a/config/samples/README.md b/config/samples/README.md index 66a580716..3871297bc 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -6,8 +6,10 @@ These samples are quick starting points for common GitOps Reverser setups. - `quickstart-gittarget.yaml`: Minimal `GitTarget` using a non-root `spec.path` and SOPS encryption auto-generation. - `quickstart-watchrule.yaml`: Minimal `WatchRule` for ConfigMaps. - `clusterprovider.yaml`: The conventional `default` provider (in-cluster) and a remote one, showing - `accessFrom` and `allowAnySourceNamespace`. + `accessFrom` and `allowCrossNamespace`. - `clusterwatchrule.yaml`: Minimal `ClusterWatchRule` for cluster-scoped resources. +- `fluxtrigger.yaml`, `argocdtrigger.yaml`: An apply trigger holding the Flux `Kustomization` or + the Argo CD `Application` that applies `example-target`'s folder, for its `spec.applyTriggerRef`. - `commitrequest.yaml`: Minimal `CommitRequest` — an on-demand "save" signal that finalizes a `GitTarget`'s open commit window. It uses `metadata.generateName`, so apply it with `kubectl create -f`: `kubectl apply` refuses a generated name. diff --git a/config/samples/argocdtrigger.yaml b/config/samples/argocdtrigger.yaml new file mode 100644 index 000000000..54ca28cf1 --- /dev/null +++ b/config/samples/argocdtrigger.yaml @@ -0,0 +1,17 @@ +# An ArgoCDTrigger holds the Argo CD Application, with automated sync off, that syncs +# example-target's folder. Name it from the GitTarget with +# spec.applyTriggerRef: {kind: ArgoCDTrigger, name: example-trigger}. See docs/bi-directional-argocd.md. +apiVersion: configbutler.ai/v1alpha3 +kind: ArgoCDTrigger +metadata: + name: example-trigger + namespace: default +spec: + # The cluster the Application is in; "default" is the operator's own. + clusterProviderRef: + name: default + # In another namespace than the trigger's, so the ClusterProvider must set + # allowCrossNamespace.appliers: true, or the trigger reports ApplierNamespaceNotAllowed. + applicationRef: + namespace: argocd + name: example-apps diff --git a/config/samples/clusterprovider.yaml b/config/samples/clusterprovider.yaml index 537d33fda..57d90cc07 100644 --- a/config/samples/clusterprovider.yaml +++ b/config/samples/clusterprovider.yaml @@ -29,7 +29,11 @@ spec: selector: matchLabels: tier: trusted - # Let the GitTargets in those namespaces choose which SOURCE namespaces to mirror. While false - # (the default) a WatchRule may watch only its own namespace, and `sourceNamespace: "*"` is - # refused outright. - allowAnySourceNamespace: true + # What those namespaces may reach outside themselves, each granted on purpose. sources: a + # WatchRule may choose which SOURCE namespaces to mirror; while false (the default) it watches + # only its own namespace, and `sourceNamespace: "*"` is refused outright. appliers: a trigger may + # suspend and apply a Kustomization or Application in another namespace. Both delegate to every + # admitted namespace; neither proves that a namespace owns what it reaches. + allowCrossNamespace: + sources: true + appliers: false diff --git a/config/samples/fluxtrigger.yaml b/config/samples/fluxtrigger.yaml new file mode 100644 index 000000000..1f7b58614 --- /dev/null +++ b/config/samples/fluxtrigger.yaml @@ -0,0 +1,20 @@ +# A FluxTrigger holds the Flux Kustomization that applies example-target's folder. Name it from the +# GitTarget with spec.applyTriggerRef: {kind: FluxTrigger, name: example-trigger}. See +# docs/bi-directional-flux.md. +apiVersion: configbutler.ai/v1alpha3 +kind: FluxTrigger +metadata: + name: example-trigger + namespace: default +spec: + # The cluster the Kustomization is in; "default" is the operator's own. + clusterProviderRef: + name: default + # In another namespace than the trigger's, so the ClusterProvider must set + # allowCrossNamespace.appliers: true, or the trigger reports ApplierNamespaceNotAllowed. + kustomizationRef: + namespace: flux-system + name: example-apps + # How long the branch stays quiet after GitOps Reverser's own pushes before the Kustomization is + # asked to catch up with them. + quietPeriod: 30s diff --git a/docs/INDEX.md b/docs/INDEX.md index 39ce43eb9..7eb5ba774 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -124,7 +124,7 @@ decided), **design, decided** (decision made, not built), **partly built**, **bu **deferred** (parked, kept as a decision record). The label is the first thing in the page, so you never have to read a proposal to find out it already shipped. -### Open (29 pages) +### Open (31 pages) | Doc | Open question | |---|---| @@ -151,7 +151,7 @@ so you never have to read a proposal to find out it already shipped. | [`watch-and-catalog-architecture.md`](design/watch-and-catalog-architecture.md) | the target three-layer watch model — **needs a human call before building** | | [`metrics-observability-plan.md`](design/metrics-observability-plan.md) | **built.** the canonical metrics document, rewritten 2026-09 and shipped in the same wave. Reads the product as two ingestion paths — object state through watch, authorship through audit — joining at the resolver, and gives each stage one counter with a bounded `outcome` so the flow and its loss paths are drawable. Records what three review rounds corrected, including two reversals worth not re-litigating: the fact-loss counters must NOT be merged (an eviction counts a fact, a trim gap an occurrence, a decode error a whole batch), and a cumulative counter cannot replace a current count, which is why `status.streams` and `status.retainedDocuments` stayed. The instrument inventory lives in [`interpreting-metrics.md`](interpreting-metrics.md), not here. Open: the Grafana JSON and alert rules | | [`reconcile-triggering.md`](design/reconcile-triggering.md) | which controllers still fail to wake up | -| [`apply-trigger.md`](design/apply-trigger.md) | **built for Flux and Argo CD; a webhook applier is the aim.** Why GitOps Reverser decides when Git is applied: an apply is safe only when nothing live is missing from the commit, and only the branch's writers can see that. Carries what building it taught (timers are triggers, Flux has no fetch-only mode, a decision must hold at the patch, a retry is a decision so only the worker makes one) and the three units that make the 1:n cases work: the branch owns quietness, the applier owns obligations and retries, the `GitTarget` (by uid) owns consent. Ends with the apply window that is still racy and three ways to shrink it | +| [`apply-trigger.md`](design/apply-trigger.md) | **built for Flux and Argo CD; a webhook applier is the aim.** Why GitOps Reverser decides when Git is applied, and why the applier never applies on its own: a Flux `Kustomization` is held suspended by its `FluxTrigger` and resumed for each apply, an Argo CD `Application` runs with automated sync off. Carries what building it taught (timers are triggers, a long interval is not exclusive control, a decision must hold at the write, a retry is a decision so only the worker makes one, an apply is an operation whose end is observed), the kustomize-controller v1.9.6 facts the suspended lifecycle rests on, its operation (resume token, outcome, the suspend after the apply), who owns `spec.suspend` (a trigger acts only while an eligible `GitTarget` references it, one trigger per applier), restart recovery, shared sources and prefetch, the measured ~50 ms cost, the apply window, and the decisions of 2026-10-17 and 2026-10-19 (the rule, the `allowCrossNamespace.appliers` grant, `Held`) | | [`push-cooldown.md`](design/push-cooldown.md) | **partly built.** Whether the fixed `5s` push cooldown still earns its place now that the commit window exists. Ordinary timer closures already space one author writing one target, but identity changes, saves with shorter timers and buffer-limit closures commit sooner, and serialization does not batch queued commits, so the cooldown is what batches them. Option C, the failure backoff, is built in #412 and shares the push timer; removing the success wait would now remove only `lastPushAt` and its branch, so that decision waits on the event-loop measurements in §9 | | [`multi-source-audit-ingress-hardening.md`](design/multi-source-audit-ingress-hardening.md) | how independent sources authenticate to a named audit route, when annotation routing is trustworthy, and how multi-provider ingestion remains fair | | [`release-image-reuse-plan.md`](design/release-image-reuse-plan.md) | PRs 2–5 unstarted | @@ -160,7 +160,7 @@ so you never have to read a proposal to find out it already shipped. | [`sensitive-resource-diagnostics-follow-up.md`](design/sensitive-resource-diagnostics-follow-up.md) | deferred diagnostics | | [`e2e-git-server-choice.md`](design/e2e-git-server-choice.md) | stay on Gitea or move to Forgejo — the `_csrf` pin is fixable in place on both, so the migration is now a preference call, not a fix; also why we adopt no SDK either way | -### Built, and kept here anyway (6 pages) +### Built, and kept here anyway (7 pages) These have shipped. They stay in `design/` under the exception above, because Go source cites them by path as the rationale for what the code does, and `finished/` declares @@ -168,6 +168,7 @@ itself non-binding. Read them as history that the code still points at. | Doc | Open question | |---|---| +| [`push-notification-and-reconcile-trigger.md`](design/push-notification-and-reconcile-trigger.md) | **built, and superseded** by [`apply-trigger.md`](design/apply-trigger.md): the record of how the apply trigger was decided. Kept here because its Part 1, how the branch worker reads the remote (the pre-push fetch, the push advertisement, base trust), is cited from the Git code by section number | | [`vocabulary-cleanup.md`](design/vocabulary-cleanup.md) | **built.** The breaking release that made the API obey [`definitions.md`](definitions.md): condition reasons that restated their type are `Succeeded`, `GitRepoConfigNotFound` is `GitProviderNotFound`, printer columns are named for what they read and the dependency projections are gone, a bare commit hash is `commit` in every status field, and `ClusterProvider`'s throttles are `spec.client`. Kept here because `definitions.md` cites it as the record of which audit findings the existing rulebooks already settled: the `insecure` flag split, bare-noun booleans, `--redis-addr`, and the lowercase `sops` enum | | [`created-root-namespace.md`](design/created-root-namespace.md) | **built.** One question with five answers: what namespace a `kustomization.yaml` the operator CREATES should carry. Decided **B, never write one** — `spec.serializeNamespace: false` means the artifact does not encode its deployment namespace, and adding a root must not quietly change that contract; the namespace comes from the documents when the field is unset or `true`, and from the installer (Flux `targetNamespace`, Argo `destination.namespace`) when it is `false`. Records the three facts an earlier draft got wrong (a namespace-less root is ordinary, both installers supply one, and what refused it was our own fidelity gate rather than kustomize), and carries the scoped fidelity rule that follows: the namespace is ignored in the render comparison ONLY when the governing root sets none, so a root that declares `namespace: shop` still rejects a live `billing` object. Shipped in #328, which is also why [`kustomization_bootstrap.go`](../internal/git/kustomization_bootstrap.go) cites the page by path. Also the sibling call: under `useKustomize` a placement no `resources:` list would name is refused rather than committed unrendered | | [`watch-manager-ownership.md`](design/watch-manager-ownership.md) | A rule edit used to be applied inline by the controller worker that observed it, and it re-planned EVERY GitTarget rather than the one the rule names: 1256 plan reconciles across 28 targets in one e2e run, peaking at 78 in a second, behind two network calls, on a shared worker pool. The watch manager had no owner, so eleven mutexes stood in for one. Now controllers post a trigger naming a GitTarget and return, one loop owns the plan and paces itself, and repeated triggers for one target collapse into a single pass. The debounce is framed around how the config is actually edited: a GitTarget and its rules are one piece of configuration applied together, so a per-target ROLLING SILENCE window of 2s (max wait ~10s) turns a five-object `kubectl apply` into one pass, the same mechanism `DefaultCommitWindow` already uses one layer down on the write path. That reverses an earlier revision's "never debounce the first declaration": declaring a GitTarget the instant it lands means declaring it with no rules yet, which manufactures a transient EMPTY plan on every cold start, and an empty plan is what vacuously cleared a write divergence in the fidelity gate. States the contract as "one settled configuration adjustment, not one function invocation": the window is a heuristic and never a correctness boundary (Kubernetes has no apply-complete event), a per-target DIRTY SEQUENCE means a change arriving mid-pass is never lost, and the pass reads a coherent rule-store snapshot rather than the rule that triggered it. Carries the deletion inventory, because the point is that the system got smaller: four trigger mechanisms collapsed to one (`signalCatalogRefresh` and `catalogRefreshCh` are gone), `refreshRunningTargetWatches` and its running-set filter are gone (that filter is why a target whose first declare never completed was never picked up again), and six mutexes went for stated reasons, with `RenderFidelityGate.mu` and the two event-channel locks kept and justified. The four steps shipped: the local-cluster discovery call is bounded (a real defect — the legacy non-context `ServerGroupsAndResources()` ran with no deadline at all), the owner loop carries the debounce, dirty sequence, per-target deadline and 2s/5s/10s/30s/1m backoff, the plan now carries NO lock while the projection is a published snapshot, and catalog invalidation is scoped by diffing each target's rendered plan across the re-projection. "What shipped" records where the implementation departed from the page, including the one bug only e2e caught: streams were parented to the PASS context, which a deadline cancels the moment the pass returns, so every stream died the instant its plan was applied — and it reads like health, because the plan logs `start:1`, every later pass reports `keep:1` and never restarts it, and nothing logs an error while readiness sits at `Replaying` and every WatchRule sits `Ready=False`. A stream's parent is the manager's lifetime; the pass deadline bounds the pass. A second e2e catch is a BEHAVIORAL consequence worth knowing: toggling a rule off and on inside the settle window is no longer a replay — it used to tear the stream down and re-establish it because each apply replanned synchronously, and it is now one pass over a plan that never changed. Correct (a net-zero change is no change; widening `prune.mode` remains the supported force) but a real difference in what an operator gesture does. Both specs that broke were also gating on the wrong thing: asking a GitTarget "are all your streams running" about a change to ONE rule, which a target that is already mirroring answers True to before that rule has been planned, and which a different controller publishes than the one that compiled the rule — so the rule's OWN StreamsRunning is the gate, and `waitForWatchRuleStreamsRunning` existed unused for exactly this. Departures: reports became a published snapshot rather than a second channel; ISOLATION came from taking the I/O off the loop rather than from the deadline (a pass never dials, the shared refresh runs on its own goroutine, and the deadline is the backstop it should have been — a first cut that kept two network calls on the loop had one unreachable cluster holding every healthy target, which is the same availability failure relocated); DELETION names an incarnation resolved when it is queued, because both production callers react to a NotFound and carry no UID, so a UID-less delete matched everything and could tear down the successor of a same-name recreate; and persistent failure surfaces as `WatchPlanFailing`, where pending means "no pass has ever landed", not "dirty right now". Step 3's type-to-target index did not earn its staleness. Still open: whether the settle window ever needs to be configurable | @@ -225,7 +226,7 @@ Four more ideas sit beside them. ## History — [`finished/`](finished/) -Twenty-two shipped plans and closed investigations. **Nothing here binds.** Read one +Twenty-three shipped plans and closed investigations. **Nothing here binds.** Read one only when you want to know *why* something is the way it is; the answer to *what it is* always lives in `spec/`. @@ -241,7 +242,7 @@ collection delete the API server sent no response body for used to lose its auth watch resume cursors still live in Redis, so this frees attribution from it rather than the install). Shipped as #283, #284, #286 and #287. -Before it, +Before that, [`analyzer-consumer-contract-asks.md`](finished/analyzer-consumer-contract-asks.md): why a refusal carries whether it can be solved and by whom, why the analyzer's report is a KRM document that names the build that produced it, and why `ResourceIdentifier.Key()` is a diff --git a/docs/README.md b/docs/README.md index 9f7c0ffa7..fddd72d90 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,10 +27,12 @@ If you only want the supported product docs, start with the files below. - [`sops-age-guide.md`](sops-age-guide.md): Secret encryption with SOPS + age - [`security-model.md`](security-model.md): controller access, trust boundaries, and the Git credentials Secret shape -- [`rbac.md`](rbac.md): the two ClusterRoles, and how to stop the reverser enumerating Secrets -- [`bi-directional.md`](bi-directional.md): safe shared-path and handoff patterns -- [`flux.md`](flux.md): the complete Flux setup when GitOps Reverser decides when Flux applies -- [`argocd.md`](argocd.md): the complete Argo CD setup when GitOps Reverser starts every sync +- [`rbac.md`](rbac.md): the two ClusterRoles (plus the optional applier grants), and how to stop the + reverser enumerating Secrets +- [`bi-directional.md`](bi-directional.md): when Flux or Argo CD applies a folder GitOps Reverser + writes, and why every other setup needs nothing +- [`bi-directional-flux.md`](bi-directional-flux.md): the complete Flux setup for such a folder +- [`bi-directional-argocd.md`](bi-directional-argocd.md): the complete Argo CD setup for such a folder - [`alternatives.md`](alternatives.md): nearby tools and when another approach fits better - [`UPGRADING.md`](UPGRADING.md): breaking changes and migration steps, newest first - [`../CONTRIBUTING.md`](../CONTRIBUTING.md): contributor workflow and validation commands diff --git a/docs/TODO.md b/docs/TODO.md index 97b746b4d..74c1e7ca7 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -91,7 +91,7 @@ rebuild fails is no longer dropped: it stays pending for that retry signing snapshot-message specs, then passed on rerun. Capture and mitigation notes live in the [e2e serial registry](spec/e2e-serial-registry.md). This should be addressed before the next feature that expands commit-message, snapshot, or - write-window behavior, otherwise new failures will be hard to separate from existing timing debt. + write-window behavior, otherwise new failures will be hard to separate from the existing timing flakes. - [ ] Use [bestpractices.dev](https://www.bestpractices.dev/en) as a project maturity checklist. Review the current gaps, decide which items matter for this project, and turn the useful ones @@ -206,7 +206,8 @@ rebuild fails is no longer dropped: it stays pending for that retry lands. - [ ] Better branching and promotion strategies. -- [ ] Bi-directional GitOps alignment with controllers such as Flux and Argo CD. +- [ ] A webhook applier, so an apply trigger can hold a tool other than Flux or Argo CD. Flux and + Argo CD are built; see the [apply trigger design](design/apply-trigger.md). Research work: diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index a7aa3c466..7ad611cfd 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -7,42 +7,79 @@ guidance that the changelog's breaking-change entries link to. We are pre-1.0, so breaking changes bump the **minor** version (release-please is configured with `bump-minor-pre-major`) rather than the major. Read the relevant entry before upgrading across it. -## `onRefusal: PushEmptyCommit` is now `onRefusal: Reapply`, and needs `spec.applier` +## `allowAnySourceNamespace` is gone; `allowCrossNamespace.sources` replaces it + +**Breaking.** `ClusterProvider.spec.allowAnySourceNamespace` is no longer read. +`spec.allowCrossNamespace.sources` takes its place, beside `spec.allowCrossNamespace.appliers` for +apply triggers. A cross-namespace `WatchRule` (a `rules[].sourceNamespace` other than its own +namespace, `"*"` included) that relied on the old field stops mirroring and reports +`SourceNamespaceAuthorized=False` (`SourceNamespaceNotAllowed`), and the message names +`allowCrossNamespace.sources`. Set it after the upgrade, on each provider that set the old field: + +```yaml +spec: + allowCrossNamespace: + sources: true + # allowAnySourceNamespace: true <- remove this line +``` + +For the chart-rendered `default` provider, set `clusterProvider.default.allowCrossNamespace.sources: +true`. A provider that still stores `allowAnySourceNamespace: true` is read as if the line were not +there, and any change to its `spec` is refused ("allowAnySourceNamespace was removed; set +allowCrossNamespace.sources") until the line is removed or set to `false`. `false`, which the old +default stored on every provider, is accepted. + +## `onRefusal: PushEmptyCommit` is now `onRefusal: Reapply`, and needs an apply trigger **Breaking.** `GitTarget.spec.onRefusal` accepts `Ignore` and `Reapply`. `PushEmptyCommit` is gone: GitOps Reverser no longer moves the branch with an empty commit when an edit is refused. With -`Reapply` it asks the target's applier, the Flux `Kustomization` named in the new -`spec.applier.kustomizationRef` or the Argo CD `Application` named in `spec.applier.applicationRef`, -to apply the current tip once the branch holds no open commit window and no pending writes. Only -that applier is asked, rather than every applier watching the branch, though a `GitRepository` that -has to fetch the tip still wakes every `Kustomization` that reads it. An allowed edit still in a -commit window is published before the apply instead of being reverted with the refused one. The API -rejects `Reapply` without `spec.applier`. - -A stored `GitTarget` that still says `PushEmptyCommit` reads as `Ignore`, and any update to it is -rejected until the field changes. The old CRD rejects `Reapply`, so move in this order: +`Reapply` it asks the target's applier to apply the current tip once the branch holds no open +commit window and no pending writes. The applier is the Flux `Kustomization` a `FluxTrigger` holds, +or the Argo CD `Application` an `ArgoCDTrigger` holds, and the `GitTarget` names the trigger in the +new `spec.applyTriggerRef`. Only that applier applies, rather than every applier watching the +branch; a `GitRepository` that has to fetch the tip wakes only the consumers no trigger holds. An +allowed edit still in a commit window is published before the apply instead of being +reverted with the refused one. The API rejects `Reapply` without `spec.applyTriggerRef`. + +A stored `GitTarget` that still says `PushEmptyCommit` reads as `Ignore`. The new CRD no longer +lists the value: an API server that ratchets CRD validation (Kubernetes 1.30 and later) still +accepts an update that leaves it in place, and an older one rejects every write to the target but a +status update until the field changes. The old CRD rejects `Reapply`, so move in this order: 1. Before upgrading, set `onRefusal: Ignore` on those targets. 2. Upgrade the CRDs, the controller and its permissions. -3. Set `spec.applier` on each target and wait for `ApplierConfigured=True`. +3. Create a `FluxTrigger` or `ArgoCDTrigger` for each applier, name it in each target's + `spec.applyTriggerRef`, and wait for `ApplyTriggerReady=True`. A trigger whose applier is in + another namespace than its own, such as `flux-system` or `argocd`, needs + `allowCrossNamespace.appliers: true` on its `ClusterProvider` (Helm: + `clusterProvider.default.allowCrossNamespace.appliers`), or it reports + `ApplierNamespaceNotAllowed`. 4. Set `onRefusal: Reapply` where you want it. -See [reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal). - -`spec.applier` also changes what else applies the folder: GitOps Reverser asks the Kustomization to -apply when somebody else pushes to the branch, and to catch up with its own pushes once the branch -has been quiet for `spec.applier.quietPeriod`. Lengthen the `GitRepository` and `Kustomization` -intervals and remove any Git-host webhook into a Flux `Receiver` for the repository, as the -[bi-directional guide](bi-directional.md#configure-flux) describes, or Flux keeps applying on its own -schedule as well. The operator needs `get` and `patch` on `kustomizations` and `gitrepositories`, -and `create` on `selfsubjectaccessreviews`, in the source cluster: Helm `applierAccess.flux: true` -grants them for the operator's own cluster. +See [who applies this folder](configuration.md#who-applies-this-folder-specapplytriggerref) and +[reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal). + +A trigger also changes what else applies the folder. A `FluxTrigger` holds its `Kustomization` +suspended, so Flux applies it only when GitOps Reverser resumes it: when somebody else pushes to the +branch, after a refusal or a reverted edit, and to catch up with its own pushes, or with the branch +once a `GitTarget` first names the trigger, after the branch has been quiet for the trigger's +`quietPeriod`. `flux get ks` shows it `Suspended`, and while the +operator is down it waits suspended: a graceful stop suspends one it had resumed, and after a crash +the next process does. Remove `spec.suspend` from whatever applies the +`Kustomization`'s manifest (the trigger reports `SuspendOwnedElsewhere` otherwise) and any Git-host +webhook into a Flux `Receiver` for the repository, as [the Flux guide](bi-directional-flux.md) +describes. A trigger acts only while an eligible `GitTarget` names it, so create the two in any +order: until one names it, it is `Ready` (`NotReferenced`) and holds nothing. Deleting the trigger, +or the last `GitTarget` naming it, hands the `Kustomization` back to Flux. The operator needs `get`, +`list` and `patch` on `kustomizations`, `list` on `helmreleases` and `helmcharts`, `get` and `patch` +on `gitrepositories`, and `create` on `selfsubjectaccessreviews`, in the trigger's cluster: Helm +`applierAccess.flux: true` grants them for the operator's own cluster. For an Argo CD `Application`, GitOps Reverser starts every sync, so switch its automated sync off -(`spec.syncPolicy.automated.enabled: false`, keeping `prune`) before naming it; one with automated -sync on reports `ApplierConfigured=False` (`AutomatedSyncEnabled`) and is never synced. The operator -needs `get` and `update` on `applications`: Helm `applierAccess.argocd: true`. See -[Using GitOps Reverser with Argo CD](argocd.md). +(`spec.syncPolicy.automated.enabled: false`, keeping `prune`) before its trigger holds it; one with +automated sync on reports `Configured=False` (`AutomatedSyncEnabled`) on the trigger and is never +synced. The operator needs `get` and `update` on `applications`: Helm `applierAccess.argocd: true`. +See [bi-directional GitOps with Argo CD](bi-directional-argocd.md). ## `git_queue_drops_total` is now `git_queue_refusals_total` diff --git a/docs/api-first-publication.md b/docs/api-first-publication.md index 685528317..111698b25 100644 --- a/docs/api-first-publication.md +++ b/docs/api-first-publication.md @@ -27,13 +27,11 @@ particular deployment; requiring a fetch before every push would charge every AP common case of a competing Git writer. The push already checks the remote branch, so a healthy worker can plan locally and spend its network budget on publication. -This guide describes the shipped behavior. The inbound Git push receiver remains a proposal: see -[how to call the receiver](configuration.md#receiving-push-notifications-specreceiver) -for the request shape it will accept. The one scheduling limitation to know about is that retained -work has no retry timer of its own; [failure timing](#failure-timing-and-memory-limits) says what -that means in practice. The [apply trigger design](design/apply-trigger.md) -contains the implementation history and the proposed receiver; [architecture](architecture.md) -covers the wider operator. +This guide describes the shipped behavior. Retained work is retried on the worker's own backoff; +[failure timing](#failure-timing-and-memory-limits) says what that means in practice. The +[push receiver](configuration.md#receiving-push-notifications-specreceiver) lets a Git host report +a push, the [apply trigger design](design/apply-trigger.md) covers what follows one, and +[architecture](architecture.md) covers the wider operator. ## The structures and their responsibilities @@ -181,7 +179,7 @@ them is retried automatically, so it is worth knowing which one you are looking | --- | --- | --- | | Sensitive-resource encryption | Secrets and configured sensitive types must never reach Git in plaintext | The write fails rather than falling back to plaintext. There is no opt-out | | Acceptance of the Git path | The target's folder must be content the writer can edit safely | The plan is refused and reported as `GitPathAccepted=False` on the `GitTarget`. The periodic refresh raises the same condition off a READ, so folder content somebody else pushed is reported before a write goes looking for it. The target re-checks roughly every ten seconds; recovery requires an accepted resync after correcting the folder or live state, or — for a refusal a read raised — a later read that finds the folder clean | -| Render fidelity | A target whose render-vs-live epoch is pending or divergent must not take live writes | Live events and atomic writes are dropped while the gate is closed. Resync stays allowed so it can measure and repair Git | +| Render fidelity | A target whose render-vs-live epoch is pending or divergent must not take live writes | Live events and atomic writes are dropped while the gate is closed, counted by `gitopsreverser_git_writes_dropped_total`, and the first drop under each gate status is logged at Info. Resync stays allowed so it can measure and repair Git. The gate is per `GitTarget`: while any of its collections is replaying, edits to all of them are dropped. A live edit queued behind a replay is not: the worker records the replay's result before it takes the next item | A refusal is not a transient error and is not retried into success. It is the common reason a live edit never appears in Git while the worker looks healthy, so check the `GitTarget` conditions before @@ -233,8 +231,9 @@ preserved within the supported layout rules. See [shared-path behavior](bi-directional.md#what-a-write-does-to-the-file). This is the consequence of API-first ownership: the captured API object drives the supported -manifest edit. A Git-side change reaches the cluster through a separate reconciler such as Flux -or Argo CD. Reverser does not apply Git to Kubernetes. +manifest edit. A Git-side change reaches the cluster through an applier such as Flux or Argo CD. +Reverser does not apply Git to Kubernetes itself; with an apply trigger it decides when the applier +does. The [deferred three-way comparison investigation](future/git-api-three-way-comparison.md) records the exact merge rules and the alternatives: timestamps, retained API contents, Git field @@ -333,19 +332,23 @@ idle target can hold an old view after a foreign push; `--git-refresh-interval=0 and restores the old behaviour, where the view is corrected by a later publication, resync, or explicit recheck. -The proposed inbound push receiver would notify the worker that its base needs refreshing. It -would coalesce notifications, refresh on the worker, and replay any retained writes. It is useful -for idle visibility and for avoiding a rejected push. The healthy publication optimization -already works without that receiver. +The [push receiver](configuration.md#receiving-push-notifications-specreceiver) tells the worker +that somebody pushed, in seconds rather than at the next refresh. A delivery is queued on the +worker like a refresh and is only a hint: on an idle branch the worker reads the advertisement and +fetches only if the branch moved; on a busy one it closes its open window and publishes, and the +compare-and-swap push finds the move. The healthy publication optimization works without a +receiver. There is a separate, existing manual mechanism: changing `reconcile.configbutler.ai/requestedAt` on a `GitTarget` requests a fresh reconcile. That performs a cluster-to-Git snapshot after fetching. It can write the cluster's current values over the -fetched manifests. A passive Git refresh endpoint for a future webhook needs separate handling. +fetched manifests, which is why a receiver delivery never triggers one. -When Flux or Argo CD shares the path, its own refresh and apply cadence also affects whether a -live edit survives long enough to be published. Those clocks belong to the other reconciler. -The [bi-directional guide](bi-directional.md) describes the integration choices and limitations. +When Flux or Argo CD applies the same path on its own schedule, its refresh and apply cadence also +affects whether a live edit survives long enough to be published. An +[apply trigger](configuration.md#who-applies-this-folder-specapplytriggerref) takes that clock +away: the applier then applies only when GitOps Reverser asks, on a quiet branch. The +[bi-directional guide](bi-directional.md) describes the integration choices and limitations. ## The other clocks around publication @@ -395,10 +398,10 @@ not rate-limit fetches needed by a new window or snapshot. A branch with nothing silent. The byte threshold forces finalization but does not free retained writes during a remote outage. -It is not a hard memory ceiling. The separate branch queue has `1000` slots by default and drops -new items when full. Current-state resync can repair the resulting mirror, but cannot recover the -exact dropped history. These throughput and recovery limits do not justify fetching before every -healthy publication. +It is not a hard memory ceiling. The separate branch queue has `1000` slots by default and refuses +new items when full; the producer offers a refused item again (`git_queue_refusals_total`), and only +a watch cursor that expires first costs that write's own commit. These throughput and recovery +limits do not justify fetching before every healthy publication. ## What the network saving measures @@ -437,7 +440,7 @@ entry point. They do not measure the complete operator startup sequence. Snapshot resync keeps its fetch because a no-op snapshot can finish without a push advertisement. Avoiding that fetch safely would require a separate freshness check on that path. -The proposed notification's savings need their own measurement. With today's primitives, an +A receiver delivery's savings need their own measurement. With today's primitives, an early notification followed by one object-transferring fetch and one successful push costs five HTTP requests. Its benefit in that scenario is avoiding the rejected advertisement, reducing six requests to five. Multiple remote moves depend on notification timing and coalescing. @@ -465,8 +468,8 @@ Push-cycle duration excludes the earlier attribution and commit-window waits, cooldown, and recovery performed before entering the cycle. It cannot by itself answer “how long from my API edit until Git accepted it?” -A flat recovery counter also cannot prove progress: the worker may be retaining work without a -retry timer. Check queue depth, failure logs, and `CommitRequest` outcomes alongside rates. +A flat recovery counter also cannot prove progress: the worker may be retaining work while it +waits out its backoff. Check queue depth, failure logs, and `CommitRequest` outcomes alongside rates. A resync's fetch lands under `forced_recheck` whether or not the branch happened to hold retained writes, so a growing `publication` series means live publication is fetching again and nothing diff --git a/docs/architecture.md b/docs/architecture.md index 06bed6d4e..92ae28135 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,21 +117,24 @@ publish them over the incoming change. **GitOps Reverser decides when Git is applied to the cluster.** An apply is safe only when nothing live is missing from the commit being applied, and only the workers writing the branch can see the -edits that are live and not yet pushed. So a `GitTarget` names its **applier**, the applier's own -triggers are switched off, and the branch worker sends **apply requests** instead. A request is sent -only on a **quiet branch**, with no open commit window and no pending writes on any worker writing -that repository and branch: +edits that are live and not yet pushed. So a `GitTarget` names an **apply trigger**, which holds +its **applier** so that it applies nothing on its own (a Flux `Kustomization` is held suspended, an +Argo CD `Application` has automated sync off), and the branch worker sends **apply requests** +instead. A request is sent only on a **quiet branch**, with no open commit window and no pending +writes on any worker writing that repository and branch: - for a commit somebody else pushed, after closing open windows and replaying the pending writes onto it, so the applied commit carries the API side's answer; - for a refused edit the target opted to revert; -- for Flux only, to keep its artifact current after GitOps Reverser's own pushes, because Flux's - timers and restarts apply whatever artifact it holds. Argo CD is never asked to apply GitOps - Reverser's own commits: with automated sync off it applies nothing on its own, and a sync is pinned - to the commit the worker checked. - -The worker owns each request until it is delivered, and decides every retry the same way. A request -asks and never reports; it runs off the publication path and cannot fail a publication. **Built:** +- for an API edit that an apply reverted on the cluster after the edit was made, to put it back; +- for Flux only, to catch up with GitOps Reverser's own pushes once the branch has been quiet for + the trigger's `quietPeriod`, because Flux's inventory learns of an object only from an apply of a + commit that holds it. Argo CD is never asked to apply GitOps Reverser's own commits: with automated + sync off it applies nothing on its own, and a sync is pinned to the commit the worker checked. + +The worker keeps each reason pending until an apply operation answers it, and decides every retry +the same way. A request runs off the publication path and cannot fail a publication; the trigger's +`status.operation` records the operation it started. **Built:** the Flux `Kustomization` applier and the Argo CD `Application` applier, with automated sync off. **The aim:** a webhook for any other tool. See [the apply trigger design](design/apply-trigger.md). @@ -252,11 +255,13 @@ capture audit and admission with **no** resulting watch event. ## Configuration model -You configure GitOps Reverser through six CRDs (group `configbutler.ai`, version `v1alpha3`). +You configure GitOps Reverser through eight CRDs (group `configbutler.ai`, version `v1alpha3`). `WatchRule` and `ClusterWatchRule` choose which Kubernetes resources enter the pipeline. `CommitRequest` can ask for the current window to be saved. `GitTarget` joins one source cluster to one Git destination. `ClusterProvider` supplies the source connection and authorization boundary; -`GitProvider` supplies the repository, credentials, commit settings, and push policy. +`GitProvider` supplies the repository, credentials, commit settings, and push policy. `FluxTrigger` +and `ArgoCDTrigger` hold the Flux `Kustomization` or Argo CD `Application` that applies a folder back, +so GitOps Reverser decides when it applies. ```mermaid graph LR @@ -264,6 +269,7 @@ graph LR WR[WatchRule] -->|gitTargetRef| GT[GitTarget] CWR[ClusterWatchRule] -->|gitTargetRef| GT CR[CommitRequest] -->|gitTargetRef| GT + GT -->|applyTriggerRef| TR["FluxTrigger /\nArgoCDTrigger"] GT -->|clusterProviderRef| CP[ClusterProvider] GT -->|gitProviderRef| GP[GitProvider] KCFG[("Remote kubeconfig Secret\noperator namespace")] @@ -280,7 +286,7 @@ graph LR CP -. "optional kubeconfig reference" .-> KCFG CP -->|"source identity + config"| WM WM -->|"source client"| API - GT -. "spec.applier" .-> APPLIER + TR -. "kustomizationRef / applicationRef" .-> APPLIER HOST -. "push webhook to spec.receiver" .-> GP style GP fill:#e8f4fd,stroke:#2196f3 @@ -298,9 +304,10 @@ source context: client, discovery catalog, followability registry, source-namesp reachability state. Its `GitTarget`s open their source watches through that context. The contexts are keyed by provider name and are released when no `GitTarget` references them. -The applier is not one of the six kinds. It belongs to Flux or Argo CD, lives in the source cluster -it applies to or in the cluster `spec.applier.clusterProviderRef` names, and is reached through that -`ClusterProvider`'s identity. +The applier is not one of the eight kinds. It belongs to Flux or Argo CD, lives in the cluster the +trigger's `spec.clusterProviderRef` names (the source cluster, or a management cluster), and is +reached through that `ClusterProvider`'s identity. The trigger is ours: it holds the applier, and +records the apply operation it started. | CRD | Scope | One line role | |---|---|---| @@ -310,6 +317,8 @@ it applies to or in the cluster `spec.applier.clusterProviderRef` names, and is | `GitTarget` | namespaced | one materialization from a source provider to `(provider, branch, path)` | | `ClusterProvider` | cluster | one source-cluster connection plus namespace access policy | | `GitProvider` | namespaced | a Git repo + credentials + commit/signing config | +| `FluxTrigger` | namespaced | the Flux `Kustomization` GitOps Reverser asks to apply, and the apply in flight | +| `ArgoCDTrigger` | namespaced | the Argo CD `Application` GitOps Reverser asks to sync, and the sync in flight | ### WatchRule / ClusterWatchRule @@ -345,7 +354,7 @@ The two namespace controls intentionally live in different planes: - `ClusterProvider.spec.accessFrom` authorizes **control-cluster** namespaces to reference the provider. It is a tenant/export boundary; it neither selects nor grants access to source namespaces. -- `ClusterProvider.spec.allowAnySourceNamespace` delegates the CHOICE of source namespace to the +- `ClusterProvider.spec.allowCrossNamespace.sources` delegates the CHOICE of source namespace to the `GitTarget`s that provider admits. It grants nothing: what may actually be read from the source cluster is bounded by that credential's own Kubernetes RBAC, which is the only bound there is. The operator reads no `Namespace` objects in a source cluster for authorization. @@ -400,12 +409,11 @@ Key fields: - `spec.commit.window`: when this target's commit windows close: `idleTimeout` (default `5s`) of silence, or `maxDuration` (default `1m`) after the window opened. - `spec.commit.message`: `liveTemplate` / `reconcileTemplate` Go templates. -- `spec.applier`: optional; exactly one of `kustomizationRef` (the Flux `Kustomization` that applies - this folder, and that GitOps Reverser asks to apply) and `applicationRef` (the Argo CD - `Application`, automated sync off, whose syncs GitOps Reverser starts), with `quietPeriod` - (default `30s`) for a `Kustomization` only. +- `spec.applyTriggerRef`: optional; the `FluxTrigger` (a Flux `Kustomization` GitOps Reverser asks + to apply, with its `quietPeriod`, default `30s`) or `ArgoCDTrigger` (an Argo CD `Application`, + automated sync off, whose syncs GitOps Reverser starts) in this namespace. - `spec.onRefusal`: `Ignore` (default) or `Reapply`, which asks the applier to reapply the tip when - an edit to an existing document is refused. `Reapply` requires `spec.applier`. + an edit to an existing document is refused. `Reapply` requires `spec.applyTriggerRef`. `spec.commit` describes the folder, not the connection, so two `GitTarget`s sharing one `GitProvider` may batch and phrase their commits differently. A branch worker serves a `(provider, branch)` pair and @@ -429,11 +437,14 @@ Status has a kstatus-compatible summary layer plus domain conditions: source API the data plane can currently reach. - `GitPathAccepted` explains the target side: the selected Git path is safe for the operator to materialize. -- `ApplierConfigured`, present only with `spec.applier`, says the applier exists in its cluster, - applies this repository and branch, can be written, and (an `Application`) has automated sync off. - It is a gate. +- `ApplyTriggerReady`, present only with `spec.applyTriggerRef`, says the trigger is `Ready` (its + applier exists, can be written, and, an `Application`, has automated sync off) and that the + applier applies this repository and branch. It is a gate. - `status.streams` is a bounded count summary, not a per-type list. -- `status.applier` is an observation: the last apply request's commit, time and reason. + +A trigger has `Configured` (a `FluxTrigger` also `Held`) and the kstatus trio, `status.source` +(what its applier applies, as last read), and `status.operation` and `status.repairPending`, which +the branch worker writes at every move of the operation it started. WatchRule and ClusterWatchRule add `ResourcesResolved` and `GitTargetReady`. `ResourcesResolved` explains the source selector. `GitTargetReady` mirrors the referenced GitTarget's write readiness. This keeps @@ -671,22 +682,27 @@ A delivery is handled on the worker's own queue, so it is never judged while a p the push has recorded its commit before the delivery is read. A rollback onto one of the worker's older commits is foreign, because the advertisement names a tip other than the last one observed. -The same quiet-branch rule sends two other requests: `Refusal`, to the refusing target's applier -when `spec.onRefusal: Reapply` is set, and `CatchUp`, to every applier on the branch once it has -been quiet for `spec.applier.quietPeriod` after the worker's own pushes (the longest among the -`GitTarget`s that share an applier, since its apply reaches all their folders). +The same quiet-branch rule sends three other requests: `Refusal`, to the refusing target's applier +when `spec.onRefusal: Reapply` is set; `Repair`, to an applier whose late write over a newer API +edit the apply window withheld; and `CatchUp`, to every Flux `Kustomization` on the branch once it +has been quiet for the trigger's `quietPeriod` after the worker's own pushes, or after a `GitTarget` +first names the trigger. -What each applier is owed lives on the worker, per applier: one attempt out at a time, and its own -retry clock. The dispatcher in `internal/applier` carries out one attempt off the worker's queue and +Which apply is due lives on the worker, per applier: one attempt out at a time, and its own retry +clock. The dispatcher in `internal/applier` carries out one attempt off the worker's queue and reports back. It reads each `GitTarget` again first (the same incarnation by uid, still naming this -applier on this branch, still admitted, still consenting to a `Refusal`), checks that the -`Kustomization`'s source fetches this repository and branch, asks the worker whether its decision -still holds (the commit is still the tip and nothing is waiting to be published), and only then -stamps `reconcile.fluxcd.io/requestedAt` on the `GitRepository`, or, for a `Refusal` at an artifact -that already holds the commit, on the `Kustomization`. It never retries: +applier through its trigger on this branch, still admitted, still consenting to a `Refusal`), checks that the +`Kustomization`'s source fetches this repository and branch, has the `GitRepository` fetch the +commit when its artifact lacks it, asks the worker whether its decision still holds (the commit is +still the tip and nothing is waiting to be published), writes the operation through to the trigger's +`status.operation`, and only then resumes the `Kustomization` the trigger holds suspended, with a +`requestedAt` token. It follows the apply until that token is handled, suspends the `Kustomization` +again with a second token, and reports the operation complete once that one is handled +([`apply-trigger.md`](design/apply-trigger.md#the-flux-applier-suspended)). It never retries: an attempt that does not deliver gives the worker back what it carried, a refusal only while it still stands, and the worker decides the retry on a quiet branch after that applier's backoff. A delivered -request records an `ApplyRequested` Event and `status.applier`. A failed request never reaches back +request records an `ApplyRequested` Event, and the worker records the operation on the trigger's +`status.operation`. A failed request never reaches back into the publication that caused it. *** @@ -1258,8 +1274,8 @@ stops a fact being published by something that cannot actually prove it. | Where is branch B on the remote, and when was that proved? | the branch worker, on its queue | `GitTarget.status.remote`, sampled by each `GitTarget` on the branch | | What is this repository being used for? | the `GitTarget`s that reference it | `GitProvider.status.branches` | | Is this folder accepted, streaming and matching live? | the watch plane, per `GitTarget` | the `GitTarget`'s conditions and `status.placement` | -| Does the applier need to apply, and is now safe? | the branch worker, on its queue | `GitTarget.status.applier` and an `ApplyRequested` Event | -| Can the applier be asked at all? | the `GitTarget` controller, reading the applier in the source cluster | `ApplierConfigured` | +| Does the applier need to apply, and is now safe? | the branch worker, on its queue | the trigger's `status.operation` and an `ApplyRequested` Event | +| Can the applier be asked at all? | the trigger controller, reading the applier in its cluster | the trigger's `Configured`, and the `GitTarget`'s `ApplyTriggerReady` | A branch worker sits at the intersection and belongs wholly to neither object: `GitTarget`s decide which branches are needed, `GitProvider` decides which repository those branches are in. So the @@ -1354,7 +1370,7 @@ meantime is a moved remote too, so the replay builds on their commit instead of holds when the server refuses our upload too, because the fallback reads both refs from one advertisement. A cycle planned on an empty repository that has since gained refs is refused at the advertisement (`RepositoryNotEmptyError`) and replayed, never pushed as an orphan. A parent that is -missing, or a default branch that does not resolve, is a recovery obligation on the worker: it +missing, or a default branch that does not resolve, opens parent recovery on the worker: it probes on its own backoff and publishes what it held back once the parent exists (see `docs/design/gittarget-parent-branch.md`). Proved end to end by the e2e spec "New write branch starts from its parent"; that publication alone finds a moved parent is proved by worker tests @@ -1556,9 +1572,12 @@ Controllers watch their dependencies so dependents reconcile quickly after spec Namespace label changes re-check provider-reference access. Rule changes re-resolve selected collections. It deliberately does **not** watch encryption Secrets, so their recovery is picked up by periodic reconciliation without retaining - every Secret value in the control-plane cache. Nor does it watch appliers: it reads the Flux - `Kustomization` on its tick to set `ApplierConfigured`, so a cluster without the Flux CRDs needs no - informer for them. + every Secret value in the control-plane cache. It watches the apply triggers its `GitTarget`s name, + for `ApplyTriggerReady`. +- `ApplyTriggerReconciler` (one per trigger kind) watches its triggers, the `GitTarget`s naming + them, `GitProvider`s, `ClusterProvider`s and `Namespace` labels. It does not watch appliers: it + reads the Flux `Kustomization` or Argo CD `Application` on its tick to set `Configured`, so a + cluster without the Flux or Argo CD CRDs needs no informer for them. - `WatchRuleReconciler` / `ClusterWatchRuleReconciler` watch `GitTarget` and `GitProvider`, populate the RuleStore, and trigger the rule-change reconcile. - `GitProviderReconciler` validates reachability and manages the signing key lifecycle. @@ -1662,11 +1681,11 @@ as one funnel and its loss paths as one selector: [Interpreting metrics](interpreting-metrics.md#reading-git_fetches_total). **Apply requests.** `apply_requests_total{applier_kind,reason,outcome}` counts every -attempt the worker decided to send (`applier_kind` is `kustomization`; `reason` is -`ForeignCommit`, `Refusal` or `CatchUp`), with `outcome` separating +attempt the worker decided to send (`applier_kind` is `kustomization` or `application`; `reason` is +`ForeignCommit`, `Refusal`, `Repair` or `CatchUp`), with `outcome` separating `sent` from `failed` (the worker decides the retry), `dropped` (no target may still ask for that reason) and `stale` (the decision no longer held at the patch). It counts attempts, not -obligations. +the requests still due. `receiver_deliveries_total{type,outcome}` counts receiver deliveries: `notified`, `unmatched`, `ignored` (a ping, a tag, another event type), `rejected` (bad signature) and `invalid`. The receiver cannot tell a foreign push from an own one; the worker decides that from the @@ -1741,12 +1760,13 @@ Current limitations: is refused with more than one replica, and it loses every unjoined fact on restart by design; a multi-replica install must use the Redis transport. - **No pull request creation;** the operator writes directly to branches. -- **GitOps Reverser is the fast path to a deployment.** With `spec.applier` set, the applier's own - triggers are off, so while the operator is down a Git-side push reaches the cluster only through a - manual sync, `flux reconcile`, or Flux's long backstop interval. +- **GitOps Reverser is the only path to a deployment.** With an apply trigger, the applier applies + nothing on its own (a `Kustomization` is held suspended, an `Application` has automated sync + off), so while the operator is down a Git-side push reaches the cluster only through a manual + sync or `flux resume`, which the operator suspends again when it starts. - **Only Flux and Argo CD can be appliers yet,** and an Argo CD `Application` only with automated sync off: one that syncs on its own is refused, not driven. The applier may live in any cluster a - `ClusterProvider` reaches (`spec.applier.clusterProviderRef`), such as a management cluster. + `ClusterProvider` reaches (the trigger's `spec.clusterProviderRef`), such as a management cluster. - **An apply request cannot reserve time.** An edit made while the requested apply runs, to a field the applier sets, is reverted on the cluster. Its value stays in Git, because the applier's write over it is not mirrored, and the next apply restores it; until then the cluster runs the older @@ -1768,7 +1788,7 @@ Current limitations: | Package | Role | |---|---| | [api/v1alpha3/](../api/v1alpha3/) | CRD types | -| [internal/applier/](../internal/applier/) | apply requests: the Flux client, the dispatcher that carries out each attempt, and the `ApplierConfigured` check | +| [internal/applier/](../internal/applier/) | apply requests: the Flux and Argo CD clients, the dispatcher that carries out each attempt, the trigger status writer, and the `Configured` check | | [cmd/](../cmd/) | operator entry point and server setup | | [internal/auditutil/](../internal/auditutil/) | audit identity, objectRef, and subresource helpers feeding attribution facts | | [internal/controller/](../internal/controller/) | Kubernetes reconcilers | diff --git a/docs/argocd.md b/docs/argocd.md deleted file mode 100644 index 2237be18f..000000000 --- a/docs/argocd.md +++ /dev/null @@ -1,442 +0,0 @@ -# Using GitOps Reverser with Argo CD - -When GitOps Reverser publishes a folder that an Argo CD `Application` syncs, GitOps Reverser takes -full control of when that `Application` syncs. The `Application` has automated sync switched off, -and every sync it runs is one GitOps Reverser started, pinned to a commit it checked. Argo CD still -renders, applies, prunes, runs hooks and reports health; it no longer decides when. This page is the -complete Argo CD setup: what to change on the `Application`, what to declare on the `GitTarget`, -where the Git host's webhook goes, how to check it works, and which tests prove each part. - -The [bi-directional guide](bi-directional.md) explains the model behind this for any applier, and -the [configuration reference](configuration.md#who-applies-this-folder-specapplier) has every field -and condition reason. [Using GitOps Reverser with Flux](flux.md) is the same setup for Flux. - -## What changes - -| Before | With GitOps Reverser | -| --- | --- | -| `spec.syncPolicy.automated` syncs every new commit | Automated sync is off: `automated.enabled: false`, or no `automated` block | -| A sync starts when Argo CD's refresh finds a new commit | A sync starts when GitOps Reverser sets the `Application`'s `operation`, pinned to a commit | -| `selfHeal` reverts live drift | Nothing reverts live drift by itself; GitOps Reverser publishes it to Git | -| The Git host's push webhook goes to Argo CD | It goes to GitOps Reverser's receiver too; Argo CD's may stay | -| Nothing ties a folder to its `Application` | The `GitTarget` names it in `spec.applier.applicationRef` | - -Argo CD's timed refresh and its `/api/webhook` can stay as they are. With automated sync off a -refresh only **compares**: it resolves the branch tip and reports `Synced` or `OutOfSync`, and -applies nothing. - -## Why automated sync must be off - -**Automated sync applies every new commit it finds, whenever its refresh finds it.** Each push -GitOps Reverser makes is a new commit, so two to three minutes after each push (Argo CD's -`timeout.reconciliation`, 120s plus jitter), or seconds after it when Argo CD's webhook is wired, -the whole `Application` is synced over whatever is pending at that moment. `selfHeal: false` only -skips a commit already synced; it does not stop the next one. That brings back the two races GitOps -Reverser exists to close: - -- **Its own pushes.** Two API edits a few seconds apart produce two commits. A sync of the first - commit after the second edit reverts that edit, and GitOps Reverser then publishes the revert as - a third commit. -- **Somebody else's push.** While an API edit is still in an open commit window, a Git-side push - arrives. Synced straight away, the push's tree reverts the edit, because the edit is not in Git - yet. GitOps Reverser instead closes the window, replays the edit onto the push, pushes the - result, and only then asks Argo CD to sync it. - -Both apply something correct, at the wrong moment. Only GitOps Reverser can see the edits that are -live and not yet pushed, so only it can tell when a sync is safe. It starts one only on a **quiet -branch**: no open commit window and no pending writes. - -An `Application` with automated sync on is reported `ApplierConfigured=False` with reason -`AutomatedSyncEnabled`, and GitOps Reverser never syncs it. It is a misconfiguration to fix, and -stalls the `GitTarget`: an `Application` that syncs on its own is outside GitOps Reverser's control. - -**There is no catch-up.** Flux needs one, because it applies on its own timers and on a restart -from whatever artifact it holds, and a stale artifact would revert edits already in Git. With -automated sync off, Argo CD applies nothing on its own, so nothing it holds can go stale. GitOps -Reverser's own commits are already live (they were read from the cluster), and after its next -refresh the `Application` reports them `Synced`. So `spec.applier.quietPeriod` is rejected with an -`applicationRef`. - -## Requirements - -Running bi-directional with Argo CD is a supported setup, with a narrow configuration. Each row is -checked by GitOps Reverser or is a condition of the guarantees on this page. - -| Requirement | Why | When it is not met | -| --- | --- | --- | -| Argo CD 3.1 or later | `spec.syncPolicy.automated.enabled` exists from 3.1.0. It is the only way to switch automated sync off and keep `prune` | Before 3.1 the `Application` schema has no `enabled`, so the API server drops it, the block reads as on, and the target reports `AutomatedSyncEnabled`. Remove the `automated` block instead, and accept that a file deleted in Git leaves its object running | -| Automated sync off | GitOps Reverser starts every sync, on a quiet branch | `AutomatedSyncEnabled`; the `Application` is never synced by GitOps Reverser | -| A single Git source: `spec.source`, on the `GitProvider`'s repository | A sync is pinned to one commit of one repository | `SourceMismatch` for `spec.sources`, `spec.sourceHydrator`, or another repository | -| `targetRevision` is the `GitTarget`'s branch (`main` or `refs/heads/main`) | The tip GitOps Reverser checks is the tip of that branch | `SourceMismatch` for `HEAD`, an empty `targetRevision`, a tag or a commit | -| The source renders the folder GitOps Reverser writes: `directory.recurse: true` for a plain directory, or a kustomization at the path | GitOps Reverser writes `////.yaml` | A non-recursive directory sees no files, and the sync applies nothing | -| Resource tracking by `annotation`, Argo CD's default | GitOps Reverser strips `argocd.argoproj.io/tracking-id` before writing; it cannot strip `app.kubernetes.io/instance`, a label people set on purpose | With `label` or `annotation+label`, Argo CD's tracking label reaches Git on every object | -| `get` and `update` on the `Application`, for the identity of the `ClusterProvider` that reaches it | A sync is an update of the `Application` | `Forbidden` | -| The Git host's push webhook goes to GitOps Reverser's receiver | A Git-side push reaches the cluster in seconds | It still arrives, by the periodic refresh (`git.refreshInterval`, `10m` by default) | -| Nothing but GitOps Reverser syncs the `Application` while GitOps Reverser runs | See [what not to do](#what-not-to-do) | Not checked: an unchecked sync can revert live edits | - -## A complete example - -One folder, `apps/checkout` on branch `main` of `github.com/acme/platform`, published by GitOps -Reverser and synced by the `Application` `argocd/checkout`. Every object below is checked by a unit -test against the code that validates and drives the applier (see -[what the tests prove](#what-the-tests-prove)), so this example cannot drift from what is accepted. - -The Argo CD side. The `Application` follows exactly the branch the `GitTarget` writes, and its -automated sync is switched off while keeping `prune` for the syncs GitOps Reverser starts: - -```yaml -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: checkout - namespace: argocd -spec: - project: default - source: - repoURL: https://github.com/acme/platform - targetRevision: main - path: apps/checkout - directory: - recurse: true - destination: - server: https://kubernetes.default.svc - namespace: checkout - syncPolicy: - automated: - enabled: false - prune: true - syncOptions: - - ServerSideApply=true -``` - -The GitOps Reverser side. The `GitProvider` points at the same repository (any URL form of it) and -serves a push receiver; the `GitTarget` names the `Application` as its applier: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: platform-receiver - namespace: team-a -stringData: - token: replace-with-a-long-random-string ---- -apiVersion: configbutler.ai/v1alpha3 -kind: GitProvider -metadata: - name: platform - namespace: team-a -spec: - url: git@github.com:acme/platform.git - allowedBranches: ["main"] - secretRef: - name: platform-git - receiver: - type: github - secretRef: - name: platform-receiver ---- -apiVersion: configbutler.ai/v1alpha3 -kind: GitTarget -metadata: - name: checkout - namespace: team-a -spec: - gitProviderRef: - name: platform - branch: main - path: apps/checkout - applier: - applicationRef: - namespace: argocd - name: checkout - onRefusal: Reapply -``` - -`directory.recurse: true` matters: GitOps Reverser writes `////.yaml`, -so a non-recursive directory source sees nothing. - -## The `Application` - -`ApplierConfigured` is `True` only when the `Application` syncs the branch the `GitTarget` writes, -from a single source, with automated sync off. Each rule is checked when the target reconciles and -again before every sync, so an `Application` changed after the check is still never synced against -the rules. - -| Rule | Reported when broken | -| --- | --- | -| `spec.source.repoURL` is the `GitProvider`'s repository. The forms compare as one repository: `https://` and `ssh://`, `git@host:org/repo` (SCP style), a trailing `.git` or `/`, letter case, and the default port | `SourceMismatch`: `syncs , not ` | -| `spec.source.targetRevision` is the `GitTarget`'s `spec.branch`, as the name or as `refs/heads/`. `HEAD` (and an empty `targetRevision`, which means `HEAD`), a tag and a commit never follow the branch | `SourceMismatch`: `targets revision "HEAD", not branch main` | -| A single `spec.source`: no `spec.sources`, no `spec.sourceHydrator` | `SourceMismatch`: `syncs only a single-source Application` | -| Automated sync is off: `spec.syncPolicy.automated` is absent, or has `enabled: false`. This is Argo CD's own rule; an `automated` block with no `enabled` is on | `AutomatedSyncEnabled` | - -**Prune.** A sync GitOps Reverser starts prunes when `spec.syncPolicy.automated.prune` is `true`, -read from the `automated` block that `enabled: false` switches off, exactly as automated sync would -read it. Without an `automated` block, its syncs do not prune: a file removed from Git leaves its -object running. Keep the block with `enabled: false` and `prune: true` when a deletion in Git should -delete the live object. - -`selfHeal` means nothing with automated sync off, and is ignored. `syncOptions` and `retry` apply to -GitOps Reverser's syncs as they would to an automated one. - -## The `GitTarget` - -```yaml -spec: - applier: - applicationRef: - namespace: argocd # default: this GitTarget's namespace - name: checkout - clusterProviderRef: # default: this GitTarget's spec.clusterProviderRef - name: management -``` - -`clusterProviderRef` is for a hub Argo CD, which keeps its `Application`s in a management cluster and -deploys to another. Name the `ClusterProvider` of the cluster the `Application` lives in; the -`GitTarget`'s own `spec.clusterProviderRef` stays the cluster it mirrors. Leave it out when Argo CD -runs in the source cluster. - -`spec.applier` names exactly one of `kustomizationRef` and `applicationRef`, and `quietPeriod` is -rejected with an `applicationRef`. - -The target reports `ApplierConfigured`, and `Ready` includes it: - -| Reason | Means | Readiness | -| --- | --- | --- | -| `Succeeded` | The `Application` exists, syncs this repository and branch, has automated sync off, and may be updated | ready | -| `ApplierNotFound` | No such `Application` in the applier's cluster, or Argo CD is not installed there | progressing: `Application`s are often created after the `GitTarget` | -| `SourceMismatch` | One of the [`Application` rules](#the-application) about its source is broken | stalled: needs a human | -| `AutomatedSyncEnabled` | Automated sync is on | stalled: needs a human | -| `Forbidden` | The applier cluster's identity may not update the `Application` | stalled: needs a human | -| `NamespaceNotAuthorized`, `ClusterProviderNotFound` | `spec.applier.clusterProviderRef` names a provider that does not admit this namespace, or does not exist | stalled: needs a human | - -## Permissions - -GitOps Reverser looks the `Application` up in the cluster of `spec.applier.clusterProviderRef`, by -default the **source cluster**, with the identity of that `ClusterProvider`. That identity needs: - -| API group | Resource | Verbs | -| --- | --- | --- | -| `argoproj.io` | `applications` | `get`, `update` | -| `authorization.k8s.io` | `selfsubjectaccessreviews` | `create`, to check the one above | - -A sync is started by an `update` of the `Application`, the way Argo CD's own automated sync starts -one. For the operator's own cluster, install the chart with `applierAccess.argocd: true`, or apply -[`config/rbac/argocd-applier-role.yaml`](../config/rbac/argocd-applier-role.yaml) and its binding. -For a remote cluster, grant the same to the identity in the `ClusterProvider`'s kubeconfig. A `Role` -in the `Application`'s namespace is enough when you would rather not grant cluster-wide. - -Argo CD's own RBAC does not apply: GitOps Reverser writes the `Application` through the Kubernetes -API, as `kubectl` would, not through `argocd-server`. - -## The Git host's webhook - -Point the Git host's push webhook at GitOps Reverser's receiver, so a Git-side push is synced in -seconds, after GitOps Reverser has replayed any pending edits onto it. The receiver is the same as -for Flux: [the Flux guide](flux.md#the-git-hosts-webhook) has the steps. - -Argo CD's own webhook into `/api/webhook` may stay. It only makes Argo CD refresh sooner, and with -automated sync off a refresh only compares. Without GitOps Reverser's receiver, a foreign push is -still found, later: by the next rejected push or by the periodic refresh (`git.refreshInterval`, -`10m`). - -## What GitOps Reverser asks, and when - -| Reason | When | Asked | -| --- | --- | --- | -| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. Pending edits are replayed onto it and pushed first | Every applier declared on the branch | -| `Refusal` | An edit was refused and the target has `onRefusal: Reapply` | The refusing target's `Application` | - -There is no `CatchUp` for an `Application` ([why](#why-automated-sync-must-be-off)), and every -request starts a sync: there is no "already current" for Argo CD. - -**A request sets the `Application`'s `operation` to the sync automated sync would start**, with -these differences: - -- `sync.revision` is **pinned to the commit GitOps Reverser checked**, so the sync applies exactly - that commit, not whatever the branch holds by then. -- `initiatedBy` is `{username: gitops-reverser, automated: true}`. Argo CD's UI and history show - the sync as GitOps Reverser's, and [sync windows](#sync-windows-that-block-automated-syncs) treat - it as an automated sync. -- `info` carries the request's reason, and a token that tells this sync's result from any other. -- A `retry` with `refresh: true` is copied without it. Argo CD's refresh-on-retry drops the pinned - revision and syncs the newest one, a commit GitOps Reverser never checked. - -`source`, `prune`, `syncOptions` and `retry` (`limit: 5` when the `Application` sets none) are taken -from the `Application`, as automated sync takes them. - -**A sync already running is never replaced.** Argo CD refuses a new operation while one is set, and -so does GitOps Reverser: it waits for it to end (at most 5 minutes), then reads the `Application` -again and decides from what it is then. The update carries the `resourceVersion` it read, so a sync -somebody starts in between makes it fail instead of being overwritten. A failed sync waiting for -Argo CD's retry still counts as running. - -Before it writes, GitOps Reverser checks that its decision still holds: the commit is still the -branch tip, and no edit is waiting to be published. When either changed, it writes nothing and -decides again at the next quiet moment. - -**Hooks run on every sync**, as they would on an automated one: every `ForeignCommit` asks every -applier on the branch, whether or not the push touched its folder. - -**An edit made while Argo CD syncs keeps its value.** Until the sync's operation completes -(`Succeeded`, `Failed` or `Error`, at most 5 minutes), GitOps Reverser treats it as running. An API -edit made in those seconds, to a field the `Application` sets, is reverted on the cluster by the -sync, because the commit it applies does not hold the edit yet. GitOps Reverser recognizes that -write by Argo CD's field manager (`argocd-controller`, for a client-side and a server-side apply -alike, or `argocd-application-controller` for a `Replace=true` sync) and does not mirror it: the -API's value stays in Git. Each such write is logged and counted in -`apply_window_collisions_total{outcome="withheld"}`. Unlike Flux, nothing applies the edit back by -itself, since there is no catch-up: the cluster runs the reverted value until the next sync. - -Only a write proven to be Argo CD's is withheld. When Argo CD and another manager wrote the object -in the same second, the write could be the edit itself, so it is mirrored as before and counted as -`ambiguous`. A delete is never withheld. - -The request runs off the branch worker, so a slow Argo CD never holds a publication. A request that -fails is decided again from scratch, after that `Application`'s own backoff, exactly as for -[Flux](flux.md#what-gitops-reverser-asks-and-when). - -## What not to do - -Each of these applies Git to the cluster without the check that nothing live is missing from it, -or makes GitOps Reverser mirror something that is not an edit. None is refused by the API; most -are invisible until an edit is lost. - -### Syncing by hand, from CI, or by rollback - -A sync somebody else starts applies the commit they chose, over whatever is pending at that moment. -A Sync button pressed while an edit sits in a commit window reverts the edit on the cluster, and -GitOps Reverser publishes the revert as Argo CD's commit. The same holds for `argocd app sync` in a -CI pipeline that runs on push: it is the Argo CD form of a Flux `Receiver`, and the push webhook into -GitOps Reverser replaces it. A sync in progress also holds GitOps Reverser's next request, for up to -5 minutes. - -`argocd app rollback` needs automated sync off, so the standard makes it possible. A rollback applies -an older commit over everything since, API edits included, and GitOps Reverser then publishes the -result as a new commit. To undo a change, revert it in Git instead, and GitOps Reverser asks for the -sync. - -A manual sync is the right tool only [while GitOps Reverser is down](#while-gitops-reverser-is-down). - -### Sync windows that block automated syncs - -GitOps Reverser's syncs carry `automated: true`, so an `AppProject` sync window treats them as -automated syncs. An active `deny` window blocks them, and so does an `allow` window outside its -schedule; `manualSync: true` on the window does not let them through. - -A blocked sync is not refused. Argo CD keeps the operation `Running`, with the message `Sync -operation blocked by sync window`, and runs it when the window opens, still pinned to the commit it -was asked for. By then the branch may hold newer commits from API edits, and the late sync reverts -them on the cluster, outside any apply window, so GitOps Reverser publishes the revert. Meanwhile -every new request waits behind the blocked operation and fails after 5 minutes. Keep sync windows -off the `AppProject`s of `Application`s that a `GitTarget` names. - -### Watching the objects your hooks create - -[Hooks](https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks/) run on every sync, so on -every `ForeignCommit` and `Refusal`, whether or not the push touched the folder. Argo CD's default -delete policy, `BeforeHookCreation`, deletes a hook's object and creates it again on each sync, and a -hook named with `generateName` is a new object every time. GitOps Reverser mirrors objects of the -kinds a `WatchRule` covers, hooks included, and with the default `spec.prune.mode` (`OnEvent`) it -mirrors their deletion too. A `WatchRule` covering a hook's kind (usually `Job`) in the namespace -it runs in therefore commits on every sync, and can remove the hook's own manifest from Git. -Keep hook kinds out of the folder's `WatchRule`s. - -### Switching automated sync back on - -There is no temporary automated sync. The moment `enabled` turns true, Argo CD syncs every new commit -on its own again, GitOps Reverser's included, and the next one can land over an unpublished edit. -GitOps Reverser stops syncing the `Application` and reports `AutomatedSyncEnabled`, but it cannot stop -Argo CD. Name another applier, or none, before switching it on. - -### Generating the `Application` from an `ApplicationSet` - -The `ApplicationSet` controller owns the spec of the `Application`s it generates, and by default -writes its template back over any change. Its template must carry `automated: {enabled: false}`; a hand edit to -the generated `Application` does not last. This setup is not covered by tests. - -### Expecting `selfHeal` or `ignoreDifferences` to protect an edit - -`selfHeal` does nothing with automated sync off; live drift is published to Git instead of -reverted. `ignoreDifferences` with `RespectIgnoreDifferences=true` still works on the syncs GitOps -Reverser starts, and hands the field to the API side in both directions: a later Git edit to it does -not reach an existing object. That is [split ownership](bi-directional.md#the-split-ownership-alternative), -not bi-directional editing. - -### Relying on `Replace=true` - -A `Replace=true` sync, by sync option or resource annotation, creates and replaces objects without a -field manager, so their writes carry `argocd-application-controller`. GitOps Reverser counts that as -Argo CD's in the apply window, but no test measures it yet. The default apply, client-side or -`ServerSideApply=true`, writes as `argocd-controller` and is what the end-to-end tests cover. - -### Expecting a retry to pick up a newer commit - -A `syncPolicy.retry` with `refresh: true` makes each retry sync the newest revision. GitOps Reverser -copies the retry without `refresh`, so a failed sync retries the commit it was asked for. A newer -commit gets its own request once the branch is quiet. - -## Reverting refused edits - -With `onRefusal: Reapply`, an edit GitOps Reverser cannot write (one that has no home in the folder, -such as a field a kustomize base owns) is reverted by asking the target's `Application` to sync the -tip, once the branch is quiet. Only that `Application` is asked. -[Reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal) has the -eligibility rules. - -## Checking that it works - -```bash -# The Application is found, syncs this repository and branch, has automated sync off, and may be updated. -kubectl get gittarget checkout -n team-a \ - -o jsonpath='{.status.conditions[?(@.type=="ApplierConfigured")]}{"\n"}' - -# What GitOps Reverser last asked for, and why. -kubectl get gittarget checkout -n team-a -o jsonpath='{.status.applier}{"\n"}' - -# The last sync: who started it, which commit, and how it ended. A request asks, it never reports. -kubectl get application checkout -n argocd \ - -o jsonpath='{.status.operationState.operation.initiatedBy}{" "}{.status.operationState.syncResult.revision}{" "}{.status.operationState.phase}{"\n"}' -# {"automated":true,"username":"gitops-reverser"} 4f2c1ab9... Succeeded -``` - -`gitopsreverser_apply_requests_total{applier_kind="application",reason,outcome}` counts the -requests; [interpreting metrics](interpreting-metrics.md#apply-requests-and-the-push-receiver) reads -it. - -## Troubleshooting - -| Symptom | Cause | Fix | -| --- | --- | --- | -| `ApplierConfigured=False`, `AutomatedSyncEnabled` | `spec.syncPolicy.automated` is present without `enabled: false` | Set `automated.enabled: false` (keep `prune`), or remove the block | -| `ApplierConfigured=False`, `ApplierNotFound` | The `Application` does not exist in the applier's cluster under that namespace and name, or Argo CD is not installed there | Create it, or fix `applicationRef`; an omitted namespace is the `GitTarget`'s | -| `ApplierConfigured=False`, `SourceMismatch` | The message names the broken [`Application` rule](#the-application) | Fix the `Application`; a request it was holding is delivered afterward | -| `ApplierConfigured=False`, `Forbidden` | The identity lacks one of the [permissions](#permissions) | Grant it, or set `applierAccess.argocd: true` | -| A Warning `ApplyRequestFailed` Event | A request could not be delivered; the message says why, such as a sync somebody else started that did not end within 5 minutes | GitOps Reverser keeps asking on a quiet branch, at most every five minutes | -| A file deleted in Git leaves its object running | The `Application` has no `automated.prune: true` | Add `automated: {enabled: false, prune: true}` | -| The last sync stays `Running`: `Sync operation blocked by sync window` | A sync window blocks automated syncs, GitOps Reverser's included | Remove the window from the `AppProject`; see [sync windows](#sync-windows-that-block-automated-syncs) | -| A commit appears on every sync, adding or removing a `Job` | A `WatchRule` covers the objects a hook creates | Keep hook kinds out of the `WatchRule`s; see [hooks](#watching-the-objects-your-hooks-create) | - -## While GitOps Reverser is down - -GitOps Reverser becomes the only path to a deployment. While it is down, a Git-side push reaches -the cluster through a manual sync (`argocd app sync`, or the UI), which is safe then because nothing -is publishing. When it starts again, its first look at the remote owes no `ForeignCommit`: a sync on -startup could revert edits made while it was down, before its watches have caught them. - -## What the tests prove - -| Covered | Where | -| --- | --- | -| The [complete example](#a-complete-example) decodes strictly, reports `ApplierConfigured=True`, and its request sets a sync pinned to the commit | [`argocd_guide_test.go`](../internal/applier/argocd_guide_test.go) | -| The operation is the one automated sync builds, but for the pinned revision, the initiator and a pinned retry; a running sync is waited for and never replaced; a sync started between the read and the update makes it fail; every `ApplierConfigured` reason; when a sync counts as finished | [`argocd_test.go`](../internal/applier/argocd_test.go) | -| An `Application` is never owed a `CatchUp`, and is asked on a `ForeignCommit` and a `Refusal`; Argo CD's late write over a newer edit is withheld, by its own field managers | [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | -| Two API edits seconds apart keep the later value, with no further commit | [Bi-directional (Argo CD)](../test/e2e/argocd_bi_directional_e2e_test.go) | -| A Git-side push is synced by a `ForeignCommit` request, pinned to that commit | Bi-directional (Argo CD) | -| A refused edit with `onRefusal: Reapply` is reverted by an Argo CD sync | Bi-directional (Argo CD) | -| An `Application` with automated sync on is `AutomatedSyncEnabled` and is never synced | Bi-directional (Argo CD) | -| Argo CD's bookkeeping never reaches Git, asserted as zero commits during a sync | Bi-directional (Argo CD) | - -`task test-e2e-bi-directional` runs the Argo CD specs. Not covered end to end yet: a hub Argo CD in -another cluster, `Forbidden` against cluster RBAC, and a `Replace=true` sync's field manager, which -is read from Argo CD's source rather than measured. diff --git a/docs/bi-directional-argocd.md b/docs/bi-directional-argocd.md new file mode 100644 index 000000000..5b31fca2d --- /dev/null +++ b/docs/bi-directional-argocd.md @@ -0,0 +1,476 @@ +# Bi-directional GitOps with Argo CD + +This page is for a folder that GitOps Reverser writes and an Argo CD `Application` syncs. +`Application`s that sync other folders need nothing from it, and keep automated sync, self-heal and +everything else exactly as they are. [Bi-directional GitOps](bi-directional.md) explains the model; +this page is the complete Argo CD setup: what to change, what to declare, how to check it works, +and which tests prove each part. + +The `Application` runs with automated sync **off**, and every sync it runs is one GitOps Reverser +started, pinned to a commit it checked. Argo CD still renders, applies, prunes, runs hooks and +reports health; it no longer decides when. + +## What changes + +| Before | With GitOps Reverser | +| --- | --- | +| `spec.syncPolicy.automated` syncs every new commit | Automated sync is off: `automated.enabled: false`, or no `automated` block | +| A sync starts when Argo CD's refresh finds a new commit | A sync starts when GitOps Reverser sets the `Application`'s `operation`, pinned to a commit | +| `selfHeal` reverts live drift | Nothing reverts live drift by itself; GitOps Reverser publishes it to Git | +| The Git host's push webhook goes to Argo CD | It also goes to GitOps Reverser's receiver; Argo CD's may stay | +| Nothing ties a folder to its `Application` | An `ArgoCDTrigger` holds it, and the `GitTarget` names the trigger in `spec.applyTriggerRef` | + +Argo CD's timed refresh and its `/api/webhook` can stay as they are. With automated sync off, a +refresh only **compares**: it reports `Synced` or `OutOfSync`, and applies nothing. + +## Why automated sync must be off + +Automated sync applies every new commit it finds, and each push GitOps Reverser makes is a new +commit. Two to three minutes after each push (Argo CD's `timeout.reconciliation`, 120s plus +jitter), or seconds after it when Argo CD's webhook is wired, the whole `Application` is synced over +whatever is pending at that moment ([why that matters](bi-directional.md#why-the-applier-cannot-decide-for-itself)). +`selfHeal: false` only skips a commit already synced; it does not stop the next one. With +`selfHeal: true`, a live edit is reverted within a second, before it can reach Git at all. The +[support-boundary record](design/support-boundary/argocd-bi-directional.md) has Argo CD's code paths +for each of these, and for the settings that look like a fix and are not. + +An `Application` with automated sync on makes its trigger report `Configured=False` with +`AutomatedSyncEnabled`, and GitOps Reverser never syncs it. + +**There is no catch-up.** With automated sync off, Argo CD applies nothing on its own, so nothing it +holds goes stale. GitOps Reverser's own commits are already live, and after the next refresh the +`Application` reports them `Synced`. So an `ArgoCDTrigger` has no `quietPeriod`. A commit that +reached the branch before a `GitTarget` named the trigger is synced with the next push, refusal or +repair ([one sync](#one-sync)). + +## Requirements + +| Requirement | Why | When it is not met | +| --- | --- | --- | +| Argo CD 3.1 or later | `spec.syncPolicy.automated.enabled` exists from 3.1.0, and is the only way to switch automated sync off and keep `prune` | Before 3.1 the API server drops `enabled`, the block reads as on, and the trigger reports `AutomatedSyncEnabled`. Remove the `automated` block instead, and accept that a file deleted in Git leaves its object running | +| Automated sync off | GitOps Reverser starts every sync, on a quiet branch | `AutomatedSyncEnabled`; the `Application` is never synced by GitOps Reverser | +| A single Git source, `spec.source`, on the `GitProvider`'s repository | A sync is pinned to one commit of one repository | `UnsupportedSource` for `spec.sources` or `spec.sourceHydrator`; `SourceMismatch` for another repository | +| `targetRevision` is the `GitTarget`'s branch (`main` or `refs/heads/main`) | The tip GitOps Reverser checks is the tip of that branch | `SourceMismatch` for `HEAD`, an empty `targetRevision`, a tag or a commit | +| The source renders the folder GitOps Reverser writes: `directory.recurse: true` for a plain directory, or a kustomization at the path | GitOps Reverser writes `////.yaml` | A non-recursive directory sees no files, and the sync applies nothing | +| Resource tracking by `annotation`, Argo CD's default | GitOps Reverser strips `argocd.argoproj.io/tracking-id`; it cannot strip `app.kubernetes.io/instance`, a label people set on purpose | With `label` or `annotation+label`, the tracking label reaches Git on every object | +| `get` and `update` on the `Application`, for the identity of the `ClusterProvider` that reaches it | A sync is an update of the `Application` | `Forbidden` | +| Nothing but GitOps Reverser syncs the `Application` while GitOps Reverser runs | See [what not to do](#what-not-to-do) | Not checked: an unchecked sync can revert live edits | + +## A complete example + +One folder, `apps/checkout` on branch `main` of `github.com/acme/platform`, published by GitOps +Reverser and synced by the `Application` `argocd/checkout`. Every object below is checked by a unit +test against the code that validates and drives the applier (see +[what the tests prove](#what-the-tests-prove)), so this example cannot drift from what is accepted. + +The Argo CD side. The `Application` follows exactly the branch the `GitTarget` writes, and its +automated sync is switched off while keeping `prune` for the syncs GitOps Reverser starts: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: checkout + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/acme/platform + targetRevision: main + path: apps/checkout + directory: + recurse: true + destination: + server: https://kubernetes.default.svc + namespace: checkout + syncPolicy: + automated: + enabled: false + prune: true + syncOptions: + - ServerSideApply=true +``` + +The GitOps Reverser side. The `GitProvider` points at the same repository (any URL form of it) and +serves a push receiver; an `ArgoCDTrigger` holds the `Application`, in the cluster the `default` +`ClusterProvider` names, and the `GitTarget` names the trigger. The `Application` is in `argocd` +and the trigger in `team-a`, so the provider lets `team-a`'s triggers reach another namespace +(`allowCrossNamespace.appliers`); with the Helm chart, set +`clusterProvider.default.allowCrossNamespace.appliers: true` instead of declaring it: + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: default +spec: + accessFrom: + names: [team-a] + allowCrossNamespace: + appliers: true +--- +apiVersion: v1 +kind: Secret +metadata: + name: platform-receiver + namespace: team-a +stringData: + token: replace-with-a-long-random-string +--- +apiVersion: configbutler.ai/v1alpha3 +kind: GitProvider +metadata: + name: platform + namespace: team-a +spec: + url: git@github.com:acme/platform.git + allowedBranches: ["main"] + secretRef: + name: platform-git + receiver: + type: github + secretRef: + name: platform-receiver +--- +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: checkout + namespace: team-a +spec: + gitProviderRef: + name: platform + branch: main + path: apps/checkout + applyTriggerRef: + kind: ArgoCDTrigger + name: checkout + onRefusal: Reapply +--- +apiVersion: configbutler.ai/v1alpha3 +kind: ArgoCDTrigger +metadata: + name: checkout + namespace: team-a +spec: + clusterProviderRef: + name: default + applicationRef: + namespace: argocd + name: checkout +``` + +Point the Git host's push webhook at GitOps Reverser's receiver +([the steps](bi-directional.md#telling-gitops-reverser-that-somebody-else-pushed)), so a Git-side +push is synced in seconds, after any pending edits are replayed onto it. + +## The `Application` + +The `GitTarget`'s `ApplyTriggerReady` is `True` only when the `Application` syncs the branch the +`GitTarget` writes, from a single source, with automated sync off. The trigger checks these when it +reconciles, for every `GitTarget` naming it, and acts only for one that matches. All of them are +checked again before every sync, so an `Application` changed after the check is still never synced +against them. + +| Rule | Reported when broken | +| --- | --- | +| `spec.source.repoURL` is the `GitProvider`'s repository. The forms compare as one repository: `https://` and `ssh://`, `git@host:org/repo` (SCP style), a trailing `.git` or `/`, letter case, and the default port | `SourceMismatch`: `the applier of ArgoCDTrigger checkout applies , not `, on the `GitTarget`, and on the trigger when no `GitTarget` naming it matches | +| `spec.source.targetRevision` is the `GitTarget`'s `spec.branch`, as the name or as `refs/heads/`. `HEAD` (and an empty `targetRevision`), a tag and a commit never follow the branch | `SourceMismatch`: `the applier of ArgoCDTrigger checkout applies branch "HEAD", not main`, on the same objects | +| A single `spec.source`: no `spec.sources`, no `spec.sourceHydrator` | trigger `UnsupportedSource`: `syncs only a single-source Application` | +| Automated sync is off: `spec.syncPolicy.automated` is absent, or has `enabled: false`. This is Argo CD's own rule; an `automated` block with no `enabled` is on | trigger `AutomatedSyncEnabled` | + +**Prune.** A sync GitOps Reverser starts prunes when `spec.syncPolicy.automated.prune` is `true`, +read from the `automated` block that `enabled: false` switches off, exactly as automated sync reads +it. Without an `automated` block, its syncs do not prune: a file removed from Git leaves its object +running. `selfHeal` means nothing with automated sync off. `syncOptions` and `retry` apply as they +would to an automated sync. + +## The `ArgoCDTrigger` + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: ArgoCDTrigger +metadata: + name: checkout + namespace: team-a # the GitTargets naming it are in this namespace +spec: + clusterProviderRef: # the cluster the Application lives in; default: "default" + name: management + applicationRef: + namespace: argocd # default: the trigger's namespace + name: checkout +``` + +`clusterProviderRef` names the cluster that holds the `Application`: the operator's own cluster for +`default`, or a management cluster for a hub Argo CD that deploys to other clusters. The +`GitTarget`'s own `spec.clusterProviderRef` stays the cluster it mirrors. An `Application` in +another namespace than the trigger's, such as `argocd`, needs `allowCrossNamespace.appliers: true` on +that `ClusterProvider`; without it the trigger reports `ApplierNamespaceNotAllowed`. That is a +delegation, not owner consent: every namespace the provider admits may then sync any `Application` +in reach of its identity whose source is the repository its `GitProvider` names, and nothing proves +it owns the `Application` or may write to the repository. See +[what a namespace may reach outside itself](configuration.md#what-a-namespace-may-reach-outside-itself-specallowcrossnamespace). +Both references are immutable: delete and recreate the trigger to hold another `Application`. The +trigger writes nothing to the `Application` until it syncs it, so deleting it, or no longer naming +it, needs no hand-back: switch automated sync on afterwards if the `Application` should sync on its +own again. + +### What state the `Application` is in + +```mermaid +stateDiagram-v2 + [*] --> OnItsOwn + OnItsOwn --> Idle: automated sync off, trigger Configured + Idle --> SyncRequested: GitOps Reverser sets operation, pinned to the commit + SyncRequested --> Syncing: Argo CD starts the operation + Syncing --> Idle: Succeeded, Failed or Error + Idle --> Syncing: somebody else syncs (Sync button, argocd app sync) + Idle --> OnItsOwn: automated sync switched on (AutomatedSyncEnabled) + + note right of OnItsOwn + Argo CD syncs on its own, + as without GitOps Reverser + end note + note right of Idle + A refresh only compares; + nothing applies + end note +``` + +A trigger acts only while an eligible `GitTarget` references it: one in the trigger's namespace that +names it, is admitted by its `ClusterProvider`s, and writes the repository and branch the +`Application` syncs. A trigger nothing eligible references starts no sync and is not in line for +the `Application`; one no `GitTarget` names also reads nothing in the `Application`'s cluster. An +`Application` has nothing to hold, so there is nothing to hand back either. + +The trigger reports `Configured`, and its `Ready` includes it: + +| Reason | Means | Readiness | +| --- | --- | --- | +| `Succeeded` | An eligible `GitTarget` names the trigger, and the `Application` exists, syncs a single source, has automated sync off, and may be updated | ready | +| `NotReferenced` (`Unknown`) | No `GitTarget` names the trigger. Nothing is read or written | ready: `Ready=True`, `NotReferenced` | +| `NotBound` (`Unknown`) | No `GitTarget` naming the trigger is eligible yet, and one cannot be judged yet | progressing | +| `SourceMismatch`, `NamespaceNotAuthorized` or `NotBound` (`False`) | Every `GitTarget` naming the trigger is refused: it writes another repository or branch, or is not admitted. The message lists each | stalled | +| `ApplierNotFound` | No such `Application` in the applier's cluster, or Argo CD is not installed there | progressing: `Application`s are often created after the trigger | +| `UnsupportedSource` | The `Application` has several sources, a source hydrator, or no source | stalled | +| `AutomatedSyncEnabled` | Automated sync is on | stalled | +| `Forbidden` | The applier cluster's identity may not update the `Application` | stalled | +| `NamespaceNotAuthorized` | The `clusterProviderRef` names a provider whose `accessFrom` does not admit the trigger's namespace | stalled | +| `ClusterProviderNotFound` | The `clusterProviderRef` names a provider that does not exist | progressing | +| `ApplierNamespaceNotAllowed` | The `Application` is in another namespace than the trigger's, and the `ClusterProvider` does not set `allowCrossNamespace.appliers` | stalled | +| `HeldByAnotherTrigger` | Another `ArgoCDTrigger` in line for the same `Application` in the same cluster, one an eligible `GitTarget` references, was created first and holds it; this one starts no sync, and takes over once that one is deleted or no longer referenced | stalled: delete one of the two | + +A sync that has not completed within 5 minutes also makes the trigger `Ready=False` with +`OperationUnresolved`, a progressing state that clears when the sync completes. + +The `GitTarget` reports `ApplyTriggerReady`: `True` when the trigger is `Ready` and its +`Application` syncs this repository and branch, `SourceMismatch` when it syncs another, +`ApplyTriggerNotFound` when the trigger does not exist, `Unknown` (`NotReferenced`) until the +trigger has seen that the `GitTarget` names it, and otherwise the trigger's own `Ready` reason. + +## Permissions + +GitOps Reverser looks the `Application` up in the cluster of the trigger's `clusterProviderRef`, +with the identity of that `ClusterProvider`. That identity needs: + +| API group | Resource | Verbs | +| --- | --- | --- | +| `argoproj.io` | `applications` | `get`, `update` | +| `authorization.k8s.io` | `selfsubjectaccessreviews` | `create`, to check the one above | + +A sync is started by an `update` of the `Application`, the way Argo CD's own automated sync starts +one. For the operator's own cluster, install the chart with `applierAccess.argocd: true`, or apply +[`config/rbac/argocd-applier-role.yaml`](../config/rbac/argocd-applier-role.yaml) and its binding. +For a remote cluster, grant the same to the identity in the `ClusterProvider`'s kubeconfig. A `Role` +in the `Application`'s namespace is enough. Argo CD's own RBAC does not apply: GitOps Reverser +writes the `Application` through the Kubernetes API, as `kubectl` would, not through `argocd-server`. + +## One sync + +`ForeignCommit`, `Refusal` and `Repair` reach an `Application` +([the reasons](bi-directional.md#when-gitops-reverser-asks-for-an-apply)); `CatchUp` never does. +Every request starts a sync: there is no "already current" for Argo CD. So naming an `ArgoCDTrigger` +from a `GitTarget` syncs nothing by itself: a commit already on the branch reaches the +`Application` with the next push, refusal or repair, or a manual sync. + +**A request sets the `Application`'s `operation` to the sync automated sync would start**, with +these differences: + +- `sync.revision` is **pinned to the commit GitOps Reverser checked**, so the sync applies exactly + that commit, not whatever the branch holds by then. +- `initiatedBy` is `{username: gitops-reverser, automated: true}`. Argo CD's UI and history show the + sync as GitOps Reverser's, and [sync windows](#sync-windows-that-block-automated-syncs) treat it + as an automated sync. +- `info` carries the request's reason, and a token that tells this sync's result from any other. +- A `retry` with `refresh: true` is copied without it, because Argo CD's refresh-on-retry drops the + pinned revision and syncs the newest one. + +`source`, `prune`, `syncOptions` and `retry` (`limit: 5` when the `Application` sets none) are taken +from the `Application`. + +```mermaid +stateDiagram-v2 + [*] --> Requested: update sets operation, guarded by resourceVersion + Requested --> Running: Argo CD moved it into operationState + Running --> Succeeded: operationState Succeeded + Running --> Failed: operationState Failed or Error + Requested --> Unanswered: the update got no answer, or the operator restarted + Unanswered --> Requested: the token turns up + Unanswered --> Failed: a guarded refresh write lands first, so the update never can + Running --> Unresolved: not completed in 5 minutes + Unresolved --> Succeeded + Unresolved --> Failed: operationState Failed or Error, or .operation removed by hand + Failed --> [*]: asked again from scratch, after the backoff + Succeeded --> [*] +``` + +`Unanswered` is not shown in `status.operation`, which stays `Requested`: GitOps Reverser writes the +`Application` itself, asking Argo CD for a refresh, under the `resourceVersion` the lost update was +made from. Once that write lands, the lost update can never land, so the operation is `Failed` and +asked for again. No timing assumption about the API server is made. + +**A sync already running is never replaced.** Argo CD refuses a new operation while one is set, and +so does GitOps Reverser: it waits for it to end (at most 5 minutes), then decides again from what +the `Application` is then. The update carries the `resourceVersion` it read, so a sync somebody +starts in between makes it fail instead of being overwritten. A failed sync waiting for Argo CD's +retry still counts as running: Argo CD keeps its `operation` set until it completed. + +**Hooks run on every sync**, as they would on an automated one: every `ForeignCommit` asks every +applier on the branch, whether or not the push touched its folder. + +**The apply window.** Until the sync's operation completes, however long that takes, an API edit to +a field the `Application` sets can be reverted on the cluster by the sync. GitOps Reverser recognizes +that write by Argo CD's field manager (`argocd-controller`, for a client-side and a server-side +apply alike, or `argocd-application-controller` for a `Replace=true` sync), withholds it from Git, +and asks for a `Repair` sync pinned to the tip that holds the edit, but only when Argo CD's entry is +newer than the edit: undoing an edit can leave an older entry of Argo CD's as the newest, and that +undo is mirrored. When Argo CD and another manager, or the edit, wrote in the same second, the write +is mirrored and counted as `ambiguous`. + +A request that fails, and a sync that ends `Failed` or `Error`, is decided again from scratch, after +that `Application`'s own backoff. A sync still running after 5 minutes is `Unresolved`: GitOps +Reverser keeps following it, records an `ApplyUnresolved` Event, and asks the `Application` nothing +more until it completes. Removing `.operation` by hand, the usual way out of a stuck hook, leaves +the operation state `Running` for good, since Argo CD processes an operation only while +`.operation` is set: once `Unresolved`, such a sync of GitOps Reverser's is `Failed` and asked again, +and a request that finds one somebody else started, unchanged for its whole wait, syncs over it. + +**Reverting refused edits.** With `onRefusal: Reapply`, the target's `Application` alone is asked to +sync the tip. + +## What not to do + +Each of these applies Git to the cluster without the check that nothing live is missing from it, or +makes GitOps Reverser mirror something that is not an edit. None is refused by the API; most are +invisible until an edit is lost. + +### Syncing by hand, from CI, or by rollback + +A sync somebody else starts applies the commit they chose, over whatever is pending at that moment, +and GitOps Reverser publishes the revert as Argo CD's commit. The same holds for `argocd app sync` +in a CI pipeline that runs on push: the push webhook into GitOps Reverser replaces it. A sync in +progress also holds GitOps Reverser's next request, for up to 5 minutes. + +`argocd app rollback` applies an older commit over everything since, API edits included, and GitOps +Reverser then publishes the result as a new commit. To undo a change, revert it in Git instead. + +A manual sync is the right tool only [while GitOps Reverser is down](#while-gitops-reverser-is-down). + +### Sync windows that block automated syncs + +GitOps Reverser's syncs carry `automated: true`, so an `AppProject` sync window treats them as +automated syncs: an active `deny` window blocks them, and so does an `allow` window outside its +schedule; `manualSync: true` on the window does not let them through. A blocked sync stays +`Running` (`Sync operation blocked by sync window`) and runs when the window opens, still pinned to +the commit it was asked for, so it reverts any newer API edits outside any apply window. Keep sync +windows off the `AppProject`s of `Application`s that a `GitTarget` names. + +### Watching the objects your hooks create + +[Hooks](https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks/) run on every sync. Argo +CD's default delete policy, `BeforeHookCreation`, deletes a hook's object and creates it again on +each sync, and a hook named with `generateName` is a new object every time. A `WatchRule` covering a +hook's kind (usually `Job`) in the namespace it runs in therefore commits on every sync, and with +the default `spec.prune.mode` can remove the hook's own manifest from Git. Keep hook kinds out of +the folder's `WatchRule`s. + +### Switching automated sync back on + +The moment `enabled` turns true, Argo CD syncs every new commit on its own again, GitOps Reverser's +included. GitOps Reverser stops syncing the `Application` and reports `AutomatedSyncEnabled`, but it +cannot stop Argo CD. Delete the trigger, or point the `GitTarget` at another, before switching it on. + +### Generating the `Application` from an `ApplicationSet` + +The `ApplicationSet` controller writes its template back over any change to the `Application`s it +generates, so the template must carry `automated: {enabled: false}`. This setup is not covered by +tests. + +### Relying on `Replace=true` + +A `Replace=true` sync writes as `argocd-application-controller`. GitOps Reverser counts that as +Argo CD's in the apply window, but no test measures it yet. The default apply, client-side or +`ServerSideApply=true`, writes as `argocd-controller` and is what the end-to-end tests cover. + +## Checking that it works + +```bash +# The Application is found, has automated sync off, and may be updated; and it syncs this target's branch. +kubectl get argocdtrigger checkout -n team-a +kubectl get gittarget checkout -n team-a \ + -o jsonpath='{.status.conditions[?(@.type=="ApplyTriggerReady")]}{"\n"}' + +# What GitOps Reverser last asked for, why, and where that sync stands. +kubectl get argocdtrigger checkout -n team-a -o jsonpath='{.status.operation}{"\n"}' + +# The last sync: who started it, which commit, and how it ended. A request asks, it never reports. +kubectl get application checkout -n argocd \ + -o jsonpath='{.status.operationState.operation.initiatedBy}{" "}{.status.operationState.syncResult.revision}{" "}{.status.operationState.phase}{"\n"}' +# {"automated":true,"username":"gitops-reverser"} 4f2c1ab9... Succeeded +``` + +`gitopsreverser_apply_requests_total{applier_kind="application",reason,outcome}` counts the +requests; [interpreting metrics](interpreting-metrics.md#apply-requests-and-the-push-receiver) reads +it. + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| Trigger `Configured=False`, `AutomatedSyncEnabled` | `spec.syncPolicy.automated` is present without `enabled: false` | Set `automated.enabled: false` (keep `prune`), or remove the block | +| Trigger `Configured=False`, `ApplierNotFound` | No `Application` with that namespace and name in the trigger's cluster, or Argo CD is not installed there | Create it, or recreate the trigger with the right `applicationRef`; an omitted namespace is the trigger's | +| `ApplyTriggerReady=False`, `SourceMismatch`, or trigger `UnsupportedSource` | The message names the broken [`Application` rule](#the-application) | Fix the `Application`; a sync that was waiting is asked for afterward | +| Trigger `Configured=False`, `ApplierNamespaceNotAllowed` | The `Application` is in another namespace than the trigger, such as `argocd` | Set `allowCrossNamespace.appliers: true` on the trigger's `ClusterProvider` (Helm: `clusterProvider.default.allowCrossNamespace.appliers`), or move the trigger and `GitTarget` to the `Application`'s namespace | +| Trigger `Configured=Unknown`, `NotBound` | A `GitTarget` names the trigger, but cannot be judged yet: its `GitProvider` or `ClusterProvider` does not exist | Create the missing object; the message names it | +| Trigger `Configured=False`, `Forbidden` | The identity lacks one of the [permissions](#permissions) | Grant it, or set `applierAccess.argocd: true` | +| A Warning `ApplyRequestFailed` Event | A request could not be delivered, such as a sync somebody else started that did not end within 5 minutes | GitOps Reverser keeps asking on a quiet branch, at most every five minutes | +| A Warning `ApplyFailed` Event | The sync ended `Failed` or `Error`; the message carries Argo CD's | Fix what the message names; GitOps Reverser asks again on a quiet branch | +| A Warning `ApplyUnresolved` Event, trigger `OperationUnresolved` | The sync has not completed in 5 minutes: hooks, waves, or Argo CD's own retries | Nothing is asked of the `Application` until it completes; look at its operation state | +| A file deleted in Git leaves its object running | The `Application` has no `automated.prune: true` | Add `automated: {enabled: false, prune: true}` | +| The last sync stays `Running`: `Sync operation blocked by sync window` | A sync window blocks automated syncs | Remove the window from the `AppProject`; see [sync windows](#sync-windows-that-block-automated-syncs) | +| A commit appears on every sync, adding or removing a `Job` | A `WatchRule` covers the objects a hook creates | Keep hook kinds out of the `WatchRule`s; see [hooks](#watching-the-objects-your-hooks-create) | + +## While GitOps Reverser is down + +GitOps Reverser becomes the only path to a deployment. While it is down, a Git-side push reaches the +cluster through a manual sync (`argocd app sync`, or the UI), which is safe then because nothing is +publishing. When it starts again, its first look at the remote asks for no `ForeignCommit`: a sync +on startup could revert edits made while it was down, before its watches caught them. + +## What the tests prove + +| Covered | Where | +| --- | --- | +| The [complete example](#a-complete-example) decodes strictly, reports `Configured=True`, binds the `GitTarget`, and its request sets a sync pinned to the commit | [`argocd_guide_test.go`](../internal/applier/argocd_guide_test.go) | +| The operation is the one automated sync builds, but for the pinned revision, the initiator and a pinned retry; a running sync is waited for and never replaced; a sync started between the read and the update makes it fail; an unanswered update is settled by a guarded write; every `Configured` reason; when a sync counts as finished | [`argocd_test.go`](../internal/applier/argocd_test.go) | +| An `Application` is never asked for a `CatchUp`, not even once a `GitTarget` first names its trigger, and is asked on a `ForeignCommit` and a `Refusal`; Argo CD's late write over a newer edit is withheld, by its own field managers | [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | +| A trigger nothing names reads nothing and is `Ready`; the binding over several `GitTarget`s; `ApplierNamespaceNotAllowed`; a sync whose `.operation` was removed while it ran ends | [`applytrigger_rule_test.go`](../internal/controller/applytrigger_rule_test.go), [`argocd_test.go`](../internal/applier/argocd_test.go) | +| Two API edits seconds apart keep the later value, with no further commit | [Bi-directional (Argo CD)](../test/e2e/argocd_bi_directional_e2e_test.go) | +| A Git-side push is synced by a `ForeignCommit` request, pinned to that commit | Bi-directional (Argo CD) | +| A refused edit with `onRefusal: Reapply` is reverted by an Argo CD sync | Bi-directional (Argo CD) | +| An `Application` with automated sync on is `AutomatedSyncEnabled` and is never synced | Bi-directional (Argo CD) | +| Argo CD's bookkeeping never reaches Git, asserted as zero commits during a sync | Bi-directional (Argo CD) | + +`task test-e2e-bi-directional` runs the Argo CD specs. Not covered end to end yet: a hub Argo CD in +another cluster, `Forbidden` against cluster RBAC, and a `Replace=true` sync's field manager, which +is read from Argo CD's source rather than measured. diff --git a/docs/bi-directional-flux.md b/docs/bi-directional-flux.md new file mode 100644 index 000000000..b1b177101 --- /dev/null +++ b/docs/bi-directional-flux.md @@ -0,0 +1,523 @@ +# Bi-directional GitOps with Flux + +This page is for a folder that GitOps Reverser writes and a Flux `Kustomization` applies. +`Kustomization`s that apply other folders need nothing from it, and keep working as Flux always +does. [Bi-directional GitOps](bi-directional.md) explains the model; this page is the complete Flux +setup: what to change, what to declare, how to check it works, and which tests prove each part. + +A `FluxTrigger` holds the `Kustomization` **suspended** while it is idle, for as long as a +`GitTarget` that writes its folder names the trigger. For each apply, GitOps Reverser resumes it, +waits for that one reconcile to end, and suspends it again. Flux still fetches, renders, applies, +prunes and reports health; it never applies on its own. + +## What changes + +| Before | With GitOps Reverser | +| --- | --- | +| The `Kustomization` applies on its interval, on every new artifact, and on a controller restart | It is suspended, and applies only when GitOps Reverser resumes it for one apply | +| A failed apply is retried after `retryInterval` | It is suspended again after a failure; GitOps Reverser retries on a quiet branch | +| The Git host's push webhook goes to a Flux `Receiver` | It goes to GitOps Reverser's receiver; the Flux `Receiver` for this repository is removed | +| Nothing ties a folder to its `Kustomization` | A `FluxTrigger` holds it, and the `GitTarget` names the trigger in `spec.applyTriggerRef` | +| `flux get ks` shows it ready | It shows it `Suspended`, which is how it is held | + +## Why suspended, and not a long interval + +A long interval makes Flux apply less often; it does not make Flux wait. A `Kustomization` that is +not suspended still applies when its interval fires, when kustomize-controller restarts, when a +failed apply retries, when its own spec changes, and when its `GitRepository` gets a new artifact +for any reason, including a fetch another consumer asked for. Each of those can land while an edit +is still on its way to Git ([why that matters](bi-directional.md#why-the-applier-cannot-decide-for-itself)). +A suspended `Kustomization` does none of them: a reconcile reads `spec.suspend` first and returns +before it resolves its source. + +## A complete example + +One folder, `apps/checkout` on branch `main` of `github.com/acme/platform`, published by GitOps +Reverser and applied by the `Kustomization` `flux-system/checkout`. Every object below is checked by +a unit test against the code that validates and drives the applier (see +[what the tests prove](#what-the-tests-prove)), so this example cannot drift from what is accepted. + +The Flux side. The `GitRepository` follows exactly the branch the `GitTarget` writes. The +`Kustomization`'s manifest does **not** set `suspend`: the `FluxTrigger` holds that field. + +```yaml +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: platform + namespace: flux-system +spec: + url: https://github.com/acme/platform + ref: + branch: main + interval: 1h + secretRef: + name: platform-git +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: checkout + namespace: flux-system +spec: + sourceRef: + kind: GitRepository + name: platform + path: ./apps/checkout + prune: true + interval: 1h +``` + +The GitOps Reverser side. The `GitProvider` points at the same repository (any URL form of it) and +serves a push receiver; a `FluxTrigger` holds the `Kustomization`, in the cluster the `default` +`ClusterProvider` names, and the `GitTarget` names the trigger. The `Kustomization` is in +`flux-system` and the trigger in `team-a`, so the provider lets `team-a`'s triggers reach another +namespace (`allowCrossNamespace.appliers`); with the Helm chart, set +`clusterProvider.default.allowCrossNamespace.appliers: true` instead of declaring it: + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: ClusterProvider +metadata: + name: default +spec: + accessFrom: + names: [team-a] + allowCrossNamespace: + appliers: true +--- +apiVersion: v1 +kind: Secret +metadata: + name: platform-receiver + namespace: team-a +stringData: + token: replace-with-a-long-random-string +--- +apiVersion: configbutler.ai/v1alpha3 +kind: GitProvider +metadata: + name: platform + namespace: team-a +spec: + url: git@github.com:acme/platform.git + allowedBranches: ["main"] + secretRef: + name: platform-git + receiver: + type: github + secretRef: + name: platform-receiver +--- +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: checkout + namespace: team-a +spec: + gitProviderRef: + name: platform + branch: main + path: apps/checkout + applyTriggerRef: + kind: FluxTrigger + name: checkout + onRefusal: Reapply +--- +apiVersion: configbutler.ai/v1alpha3 +kind: FluxTrigger +metadata: + name: checkout + namespace: team-a +spec: + clusterProviderRef: + name: default + kustomizationRef: + namespace: flux-system + name: checkout + quietPeriod: 30s +``` + +No Flux `Receiver` for `acme/platform` appears, on purpose: remove yours if you have one, and point +the Git host's push webhook at GitOps Reverser's receiver instead +([the steps](bi-directional.md#telling-gitops-reverser-that-somebody-else-pushed)). + +## The `GitRepository` + +The `GitTarget`'s `ApplyTriggerReady` is `True` only when the `Kustomization`'s source fetches the +branch the `GitTarget` writes. The trigger checks the rules about the source when it reconciles, and +the repository and branch of every `GitTarget` naming it: it holds the `Kustomization` only for one +that matches. The `GitTarget` reports its own match against the source the trigger publishes. All +of them are checked again before every apply. + +| Rule | Reported when broken | +| --- | --- | +| `spec.url` is the `GitProvider`'s repository. The forms compare as one repository: `https://` and `ssh://`, `git@host:org/repo` (SCP style), a trailing `.git` or `/`, letter case, and the default port | `SourceMismatch`: `the applier of FluxTrigger checkout applies , not `, on the `GitTarget`, and on the trigger when no `GitTarget` naming it matches | +| `spec.ref.branch` is exactly the `GitTarget`'s `spec.branch` | `SourceMismatch`: `the applier of FluxTrigger checkout applies branch "release", not main`, on the same objects | +| `spec.ref` pins nothing stronger: no `commit`, `name`, `semver` or `tag`, even alongside the branch. Flux resolves those before the branch, so the artifact would never follow it | trigger `UnsupportedSource`: `pins spec.ref.tag "v1.0.0", so it never follows a branch` | +| `spec.ref.branch` is set | trigger `UnsupportedSource`: `names no spec.ref.branch` | + +A suspended `Kustomization` ignores a new artifact, so the source's own `interval` decides when it +fetches, never when anything applies. Keep it longer than an apply takes, `1h` in the example: an +apply reads the artifact again just before it resumes the `Kustomization`, and a fetch of the +source's own between that read and the reconcile makes Flux apply another commit than the one +GitOps Reverser checked. That is logged, counted in `apply_revision_mismatches_total`, and still +covered by the apply window, but it is avoidable. An apply of an *older* revision than the one +checked counts as failed, and GitOps Reverser asks again. + +**Shared sources.** Several `Kustomization`s may read one `GitRepository`. The trigger lists its +source's other consumers in `status.source.otherConsumers`: the `Kustomization`s no `FluxTrigger` +holds and the `HelmRelease`s whose chart comes from it, directly or through a `HelmChart` their +`chartRef` names. Each of them applies a new artifact the moment it arrives. The list is +information only and never affects `Ready`: one source for `apps` and `infra` is a normal setup. + +- While the list is empty and the trigger holds its own `Kustomization`, `status.source.prefetch` is + `true`: when GitOps Reverser learns of a push while the branch is busy, it has the source fetch at + once, so the artifact is ready when the apply is due. A consumer counts as held only while its + `FluxTrigger` reports `Held=True`, checked again on the cluster just before the fetch. +- Appliers on one branch that share a source apply one at a time. + +## The `Kustomization` + +- `spec.sourceRef` names a `GitRepository`. An `OCIRepository` or a `Bucket` makes the trigger + report `UnsupportedSource`: GitOps Reverser writes Git, and can only ask a Git source to fetch. +- **The manifest omits `spec.suspend`.** Whatever applies the `Kustomization` object (a parent + `Kustomization`, Helm, flux-operator) must leave the field alone, or it resumes the + `Kustomization` on its next apply. When another field manager applies `spec.suspend`, the trigger + reports `SuspendOwnedElsewhere`, naming the manager, and holds nothing. A one-off write such as + `flux resume` is no such owner; see [overriding by hand](#overriding-by-hand). +- `spec.interval` and `spec.retryInterval` decide nothing while it is suspended. +- `spec.wait`, health checks and `spec.timeout` lengthen each apply, and GitOps Reverser follows it, + with its apply window open, for as long as it runs. +- `spec.dependsOn`: a dependency whose apply failed is suspended with its generation unapplied, so + its dependents wait until GitOps Reverser's retry of it succeeds. +- It lives in the trigger's namespace unless the trigger's `ClusterProvider` sets + `allowCrossNamespace.appliers: true`; a trigger naming one elsewhere without it reports + `ApplierNamespaceNotAllowed`. A `Kustomization` in `flux-system` therefore needs the grant. It is + a delegation, not owner consent: every namespace the provider's `accessFrom` admits may then + suspend and apply any `Kustomization` in reach of the provider's identity whose source fetches the + repository its `GitProvider` names. Nothing proves that the namespace owns the `Kustomization`, or + may write to that repository: the binding compares URLs. See + [what a namespace may reach outside itself](configuration.md#what-a-namespace-may-reach-outside-itself-specallowcrossnamespace). +- Several `GitTarget`s may name one `FluxTrigger`, for a `Kustomization` whose `path` covers several + folders. It is applied once for all of them. +- Two `GitProvider`s may name one repository. The branch counts as quiet only when both of their + workers are. + +## The `FluxTrigger` + +```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: FluxTrigger +metadata: + name: checkout + namespace: team-a # the GitTargets naming it are in this namespace +spec: + clusterProviderRef: # the cluster Flux runs in; default: "default" + name: default + kustomizationRef: + namespace: flux-system # default: the trigger's namespace + name: checkout + quietPeriod: 30s # default 30s, at most 1h +``` + +`clusterProviderRef` names the cluster that holds the `Kustomization`: the operator's own cluster +for `default`, or a management cluster where Flux runs. Both references are immutable: delete and +recreate the trigger to hold another `Kustomization`. + +`quietPeriod` is how long the branch must stay quiet after GitOps Reverser's own pushes before the +`Kustomization` applies them (`CatchUp`), and before the first apply once a `GitTarget` names the +trigger. Every new write restarts it. + +### How the trigger holds the `Kustomization` + +A trigger acts only while an eligible `GitTarget` references it: one in the trigger's namespace that +names it in `spec.applyTriggerRef`, is admitted by its `ClusterProvider`s, and writes the repository +and branch the `Kustomization`'s `GitRepository` fetches. A trigger no `GitTarget` names holds +nothing, reads nothing in the `Kustomization`'s cluster, and is `Ready`. One that only ineligible +`GitTarget`s name holds nothing either, and its `Configured` says why. + +```mermaid +stateDiagram-v2 + [*] --> NotReferenced + NotReferenced --> NotConfigured: a GitTarget names it, not yet Configured + NotConfigured --> NotReferenced: no GitTarget names it any more + NotConfigured --> Suspending: Configured turns True, so it adopts + Suspending --> Suspended: the suspend is handled + Suspended --> Applying: an operation is recorded as Requested + Applying --> Suspended: the operation completed, its suspend handled + Applying --> Suspending: nobody followed it past the grace, the trigger suspends it + Suspended --> Suspending: found resumed (flux resume) or suspended by another (flux suspend) + Suspended --> NotConfigured: the applier's configuration turns invalid, left suspended + Suspended --> Releasing: no eligible reference left, or trigger deleted + Applying --> Releasing: the same, suspended first + Releasing --> Suspended: a reference returns before the release + Releasing --> NotReferenced: released, Flux applies on its own again + Releasing --> NotConfigured: released, while a refused GitTarget still names it + Releasing --> [*]: trigger deleted + + note right of NotConfigured + Configured says why + end note + note right of Applying + the only state in which Flux applies; + status.operation shows the apply + end note +``` + +- **Adopting.** Once an eligible `GitTarget` names it and it is `Configured`, the trigger puts its + finalizer `configbutler.ai/flux-trigger` in place, suspends the `Kustomization` with a server-side + apply under the field manager `gitops-reverser` and a new `reconcile.fluxcd.io/requestedAt` token, + and records an `Adopted` Event. The suspend is handled once the `Kustomization` reports that + token: reconciles of one object never overlap, so by then nothing that was running before is. +- **Several `GitTarget`s.** One eligible `GitTarget` among those naming the trigger is enough. While + none is, and one cannot be judged yet (its `GitProvider` is missing), the trigger reports + `NotBound` and waits. Once each is refused, it reports the reason they share (`SourceMismatch`, + `NamespaceNotAuthorized`), or `NotBound` for mixed ones, with every `GitTarget` and its reason in + the message. +- **One trigger per `Kustomization`.** A trigger is in line for its `Kustomization` while an + eligible `GitTarget` references it, and while it holds the `Kustomization` or is handing it back. + When two in line name the same `Kustomization` in the same cluster, one holds it: the one with an + apply in flight, otherwise the one that holds it already, otherwise the oldest. The other reports + `HeldByAnotherTrigger`, does nothing, and takes over once the holder lets go. A trigger nothing + eligible references is not in line, so it keeps no other out. +- **The trigger hands the `Kustomization` back** when no eligible `GitTarget` references it any + more, and when it is deleted. `Held` reads `Releasing`: it suspends the `Kustomization` once more + if an apply has it resumed, waits until that is handled, and releases `spec.suspend`, so Flux + applies it on its own again (a `Released` Event), even one somebody had `flux suspend`ed before. + Until that is done it stays the holder, so no other trigger acts on the `Kustomization`. One + whose `spec.suspend` another manager also applies stays suspended, and the Event names that + manager. With another `FluxTrigger` in line, the `Kustomization` stays suspended for that one. A + `ClusterProvider` that is gone does not stop the hand-back: it goes on through whatever still + reaches the cluster, which for a provider without `kubeConfig` is the operator's own. When nothing + does, or the hand-back has not finished 10 minutes after `Held` turned `Releasing`, the trigger + lets go with a `ReleaseFailed` Event saying where the `Kustomization` was left. A reference that + returns before the release is done holds the `Kustomization` again; one that returns after adopts + it anew. +- **Deleting or re-creating a `GitTarget` hands the `Kustomization` back.** Flux then applies the + Git tip at once, which overwrites live edits not yet in Git, and the trigger adopts the + `Kustomization` again once the `GitTarget` is back. That is what Flux does without GitOps + Reverser, and with no `GitTarget` nobody would carry those edits to Git anyway. +- **A trigger whose configuration turns invalid keeps its hold.** It reports `Held=Unknown` + (`NotConfigured`), and the message says the `Kustomization` is left suspended under GitOps + Reverser's field manager; deleting the trigger, or its `GitTarget`s, hands it back. + +The trigger reports `Configured` and `Held`, and its `Ready` includes both: + +| Reason | Means | Readiness | +| --- | --- | --- | +| `Configured`: `Succeeded` | An eligible `GitTarget` names the trigger, and the `Kustomization` exists, applies a `GitRepository` that follows a branch, and may be patched | ready | +| `Configured`: `NotReferenced` (`Unknown`) | No `GitTarget` names the trigger. Nothing is read or written | ready: `Ready=True`, `NotReferenced` | +| `Configured`: `NotBound` (`Unknown`) | No `GitTarget` naming the trigger is eligible yet, and one cannot be judged yet | progressing | +| `Configured`: `SourceMismatch`, `NamespaceNotAuthorized` or `NotBound` (`False`) | Every `GitTarget` naming the trigger is refused: it writes another repository or branch, or is not admitted. The message lists each | stalled | +| `Configured`: `ApplierNotFound` | No such `Kustomization` in the applier's cluster, or Flux is not installed there | progressing: Flux objects are often created after the trigger | +| `Configured`: `UnsupportedSource` | The source is not a `GitRepository`, or breaks one of the [`GitRepository` rules](#the-gitrepository) | stalled | +| `Configured`: `SuspendOwnedElsewhere` | Another field manager applies `spec.suspend` | stalled: remove the field from the manifest it applies | +| `Configured`: `Forbidden` | The applier cluster's identity may not patch the `Kustomization` or its `GitRepository` | stalled | +| `Configured`: `NamespaceNotAuthorized` | The `clusterProviderRef` names a provider whose `accessFrom` does not admit the trigger's namespace | stalled | +| `Configured`: `ClusterProviderNotFound` | The `clusterProviderRef` names a provider that does not exist | progressing | +| `Configured`: `ApplierNamespaceNotAllowed` | The `Kustomization` is in another namespace than the trigger's, and the `ClusterProvider` does not set `allowCrossNamespace.appliers` | stalled | +| `Configured`: `HeldByAnotherTrigger` | Another `FluxTrigger` holds this `Kustomization` | stalled: delete one of the two | +| `Held`: `Suspended` | Suspended under GitOps Reverser, and the last suspend is handled. Nothing applies | ready | +| `Held`: `Applying` | An apply GitOps Reverser started is in flight: `status.operation` shows it | ready | +| `Held`: `Suspending` | Just suspended; waiting for the `Kustomization` to handle it | progressing | +| `Held`: `Releasing` | Handing the `Kustomization` back: suspending it if needed, then releasing `spec.suspend`, within 10 minutes | progressing | +| `Held`: `NotReferenced` | Nothing to hold: no `GitTarget` names the trigger | ready | +| `Held`: `NotConfigured` | The trigger is not `Configured`, so nothing is held, or the `Kustomization` is left suspended as the trigger last held it (the message says which) | follows `Configured` | +| `Ready`: `OperationUnresolved` | An apply has not completed within 5 minutes, or the `Kustomization` was [resumed by hand](#overriding-by-hand) | progressing; it clears when the apply does | + +`Ready=True` for a trigger no `GitTarget` names is deliberate: it does all it is asked to, which is +nothing. A Flux health check reads `Ready`, so a `Kustomization` that creates the trigger and waits +for it would otherwise never become ready, and the `Kustomization` that creates the `GitTarget` +after it would never apply. + +The `GitTarget` reports `ApplyTriggerReady`: `True` when the trigger is `Ready` and its +`GitRepository` fetches this repository and branch, `SourceMismatch` when it fetches another, +`ApplyTriggerNotFound` when the trigger does not exist, `Unknown` (`NotReferenced`) until the +trigger has seen that the `GitTarget` names it, and otherwise the trigger's own `Ready` reason. + +## Permissions + +GitOps Reverser looks the `Kustomization` up in the cluster of the trigger's `clusterProviderRef`, +with the identity of that `ClusterProvider`. That identity needs: + +| API group | Resource | Verbs | +| --- | --- | --- | +| `kustomize.toolkit.fluxcd.io` | `kustomizations` | `get`, `list`, `patch` | +| `source.toolkit.fluxcd.io` | `gitrepositories` | `get`, `patch` | +| `helm.toolkit.fluxcd.io` | `helmreleases` | `list`, to find the source's other consumers; without it the source is never prefetched | +| `source.toolkit.fluxcd.io` | `helmcharts` | `list`, for the same reason | +| `authorization.k8s.io` | `selfsubjectaccessreviews` | `create`, which every authenticated identity has | + +For the operator's own cluster, install the chart with `applierAccess.flux: true`, or apply +[`config/rbac/flux-applier-role.yaml`](../config/rbac/flux-applier-role.yaml) and its binding. For a +remote cluster, grant the same to the identity in the `ClusterProvider`'s kubeconfig. The chart's +grant is cluster-wide for convenience; GitOps Reverser only patches the `Kustomization` and +`GitRepository` a trigger names. + +## One apply + +All four [reasons](bi-directional.md#when-gitops-reverser-asks-for-an-apply) reach a +`Kustomization`. A `ForeignCommit` or `CatchUp` asks nothing when the `Kustomization` has already +applied the commit; a `Refusal` and a `Repair` always apply, since they ask for an unchanged commit +to be applied again. A `Kustomization` a `GitTarget` names for the first time gets a `CatchUp`, so +a commit already on the branch reaches it once the branch is quiet. On a quiet branch, GitOps +Reverser: + +1. Makes the `GitRepository`'s artifact hold the commit: when it does not, it stamps + `reconcile.fluxcd.io/requestedAt` on the `GitRepository` and waits for that fetch. The suspended + `Kustomization` ignores the new artifact. +2. Reads the artifact again and checks that the decision still holds. +3. Records the operation on the trigger's `status.operation` (`Requested`), so a restart finds it. +4. Resumes the `Kustomization`: one server-side apply sets `spec.suspend: false` and a new + `requestedAt`, guarded by the `uid` and `resourceVersion` it read. The reconcile that write + enqueues applies (`Running`). +5. Waits for the `Kustomization` to report that token handled, whether the apply succeeded, failed, + or waited on a dependency. +6. Suspends it again with another token (`Settling`). Once that suspend is handled, nothing + that started before it is still running, and the operation completes: `Succeeded` for + `Ready=True`, `Failed` with Flux's reason for `Ready=False`. + +```mermaid +sequenceDiagram + participant Rev as GitOps Reverser + participant Trig as FluxTrigger status + participant Src as GitRepository + participant Ks as Kustomization + + Rev->>Src: requestedAt (only when the artifact lacks the commit) + Src-->>Rev: artifact holds the commit + Rev->>Rev: decision still holds? + Rev->>Trig: operation Requested + Rev->>Ks: suspend: false + token + Ks-->>Rev: token handled, Ready says how it went + Rev->>Ks: suspend: true + suspend token + Ks-->>Rev: suspend handled + Rev->>Trig: operation Succeeded or Failed +``` + +**A failed apply is suspended again** like any other, which stops Flux's own `retryInterval` retry. +GitOps Reverser retries on its own backoff (2 seconds doubling to 5 minutes; at least 15 seconds +when the `Kustomization`'s configuration rejects the request), at the tip as it is then. An +`ApplyFailed` Event marks the first failure of a run. A resume no reconcile ran on (somebody ran +`flux suspend` in between) counts as failed too: a handled token alone does not prove Flux applied +anything. + +**An apply that takes long is followed, not timed out.** After 5 minutes the operation is +`Unresolved`: the `Kustomization` is suspended at once, which stops whatever would follow but not +the apply already running, and an `ApplyUnresolved` Event is recorded. An operation nobody follows +for a minute (its `GitTarget` deleted mid-apply, for example) is suspended by the trigger, completes +as failed, and records an `OperationUnfollowed` Event. A resumed `Kustomization` is never left with +nothing watching it. + +**The apply window.** From the moment an operation is asked for until 10 seconds after it completes, +a write by `kustomize-controller` over a newer API edit is withheld from Git, and GitOps Reverser +asks for a `Repair`; the trigger shows `repairPending` until that apply succeeds. Only a `kustomize-controller` entry newer +than the edit counts: undoing an edit can leave an older one as the newest entry, and that undo is +mirrored. When `kustomize-controller` and another manager, or the edit, wrote in the same second, +the write is mirrored and counted as `ambiguous`. + +**Reverting refused edits.** With `onRefusal: Reapply`, the target's `Kustomization` is resumed +for one apply of the tip; no new commit is needed. If its `GitRepository` has to fetch to reach the +tip, the consumers GitOps Reverser holds ignore the new artifact, and one in +`status.source.otherConsumers` applies it, as it would for any push. + +## Overriding by hand + +- **`flux resume ks checkout`** applies the `Kustomization` at once, unchecked. Within a minute the + trigger suspends it again, records a `ResumedElsewhere` Event, and shows the apply as an + `Unresolved` operation that completes once nothing it started is running. It does so every time. +- **`flux reconcile ks checkout`** does nothing visible while the `Kustomization` is suspended: Flux + marks the request handled without applying. +- **`flux suspend ks checkout`** is taken back with a token of GitOps Reverser's. +- To hand the `Kustomization` back to Flux for good, delete the `FluxTrigger`, or stop naming it from + every `GitTarget`. + +## Checking that it works + +```bash +# The applier is found, applies a branch, may be patched, and is held; and it applies this branch. +kubectl get fluxtrigger checkout -n team-a +kubectl get gittarget checkout -n team-a \ + -o jsonpath='{.status.conditions[?(@.type=="ApplyTriggerReady")]}{"\n"}' + +# The last apply, why, and where it stands; and what else applies the same source. +kubectl get fluxtrigger checkout -n team-a -o jsonpath='{.status.operation}{"\n"}{.status.source}{"\n"}' +# {"appliedRevision":"main@sha1:4f2c1ab9...","commit":"4f2c1ab9...","phase":"Succeeded","reason":"ForeignCommit",...} + +# Each apply asked for, each one that could not be asked or did not complete. +kubectl get events -n team-a --field-selector involvedObject.name=checkout,involvedObject.kind=GitTarget + +# Flux's own view: suspended, and the last revision it applied. +flux get ks checkout -n flux-system +``` + +`gitopsreverser_apply_requests_total{reason,outcome}` counts the requests; +[interpreting metrics](interpreting-metrics.md#apply-requests-and-the-push-receiver) reads it. + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| Trigger `Configured=False`, `ApplierNotFound` | No `Kustomization` with that namespace and name in the trigger's cluster, or Flux is not installed there | Create it, or recreate the trigger with the right `kustomizationRef`; an omitted namespace is the trigger's | +| Trigger `Configured=False`, `SuspendOwnedElsewhere` | The manifest that applies the `Kustomization` sets `spec.suspend` | Remove the field from that manifest | +| `ApplyTriggerReady=False`, `SourceMismatch`, or trigger `UnsupportedSource` | The message names the broken [`GitRepository` rule](#the-gitrepository) | Fix the `GitRepository`; the trigger then adopts the `Kustomization`, and the apply that was waiting is asked for | +| Trigger `Configured=False`, `ApplierNamespaceNotAllowed` | The `Kustomization` is in another namespace than the trigger, such as `flux-system` | Set `allowCrossNamespace.appliers: true` on the trigger's `ClusterProvider` (Helm: `clusterProvider.default.allowCrossNamespace.appliers`), or move the trigger and `GitTarget` to the `Kustomization`'s namespace | +| Trigger `Configured=Unknown`, `NotBound` | A `GitTarget` names the trigger, but cannot be judged yet: its `GitProvider` or `ClusterProvider` does not exist | Create the missing object; the message names it | +| Trigger `Ready=True`, `NotReferenced`, and the `Kustomization` is not suspended | No `GitTarget` names the trigger, so it holds nothing | Name it in the `GitTarget`'s `spec.applyTriggerRef` | +| Trigger `Configured=False`, `Forbidden` | The identity lacks one of the [permissions](#permissions) | Grant it, or set `applierAccess.flux: true` | +| A Warning `ApplyRequestFailed` Event | An apply could not be asked for; the message says why | GitOps Reverser keeps asking on a quiet branch, at most every five minutes | +| A Warning `ApplyFailed` Event | Flux ran the apply and reported it failed | Read the `Kustomization`'s status; GitOps Reverser retries on its backoff | +| A Warning `ApplyUnresolved` Event, trigger `OperationUnresolved` | The apply has not completed in 5 minutes | It is suspended and still followed; look at the `Kustomization`'s status and health checks | +| A Warning `OperationUnfollowed` Event | An apply was left in flight with nothing following it | It was suspended and completed as failed. A recurring one is a bug worth reporting with the operator's logs | +| A Warning `ReleaseFailed` Event | A trigger being deleted, or no longer referenced, could not hand its `Kustomization` back: a write it may not make, a `ClusterProvider` gone with no way left to its cluster, or 10 minutes passed | The message says where it was left; `flux resume` it if it should apply on its own | +| A dependent `Kustomization` waits | Its dependency's last apply failed | Fix the dependency's failure; GitOps Reverser's retry unblocks it | +| A Git-side push takes minutes to reach the cluster | No receiver, so the refresh finds it | Configure [the push webhook](bi-directional.md#telling-gitops-reverser-that-somebody-else-pushed) | +| A refused edit is not reverted | The target is suspended, `onRefusal` is not `Reapply`, or the refusal is not eligible | See [reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal) | + +## While GitOps Reverser is down + +**A `Kustomization` waits suspended while GitOps Reverser is down.** A Git-side push waits until +it is back, or until somebody runs `flux resume`, after which GitOps Reverser suspends the +`Kustomization` again when it starts. + +- **A graceful stop suspends what it had resumed.** An upgrade or a node drain stops the operator + mid-apply; the stop suspends that `Kustomization` before the process exits. The apply already + running still finishes, and the next process sees it to its end. +- **After a crash, the apply that was running finishes, and the next process suspends it.** Until + that process starts, the `Kustomization` stays resumed, so Flux may also apply a new artifact on + its own in between. + +An apply that was running when it stopped is not lost. The operation is recorded on the trigger +before the `Kustomization` is resumed, so the next process follows it to its end by its tokens, with +the apply window open. After its first look at the remote it asks every `Kustomization` for a +`CatchUp`, and for no `ForeignCommit`: an apply on startup could revert edits made while it was +down, before its watches caught them. + +## What the tests prove + +| Covered | Where | +| --- | --- | +| The [complete example](#a-complete-example) decodes strictly, reports `Configured=True`, binds the `GitTarget`, and an apply fetches and resumes the `Kustomization`; the manifest omits `spec.suspend` | [`flux_guide_test.go`](../internal/applier/flux_guide_test.go) | +| Every fact of kustomize-controller v1.9.6 the lifecycle relies on, in a fake that applies them | [`fluxsim`](../internal/applier/fluxsim/fluxsim_test.go) | +| An operation is not finished when the revision is attempted; a token handled while suspended is no apply; a resume alone reverts a refusal; a suspend does not stop a running apply; the artifact is rechecked before the resume; a failed apply is suspended and Flux's requeue applies nothing; a shared source applies only the consumer asked | [`flux_lifecycle_test.go`](../internal/applier/flux_lifecycle_test.go), [`flux_consumers_test.go`](../internal/applier/flux_consumers_test.go) | +| Adoption, an external resume suspended again with an Event each time, `SuspendOwnedElsewhere`, the hand-back on deletion (a `ClusterProvider` gone included), and the source's other consumers | [`fluxtrigger_hold_test.go`](../internal/controller/fluxtrigger_hold_test.go) | +| A trigger nothing names reads nothing and is `Ready`; the first eligible `GitTarget` adopts and the last releases, suspending an apply in flight first; a reference returning mid-release holds again; the binding over several `GitTarget`s; an older trigger not in line keeps none out, and a retiring holder stays first until its release is done; `ApplierNamespaceNotAllowed` | [`applytrigger_rule_test.go`](../internal/controller/applytrigger_rule_test.go) | +| `allowAnySourceNamespace: true` is refused on create and on any spec change, a stored one included; `false` and its removal are accepted | [`removed_allow_any_source_namespace_test.go`](../internal/controller/removed_allow_any_source_namespace_test.go) | +| Every operation checks the target again (incarnation, branch, admission, consent) and the source binding before it resumes, records itself first, and never asks for something weaker; a stop mid-apply suspends the `Kustomization` | [`dispatcher_test.go`](../internal/applier/dispatcher_test.go) | +| A failed attempt is decided again on a quiet branch; each `Kustomization` keeps its own backoff; a shared one is asked once; appliers sharing a source apply one at a time; a push is prefetched only while the branch is busy; a newly named `Kustomization` gets a `CatchUp`; an undo of an edit in the apply window is mirrored | [`apply_attempt_test.go`](../internal/git/apply_attempt_test.go), [`apply_groups_test.go`](../internal/git/apply_groups_test.go), [`apply_trigger_test.go`](../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../internal/git/apply_window_test.go) | +| A trigger nothing names is `Ready` and holds nothing; named from a `GitTarget` on another branch than its source, it is `SourceMismatch` and adopts nothing | [Manager Flux Applier](../test/e2e/flux_applier_e2e_test.go) | +| A `ForeignCommit` for a `SourceMismatch` applier never stamps the other branch's source and records `ApplyRequestFailed`; once the source is fixed the trigger adopts the `Kustomization` and Flux reaches the tip; dropping the reference hands it back, and naming the trigger again adopts it anew | Manager Flux Applier | +| A Gitea push webhook to the operator's receiver gets a Git-side push applied; with no receiver and no API edit, the refresh finds one | Manager Flux Applier | +| A failed apply is suspended again and retried; a `flux resume` is suspended again with a `ResumedElsewhere` Event; deleting the trigger hands the `Kustomization` back | Manager Flux Applier | +| A manager restarted during an apply suspends the `Kustomization` as it stops; the new one completes that same operation and leaves it suspended | [Flux Restart Recovery](../test/e2e/flux_restart_recovery_e2e_test.go) | +| Two API edits seconds apart end at the later value with no extra commit, and Flux catches up by itself | [Bi-directional (Flux)](../test/e2e/flux_bi_directional_e2e_test.go) | +| A Git-side push with an API edit on top lands both, and Flux applies the replayed commit | Bi-directional (Flux) | +| A new `GitRepository` revision from a commit outside the applied folder is enough for the `Kustomization` to apply and revert drift | Bi-directional (Flux) | +| A Git to cluster to Git round trip settles without a commit loop; a SOPS-encrypted Secret round-trips without a reverse commit; a Git revert removes the live object and the committed file | Bi-directional (Flux) | +| A refused edit with `onRefusal: Reapply` is reverted by a resume, with no commit, and settles | [Manager Refusal Reapply](../test/e2e/refusal_reapply_e2e_test.go) | + +`task test-e2e` runs the Manager specs and Flux Restart Recovery, and +`task test-e2e-bi-directional` the bi-directional corner. Not covered end to end yet: a remote +applier cluster, `Forbidden` against cluster RBAC, and the receiver installed by the Helm chart +rather than the e2e manifests. diff --git a/docs/bi-directional.md b/docs/bi-directional.md index 771338502..69982b69c 100644 --- a/docs/bi-directional.md +++ b/docs/bi-directional.md @@ -1,23 +1,65 @@ -# Bi-directional usage guide +# Bi-directional GitOps -GitOps Reverser publishes live cluster state to Git. Flux or Argo CD applies Git back to the -cluster. Run both and a resource can be changed from either side. This guide covers who decides -what, how to configure each actor, and which parts are proven by tests. +GitOps Reverser writes live cluster state to Git. Flux and Argo CD apply Git to the cluster. This +page covers the one setup where the two meet on the same folder, so that a resource can be changed +from either side: who decides what, which states the system moves through, and what stays exposed. +The complete setups are [bi-directional GitOps with Flux](bi-directional-flux.md) and +[bi-directional GitOps with Argo CD](bi-directional-argocd.md). -**The short version.** GitOps Reverser decides when the cluster is applied from Git. Flux or Argo -CD keeps doing what it is good at (fetching, rendering, applying, pruning, reporting health), but it -applies when GitOps Reverser asks, not on its own schedule. You configure that in three places: +## Do you need any of this? -1. Name each folder's **applier** on its `GitTarget`: the Flux `Kustomization` or the Argo CD - `Application` that applies it. See [declaring the applier](#declaring-the-applier). -2. Switch off the applier's own triggers: for Flux, short intervals and any Git-host webhook into a - Flux `Receiver` ([configure Flux](#configure-flux)); for Argo CD, automated sync - ([configure Argo CD](#configure-argo-cd)). -3. Point your Git host's push webhook at GitOps Reverser. See - [telling GitOps Reverser that somebody else pushed](#telling-gitops-reverser-that-somebody-else-pushed). +Only when Flux or Argo CD applies a folder that GitOps Reverser also writes. Every other setup is +ordinary GitOps and needs nothing from this page. -There is no merge and no lock. What remains exposed is small and named in -[what is still exposed](#what-is-still-exposed). +| Your setup | What you configure | +| --- | --- | +| GitOps Reverser mirrors a folder that nothing applies: an audit trail, a backup, the start of a repository | A `GitTarget` and its watch rules | +| Flux or Argo CD applies other folders or other repositories, next to GitOps Reverser | Nothing. Flux and Argo CD stay exactly as they are | +| Flux or Argo CD applies a folder that GitOps Reverser writes | An apply trigger, as this page describes | + +```mermaid +flowchart TD + Q1{"Does Flux or Argo CD apply a folder
that GitOps Reverser writes?"} + Q1 -->|no| ONEWAY["One-way. Mirror only, or GitOps as usual beside it.
Nothing on this page applies"] + Q1 -->|yes| Q2{"Does an apply trigger hold that applier?"} + Q2 -->|yes| BI["Bi-directional. GitOps Reverser decides when it applies"] + Q2 -->|no| EXPOSED["Shared folder without a trigger.
Works, and the applier can revert live edits"] + + style ONEWAY fill:#e8f5e9,stroke:#43a047 + style BI fill:#e8f5e9,stroke:#43a047 + style EXPOSED fill:#fff3e0,stroke:#ff9800 +``` + +Everything bi-directional is opt-in and off by default: the apply triggers, the push receiver, +`onRefusal: Reapply`, the chart's `applierAccess` grants, and the `ClusterProvider`'s +`allowCrossNamespace.appliers`, which a trigger needs to hold an applier outside its own namespace, +such as one in `flux-system` or `argocd`. A `GitTarget` without `spec.applyTriggerRef` never reads, +patches or syncs anything of Flux's or Argo CD's, and neither does a trigger no `GitTarget` names, so +a `Kustomization` keeps its interval and an `Application` keeps automated sync. + +**A shared folder without a trigger works, and is exposed.** The applier keeps its own schedule, +and that schedule cannot see the edits GitOps Reverser has not pushed yet +([why](#why-the-applier-cannot-decide-for-itself)). An apply at the wrong moment reverts a live edit, +and GitOps Reverser then commits the revert as the applier's change. With Argo CD's +`selfHeal: true`, a live edit is reverted within a second, before it can reach Git at all. If you +run a shared folder this way, keep [controller bookkeeping out of Git](#keep-controller-bookkeeping-out-of-git), +and consider [split ownership](#the-split-ownership-alternative) for the fields both sides change. + +## How it works + +GitOps Reverser decides when the cluster is applied from Git. Flux or Argo CD keeps fetching, +rendering, applying, pruning and reporting health, and applies when GitOps Reverser asks. Three +things set that up: + +1. **An apply trigger holds the applier.** A `FluxTrigger` holds a Flux `Kustomization`, an + `ArgoCDTrigger` an Argo CD `Application`, and the `GitTarget` names the trigger in + `spec.applyTriggerRef`. A trigger acts only while an eligible `GitTarget` names it: one in its + namespace, admitted by its `ClusterProvider`s, that writes the repository and branch the applier + fetches. +2. **The applier's own triggers are off.** The `FluxTrigger` keeps its `Kustomization` suspended; + the `Application` runs with automated sync off. +3. **The Git host's push webhook reaches GitOps Reverser**, so a Git-side push is applied in + seconds. See [telling GitOps Reverser that somebody else pushed](#telling-gitops-reverser-that-somebody-else-pushed). ```mermaid flowchart LR @@ -40,64 +82,62 @@ flowchart LR style APP fill:#fff3e0,stroke:#ff9800 ``` -## Who does what +The two appliers get the same contract by different means: -| Actor | Does | Does not | +| | Flux | Argo CD | | --- | --- | --- | -| Kubernetes API | Takes edits from people and controllers | | -| GitOps Reverser | Mirrors every edit into Git, and decides when the applier may apply | Apply anything to the cluster itself | -| Git host | Stores the branch, and tells GitOps Reverser when somebody else pushed | Notify the applier | -| Applier (a Flux `Kustomization` or an Argo CD `Application`) | Fetches, renders, applies and prunes, when asked | Apply on a short timer, a webhook or an automated sync of its own | -| A human | Can still run `flux reconcile` or press Sync | | - -## Why GitOps Reverser owns the trigger - -An apply is safe only when nothing live is missing from the commit being applied. Between an edit -and its push, the edit is live in the cluster and absent from Git: it sits in an open commit window -or in the pending writes. An apply at that moment writes Git's older value over it. GitOps Reverser -would then mirror that revert faithfully, as a commit authored by the applier, and the edit would be -gone. (An edit that lands while an apply GitOps Reverser asked for is running keeps its value in -Git: the applier's write over it is not mirrored. See -[the Flux guide](flux.md#what-gitops-reverser-asks-and-when) and -[the Argo CD guide](argocd.md#what-gitops-reverser-asks-and-when).) - -Only GitOps Reverser can see those pending edits. The applier cannot. So an apply nobody asked -GitOps Reverser about is an apply nobody checked, and every trigger the applier runs by itself is -one of those: - -- **A Git-host webhook into the applier** fires on every push, including GitOps Reverser's own. Two - API edits a few seconds apart are enough to have the first commit applied over the second edit. -- **Argo CD's timed refresh** (`timeout.reconciliation`, 120s plus up to 60s of jitter) resolves - the branch tip, and automated sync runs whenever that tip is a commit it has not synced. With - `selfHeal: false` it skips only a commit it already synced. Every push GitOps Reverser makes is a - new commit, so two to three minutes after each push the whole `Application` is synced over - whatever is pending at that moment. -- **Flux's intervals** apply on every reconcile, whatever woke it: kustomize-controller corrects - drift each time. A short `GitRepository` interval turns each of GitOps Reverser's pushes into an - apply a minute later. - -None of these can be filtered from outside. They have to be switched off, and GitOps Reverser has -to make the requests instead. - -## When GitOps Reverser requests an apply - -Every request waits for a **quiet branch**: no open commit window and no pending writes on the -branch worker. That is the moment nothing live is missing from the tip. +| Held by | A `FluxTrigger` | An `ArgoCDTrigger` | +| Kept from applying on its own by | `spec.suspend`, which the trigger owns | Automated sync off, which you set on the `Application` | +| One apply is | A resume for one reconcile, then a suspend | A sync operation pinned to the commit | +| Catches up with GitOps Reverser's own pushes | Yes, once the branch is quiet for `quietPeriod` | No: nothing it holds goes stale | +| The tool's own push webhook | Removed: GitOps Reverser's receiver replaces it | May stay: with automated sync off it only compares | +| A human override | `flux resume` applies once; it is suspended again within a minute | Sync applies; GitOps Reverser waits for it to end | +| Handing it back | Delete the trigger, or stop naming it from every `GitTarget`: the `Kustomization` is resumed | Delete the trigger, or stop naming it, then switch automated sync on | +| Complete setup | [Bi-directional GitOps with Flux](bi-directional-flux.md) | [Bi-directional GitOps with Argo CD](bi-directional-argocd.md) | + +## Why the applier cannot decide for itself + +An apply is safe only when nothing live is missing from the commit it applies. Between an edit and +its push, the edit is live in the cluster and absent from Git: it sits in an open commit window or +in the branch worker's pending writes. An apply at that moment writes Git's older value over it, +GitOps Reverser mirrors that revert faithfully, and the edit is gone. + +Only GitOps Reverser can see the pending edits, so every trigger the applier runs by itself is an +apply nobody checked: + +- **A Git-host webhook into the applier** fires on every push, GitOps Reverser's own included. Two + API edits a few seconds apart are enough for the first commit to be applied over the second edit. +- **Argo CD's automated sync** syncs every commit it has not synced yet, and every push GitOps + Reverser makes is one. Two to three minutes after each push (`timeout.reconciliation`, 120s plus + jitter), the `Application` is synced over whatever is pending. `selfHeal: false` does not change + that. +- **Flux's intervals** apply on every reconcile, whatever woke it: on the interval, on a new + artifact, on a controller restart. + +None of these can be filtered from outside. They are switched off, and GitOps Reverser makes the +requests instead. + +## The quiet branch + +Every apply GitOps Reverser asks for waits for a **quiet branch**: no open commit window and no +pending writes, on this branch worker or on any other worker writing the same repository and branch +through another `GitProvider`. That is the moment nothing live is missing from the tip. -| Reason | When | Which applier | -| --- | --- | --- | -| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push | Every applier declared on the branch | -| `Refusal` | An edit was refused and the target has `onRefusal: Reapply` | The refusing target's applier | -| `CatchUp` | The branch has been quiet for `spec.applier.quietPeriod` after GitOps Reverser's own pushes | Every Flux `Kustomization` on the branch | - -The cluster already holds the values of GitOps Reverser's own commits, because they were read from -it, so applying them changes nothing. Flux is asked to catch up with them anyway, and -[configure Flux](#configure-flux) says why. Argo CD is not: with automated sync off it applies -nothing by itself, so nothing it holds goes stale. +```mermaid +stateDiagram-v2 + direction LR + Quiet --> Busy: an API edit opens a commit window + Busy --> Busy: more edits, commits and pushes + Busy --> Quiet: every window closed, every write pushed + note right of Quiet + An apply starts only here, + after the decision is checked again + end note +``` -A foreign commit that arrives while edits are pending does not wait for their windows to close. -GitOps Reverser closes them, replays the pending writes onto the foreign commit, pushes, and asks -the applier to apply the commit it just pushed: +A foreign commit does not wait for open windows to close on their own. GitOps Reverser closes them, +replays the pending writes onto the foreign commit, pushes, and asks for the commit it just pushed. +The applied commit holds the API side's values, so applying it reverts nothing: ```mermaid sequenceDiagram @@ -117,112 +157,74 @@ sequenceDiagram Note over API: W's objects already hold W's values,
so only Bob's change lands ``` -The applied commit carries the API side's answer, so applying it reverts nothing Alice did. That -is the [contested-cell rule](#what-happens-when-somebody-else-pushed-first) seen from the cluster: -for an object both sides changed, the API value wins. - -A request asks; it never reports that the apply happened. Requests are sent off the publication -path, and a failed request never fails a publication that succeeded: the worker keeps what the -applier is owed and decides the retry on a quiet branch. Read the applier's own status to see what -it applied. - -## Declaring the applier - -`spec.applier` names exactly one applier per `GitTarget`, a Flux `Kustomization` or an Argo CD -`Application`: - -```yaml -apiVersion: configbutler.ai/v1alpha3 -kind: GitTarget -metadata: - name: checkout - namespace: team-a -spec: - gitProviderRef: - name: platform-repo - branch: main - path: apps/checkout - applier: - kustomizationRef: - namespace: flux-system - name: checkout - onRefusal: Reapply -``` - -```yaml -spec: - applier: - applicationRef: - namespace: argocd - name: checkout -``` - -GitOps Reverser looks the applier up in the **source cluster**, the cluster it applies to, with -the identity the `ClusterProvider` already uses, or in the cluster `spec.applier.clusterProviderRef` -names. That identity needs permission to read and write the applier; the chart grants it for the -operator's own cluster with `applierAccess.flux: true` or `applierAccess.argocd: true`. -The applier must apply this folder's repository and branch, or the target reports -`ApplierConfigured=False`. The -[configuration reference](configuration.md#who-applies-this-folder-specapplier) has every field -and reason. - -Several `GitTarget`s may name the same applier, for example a `Kustomization` whose path covers -several folders. Requests to one applier are sent once. - -## Configure Argo CD - -[Using GitOps Reverser with Argo CD](argocd.md) is the complete setup, with a worked example a unit -test holds to the code. The standard is full control: the `Application` named by -`spec.applier.applicationRef` has **automated sync off**, and every sync it runs is one GitOps -Reverser started, pinned to a commit it checked. - -```yaml -spec: - syncPolicy: - automated: - enabled: false # automated sync off; GitOps Reverser starts every sync - prune: true # kept for the syncs GitOps Reverser starts -``` - -An `Application` with automated sync on is `ApplierConfigured=False` with `AutomatedSyncEnabled`, -and GitOps Reverser never syncs it. `selfHeal` means nothing with automated sync off. Argo CD's -timed refresh and its `/api/webhook` may stay: with automated sync off they only compare. A sync -GitOps Reverser starts is the one automated sync would (`prune`, `syncOptions` and `retry` from the -`Application`), with the revision pinned and `initiatedBy` set to `gitops-reverser`; it waits for a -sync already running rather than replace it. +## When GitOps Reverser asks for an apply -Without `spec.applier`, an `Application` with automated sync keeps its own triggers, with the -exposure [why GitOps Reverser owns the trigger](#why-gitops-reverser-owns-the-trigger) describes: -each of GitOps Reverser's pushes is synced over whatever is pending at that moment, and with -`selfHeal: true` a live edit is reverted in about a second, before GitOps Reverser can publish it. - -### The split-ownership alternative +| Reason | When | Which applier | +| --- | --- | --- | +| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push | Every applier declared on the branch | +| `Refusal` | An edit was refused and the target has `onRefusal: Reapply` | The refusing target's applier | +| `Repair` | An apply reverted an API edit made while it ran; Git kept the edit | That applier | +| `CatchUp` | The branch has been quiet for the `FluxTrigger`'s `quietPeriod` after GitOps Reverser's own pushes, since the operator started, or since a `GitTarget` first named the trigger | Every Flux `Kustomization` on the branch | -`ignoreDifferences` on a field, with `RespectIgnoreDifferences=true` in `syncOptions`, hands that -one field to the API. Its differences alone do not make the `Application` `OutOfSync`, and the -option copies the live value into the apply target, so an ordinary sync preserves it. +GitOps Reverser's own commits hold values it read from the cluster, so applying them changes +nothing, and only Flux is asked to catch up with them: its inventory learns of an object only from +an apply of a commit that holds it, and an object it never applied is not pruned when Git later +deletes it. An `Application` with automated sync off applies nothing by itself, so nothing it holds +goes stale. -The cost is that the field stops being Git-driven on an existing object: Git seeds it at creation, -but later Git edits to it do not reach the object while the ignore is active. It protects no other -field from GitOps Reverser's whole-object replay. +Several `GitTarget`s may name one trigger, for a `Kustomization` or `Application` whose path covers +several folders: the applier is asked once for all of them. -## Configure Flux +## The life of an apply -[Using GitOps Reverser with Flux](flux.md) is the complete setup, with a worked example a unit test -holds to the code. In short: Flux has no fetch-only mode (a new `GitRepository` artifact makes every -`Kustomization` reading it apply, and `suspend: true` also blocks GitOps Reverser's requests), so -set both intervals to a long backstop such as `24h`, keep the `Kustomization` unsuspended, and -remove any Git-host webhook into a Flux `Receiver` for the repository. GitOps Reverser then asks the -`GitRepository` to fetch: after a foreign push, after a refusal, and, once the branch has been -quiet for `spec.applier.quietPeriod` (`30s` by default), to catch up with its own pushes, so a -Flux restart never applies a stale artifact. +An apply is an **operation**, recorded on the trigger's `status.operation` before the applier is +asked, so a restarted operator finds it and follows it to its end: -Flux's equivalent of split ownership is `Kustomization.spec.ignore`, added in kustomize-controller -`v1.9.0`, with the same trade as Argo CD's. The -[source review](facts/gitops-apply-and-field-ignore.md) has the code paths. +```mermaid +stateDiagram-v2 + [*] --> Due: ForeignCommit, Refusal, Repair or CatchUp + Due --> Due: tip moved or an edit is pending, decide again later + Due --> Requested: quiet branch, decision still holds, operation recorded + Requested --> Running: the applier is seen working on it + Running --> Settling: Flux only, applied and suspended again + Running --> Succeeded + Running --> Failed + Settling --> Succeeded + Settling --> Failed + Requested --> Unresolved: not completed in 5 minutes + Running --> Unresolved + Settling --> Unresolved + Unresolved --> Succeeded + Unresolved --> Failed + Failed --> Due: after the applier's backoff + Succeeded --> Due: an edit was withheld during the apply (Repair) + Succeeded --> [*] +``` -When upgrading a chart whose pin is declared in Git, reconcile the owning `Kustomization` first -so it applies the new `HelmRelease` and source configuration, then the `HelmRelease` if needed. +`Due` lives only in the branch worker's memory: an apply it means to ask for, and nothing shows it +until it is asked. Every other state is `status.operation.phase`. While an operation is in flight +(`Requested`, `Running`, `Settling` or `Unresolved`), the applier is asked nothing else. + +- **Just before it asks, GitOps Reverser checks the decision again**: the commit is still the branch + tip, no edit waits to be published, and each `GitTarget` still names this trigger on this branch, + is still admitted by its `ClusterProvider`s, and, for a `Refusal`, still consents; and the trigger + still holds the applier. When any of that + changed, it asks nothing and decides again at the next quiet moment. It never asks for something + weaker than what it was told. +- **An edit made while the apply runs keeps its value in Git.** The apply writes the older commit + over it on the cluster. GitOps Reverser recognizes that write by the applier's field manager, + withholds it from Git, and asks for a `Repair`, which applies the tip that holds the edit. Only a + write proven to be the applier's is withheld: its field manager alone, with an entry newer than the + edit. One that ties with the edit is mirrored, and a delete never is withheld. Each such write is + counted in `apply_window_collisions_total`. +- **A failed operation is decided again from scratch**, after that applier's own backoff (2 seconds + doubling to 5 minutes), on a quiet branch, at the tip as it is then. +- **An operation still in flight after 5 minutes is `Unresolved`.** It is still followed, nothing + ends it by timer, and nothing more is asked of that applier until it completes. + +A request asks; it never reports that the apply happened. Requests run off the publication path, +and a failed one never fails a publication that succeeded. Read the applier's own status for what +it applied. ## Telling GitOps Reverser that somebody else pushed @@ -232,254 +234,150 @@ GitOps Reverser learns that the branch moved in three ways, and each feeds the s | --- | --- | --- | | A push webhook from the Git host | Seconds | `GitProvider.spec.receiver` | | A rejected compare-and-swap push | At the next publication | Nothing: it is how every push works | -| The periodic refresh | Within `--git-refresh-interval` (10m) plus the 5-minute reconcile tick | Nothing: on by default | - -Point the Git host's push webhook at the URL in `GitProvider.status.webhookPath`, served on the -receiver listener. The receiver speaks GitHub's, GitLab's and Gitea's native push payloads, plus a -`generic-hmac` body for a relay you run yourself. See -[receiving push notifications](configuration.md#receiving-push-notifications-specreceiver). - -A delivery is only a hint. GitOps Reverser does not trust the commit it names: it reads the -branch's ref advertisement (one request) and decides from what the remote says. So a late delivery -for one of GitOps Reverser's own pushes, a duplicate, or a rollback onto one of its older commits -all come out right. When it cannot tell whether a commit is its own, it treats it as foreign. - -Without a receiver everything still works, later: a foreign push is applied once the refresh finds -it. - -**A notification that the branch moved never triggers a cluster snapshot.** The cluster still holds -the values from before the push, so a snapshot would publish them over the incoming change. A push -tells GitOps Reverser about Git; it says nothing about the cluster. - -Read where the branch is, what proved it, and what was last requested: - -```bash -kubectl get gittarget checkout -n team-a \ - -o jsonpath='{.status.remote}{"\n"}{.status.applier}{"\n"}' -# {"commit":"4f2c1ab9...","lastVerifiedAt":"2026-10-20T10:14:02Z","verifiedBy":"Fetch"} -# {"requestedCommit":"4f2c1ab9...","requestedAt":"2026-10-20T10:14:03Z","reason":"ForeignCommit"} -``` - -## One queue per branch - -Every watched event for a given `GitProvider` and branch lands on one queue owned by one branch -worker. Events are applied in arrival order and never overtake each other, and two `GitTarget` -objects on the same provider and branch share that worker, its ordering, and its quiet moments. - -1. **Commit.** Events collapse into one local commit per author after - [`spec.commit.window.idleTimeout`](configuration.md#the-commit-window-speccommitwindow) of silence, - `5s` by default, or at most `maxDuration` (`1m`) after the first. `0s` gives one commit per event. -2. **Push.** Local commits go to the remote at most once every five seconds, as a - compare-and-swap. The push declares the commit it expects the branch to be at, and the Git host - refuses the update if the branch has moved off the commits' base. - -A publishing target reads the remote only through the push it was making anyway. An idle target -re-proves where its branch is every -[`--git-refresh-interval`](configuration.md#keeping-an-idle-target-fresh---git-refresh-interval), -at the cost of one ref advertisement, and fetches only when the branch actually moved. - -## What happens when somebody else pushed first - -Your push is rejected and the worker **replays**: it fetches the new tip, re-plans every pending -write against it, and pushes again. Three attempts, then the writes stay pending for a later try. -Whatever the worker learned along the way, it now knows the branch gained a foreign commit, and the -appliers are asked to apply once the replayed push lands. - -```mermaid -flowchart TD - PUSH["Push, declaring the commit the writes were based on"] --> MOVED{"Is the branch still there?"} - MOVED -->|yes| DONE["Accepted"] - MOVED -->|"no: somebody else pushed"| FETCH["Fetch the new tip"] - FETCH --> REPLAN["Re-plan every pending write against the new tree"] - REPLAN --> DIFF{"Does the captured object still differ from what Git now holds?"} - DIFF -->|no| NOOP["No commit. The other writer's value stands."] - DIFF -->|yes| COMMIT["Commit on top, then push again"] - COMMIT --> MOVED - - style DONE fill:#e8f5e9,stroke:#43a047 - style NOOP fill:#e8f5e9,stroke:#43a047 -``` - -Replay re-plans; it does not rebase a diff. Each pending write still carries the object as the -watch captured it, and writing that object onto the new tree has one of two outcomes: - -- **No change, so no commit.** The file in Git already says what the captured object says, - because whoever pushed made the same change. -- **A change, committed on top.** The captured object is written over the other writer's value, - and the branch ends at the API side's value. - -A write with no legal destination is refused instead, and there is no merge: a watch event carries -the object's whole state, with no baseline that would say which fields changed on each side. +| The periodic refresh | Within `--git-refresh-interval` (`10m`) plus the 5-minute reconcile tick | Nothing: on by default | + +To serve the receiver: + +1. Set the chart's `servers.receiver.enabled: true`, which passes `--receiver-bind-address=:9292` + (`servers.receiver.bindAddress`). The chart adds a `-receiver` `Service`. The listener is + plain HTTP; expose it to your Git host with an Ingress or Gateway route that terminates TLS, the + way Flux's `webhook-receiver` is exposed. +2. Create a Secret with the webhook secret under the key `token`, in the `GitProvider`'s namespace. + It is the key a Flux `Receiver` Secret uses, so an existing one can be copied over. +3. Set `GitProvider.spec.receiver` with that Secret and the host's `type`: `github`, `gitlab`, + `gitea`, or `generic-hmac` for a relay of your own. +4. Read the path from `status.webhookPath` and register `https://` as a push + webhook with the same secret: + + ```bash + kubectl get gitprovider platform -n team-a -o jsonpath='{.status.webhookPath}' + # /git-push/3f9a…c41e + ``` + + The path is derived from the token, as a Flux `Receiver` path is. A wrong path and a wrong + secret both answer `404`; the operator's log says which. Rotating the token moves the path. + +A delivery is only a hint. GitOps Reverser reads the branch's ref advertisement and decides from +what the remote says, so a late delivery, a duplicate, or one for its own push applies nothing it +should not. When it cannot tell whether a commit is its own, it treats it as foreign. The +[configuration reference](configuration.md#receiving-push-notifications-specreceiver) has every +field. + +**A push never triggers a cluster snapshot.** The cluster still holds the values from before the +push, so a snapshot would publish them over the incoming change. + +## When both sides edit the same object + +The API side wins. When a push is rejected because somebody else pushed first, the branch worker +fetches the new tip, re-plans every pending write against it, and pushes again. Each pending write +carries the whole object as the watch captured it, so writing it onto the new tree either changes +nothing (the other writer made the same change) or writes the API side's value over the other +writer's. There is no merge: a watch event has no baseline that says which fields changed on each +side. [API-first publication](api-first-publication.md) has the replay in detail. ### What a write does to the file The writer edits fields in place within the existing document, so what matters is which fields -change rather than which files. For a plain manifest: +change rather than which files: -| In the file | What a replay does to it | +| In the file | What a write does to it | | --- | --- | | A field the live object also carries | Set to the live value | | A field in Git that the live object does not have | **Removed** | | Comments, key order, other documents in the file | Preserved | | An edit that cannot be placed safely | Refused, and the file is left untouched | -In a kustomize layout the live object is the rendered result rather than the contents of any one -file, so the edit is projected back through the build first and only the fields that document is -responsible for are touched. An edit the projection cannot place is refused rather than guessed -at, which is why the last row exists. +In a kustomize layout, the edit is projected back through the build first, and only the fields that +document is responsible for are touched. An edit the projection cannot place is refused. **The consequence to plan around.** If Alice changes `replicas` through the API while Bob changes -`image` in Git on the same object, replaying Alice's captured object writes her `replicas` and -also the `image` she was looking at. Bob's change to a different field of the same object does -not survive, and a field Bob added that the cluster never had is removed outright. Separate -objects are independent of each other. Separate fields of one object are not. - -## There are no timing guarantees - -Committing and pushing takes on the order of a second at best, and the Kubernetes API accepts a -great many changes in a second. Treat live editing as a different way of making changes rather -than a faster `git push`: - -- You make a change and then observe that it arrived. You do not save and receive a confirmation. -- Intermediate values may never reach Git. Two edits inside one commit window produce one commit - holding the later value. -- Git history records the states the worker captured and published. Expect gaps between them. +`image` in Git on the same object, Alice's captured object carries her `replicas` and the `image` she +was looking at. Bob's change to that object does not survive, and a field Bob added that the cluster +never had is removed. Separate objects are independent of each other; separate fields of one object +are not. -When a specific change has to be confirmed in Git, use a -[`CommitRequest`](configuration.md#commitrequest). `Pushed=True` with `status.commit` and -`status.branch` names the commit it produced. - -### What is still exposed - -Owning the trigger removes every apply that could revert an edit for no reason. It cannot reserve -the seconds an apply takes: - -- **An edit made while a requested apply runs** is reverted, then mirrored as the applier's - commit. That happens only after somebody else pushed, after a refusal with `Reapply`, or after a - `quietPeriod` of silence on a Flux branch. It no longer follows every commit GitOps Reverser - makes. -- **Flux's backstop timers** still apply on their own, at the tip as the last catch-up left it. - Argo CD with automated sync off has none. -- **A human** running `flux reconcile` or pressing Sync applies whenever they do. +### The split-ownership alternative -## Why the loop stops +Hand a field to one side instead. Argo CD's `ignoreDifferences` with `RespectIgnoreDifferences=true` +in `syncOptions`, and Flux's `Kustomization.spec.ignore` (kustomize-controller `v1.9.0` and later), +make the applier leave that field alone. The cost is that the field stops being Git-driven on an +existing object: Git seeds it at creation, and later Git edits to it do not reach the object. It +protects no other field from the whole-object replay. The +[source review](facts/gitops-apply-and-field-ignore.md) has the code paths. -The obvious fear is a feedback loop: GitOps Reverser commits, the applier applies, the apply -produces a watch event, GitOps Reverser commits again. It settles instead, for two reasons. +## There are no timing guarantees -GitOps Reverser does not ask for its own commits to be applied, except Flux's catch-up. And when a -commit it wrote is applied, the live object already holds those values, because they were read -from that object in the first place. The apply changes nothing, the event it produces carries -content identical to the file already in Git, and no commit is made. In general: **applying a -resource that already matches does not produce a change.** What can break that reasoning is -controller bookkeeping. +Committing and pushing takes on the order of a second at best, and the Kubernetes API accepts a +great many changes in a second. Treat a live edit as a change you observe arriving, not a save that +returns a confirmation. Two edits inside one commit window produce one commit holding the later +value, so Git history records the states the worker captured, with gaps between them. When a +specific change has to be confirmed in Git, use a [`CommitRequest`](configuration.md#commitrequest). ## Keep controller bookkeeping out of Git Argo CD stamps `argocd.argoproj.io/tracking-id` on every non-CRD object it applies, Flux stamps `kustomize.toolkit.fluxcd.io/` labels and annotations, and a client-side apply writes -`kubectl.kubernetes.io/last-applied-configuration`. None of it is user intent. If it reached Git -the sanitized live object would stop matching the committed file, and every apply would produce a -commit. +`kubectl.kubernetes.io/last-applied-configuration`. None of it is user intent. If it reached Git, +the live object would stop matching the committed file, and every apply would produce a commit. GitOps Reverser strips all of it before writing, along with `argocd.argoproj.io/installation-id`, -`kro.run/`, `applyset.kubernetes.io/`, and `kcp.io/cluster`. The rules and the reasoning for each -are in [`internal/sanitize/types.go`](../internal/sanitize/types.go). Under -`argocd.argoproj.io/` only the exact tracking keys go: siblings such as `sync-wave`, -`sync-options`, and `hook` are user intent that belongs in Git. - -**Leave Argo CD on its default `annotation` resource tracking for mirrored paths.** The `label` -and `annotation+label` methods stamp `app.kubernetes.io/instance`, which is indistinguishable from -the standard recommended label that Helm and Kustomize set for application reasons. GitOps Reverser -therefore does not strip it, and label tracking would put Argo CD's bookkeeping into your commits. - -## When a write is refused or a push fails - -A live edit can succeed while its publication does not. The cluster holds the change and Git does -not. Watch push failures and the target's write-acceptance conditions: a successful Kubernetes -edit confirms the API write and nothing beyond it. +`kro.run/`, `applyset.kubernetes.io/`, and `kcp.io/cluster`; the rules and reasons are in +[`internal/sanitize/types.go`](../internal/sanitize/types.go). Under `argocd.argoproj.io/` only the +exact tracking keys go: `sync-wave`, `sync-options` and `hook` are user intent that belongs in Git. +That is also why the loop settles: an applied commit GitOps Reverser wrote matches the live object, +so the watch event it produces commits nothing. -Pending writes are held in worker memory. They do not survive a restart, and an edit that failed -before becoming a pending write is gone from GitOps Reverser's side. Recovery is a resync, which -derives state from the cluster as it exists at that moment, so it cannot reproduce an intermediate -value that has since been overwritten. +**Leave Argo CD on its default `annotation` resource tracking for mirrored paths.** The `label` and +`annotation+label` methods stamp `app.kubernetes.io/instance`, the same label Helm and Kustomize set +for application reasons, so GitOps Reverser cannot strip it, and Argo CD's bookkeeping would reach +your commits. -### Choosing `spec.onRefusal` +## Choosing `spec.onRefusal` -A refused edit has no legal destination in Git, so nothing is committed for it and it stays live +A refused edit has no legal destination in Git, so nothing is committed for it, and it stays live until something applies Git over it. -| Policy | Response to a refused edit | Operator's trade | +| Policy | Response to a refused edit | Trade | | --- | --- | --- | -| `Ignore` (default) | Report the refusal | Leave recovery to a human or a later apply | -| `Reapply` | Ask the target's applier to apply the current tip, once the branch is quiet | The refused edit is reverted; nothing still waiting to be published is | - -`Reapply` needs a declared applier, and the request goes to that applier alone: an `Application` -syncs the tip. When a `Kustomization`'s `GitRepository` has to fetch to reach the tip, though, every -`Kustomization` on that `GitRepository` applies the new revision, exactly as it would for any push: -moving the source is the only way to apply the tip, and Flux wakes everything that reads it. Give -the folder its own `GitRepository` if the apply must reach that folder alone. Waiting for a quiet -branch means an allowed edit still in a commit window is published first, so the apply reverts only -what has no home in Git. - -Eligibility is narrow: the folder must be accepted, the refused edit must concern a document the -folder already holds, and the write must remove no document, because re-applying would put back an -object somebody deliberately deleted. Invalid or unsupported folders, new objects and removals do -not trigger it. A standing refusal is requested once, not on every recheck, and a suspended target -requests nothing. See -[reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal). - -### A restored value does not prove the target recovered - -A refusal still reports `GitPathAccepted=False`, `Stalled=True`, and `Ready=False`, under either -policy. The target keeps rechecking, roughly every ten seconds; `Stalled` does not mean the -controller has stopped running. - -Re-applying neither clears those conditions nor makes an unrepresentable edit writable. A -successful resync covering the refused scope must establish acceptance again. Treat recovery as two -observations: the live resource has the intended value, and the `GitTarget` returns to -`Ready=True`. - -## What it costs +| `Ignore` (default) | Report the refusal | Recovery is left to a human or a later apply | +| `Reapply` | Ask the target's applier to apply the tip, once the branch is quiet | The refused edit is reverted; nothing still waiting to be published is | -**GitOps Reverser becomes the only fast path to a deployment.** While it is down, a push by a -human reaches the cluster only through a manual sync, `flux reconcile`, or Flux's backstop -interval. Bi-directional editing is already stopped in that state; Git-side deployments now slow -down with it. Monitor the operator the way you monitor the applier. +`Reapply` needs an apply trigger. It fires only for an edit to a document the folder already holds, +never for a write that removes one, and a reverted edit does not clear the refusal: the target +returns to `Ready=True` only after a resync covering the refused scope. The eligibility rules are in +[reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal). -**Round trips to the Git host are the seconds you wait through.** An uncontended publication talks -to the remote twice, and a foreign push costs a fetch, a replay on a checkout already on disk, and -one push. If your workload is mostly Git-side edits with occasional live changes, you are using the -tool against its grain. The per-operation numbers are measured, in -[`git-roundtrip-ledger.golden`](../internal/git/testdata/git-roundtrip-ledger.golden). +## What is still exposed, and what it costs + +- **An edit made while a requested apply runs** is reverted on the cluster until the `Repair`. It + keeps its value in Git. It can happen only during an apply GitOps Reverser asked for: after a + foreign push, a refusal with `Reapply`, or a Flux catch-up. +- **A human** running `flux resume` or pressing Sync applies whenever they do. GitOps Reverser + suspends a resumed `Kustomization` again within a minute. +- **GitOps Reverser becomes the only path to a deployment.** While it is down, the shared folders + wait: a push reaches the cluster only through a manual sync or `flux resume`. A graceful stop + suspends a `Kustomization` it had resumed; after a crash, the apply that was running finishes, + and the next process suspends it. Monitor the operator the way you monitor the applier. +- **A trigger may hold any applier its `ClusterProvider` reaches.** With + `allowCrossNamespace.appliers`, every namespace the provider admits may hold a `Kustomization` or + `Application` in another namespace whose source is the repository its `GitProvider` names. That + is a delegation by the platform admin: nothing proves the namespace owns the applier, or may + write to the repository, since the binding compares URLs. +- **Pending writes live in worker memory.** They do not survive a restart; recovery is a resync of + the cluster as it is then. +- **Round trips to the Git host are the seconds you wait through.** A foreign push costs a fetch, a + replay and one push. If your workload is mostly Git-side edits with occasional live changes, you + are using the tool against its grain. The measured numbers are in + [`git-roundtrip-ledger.golden`](../internal/git/testdata/git-roundtrip-ledger.golden). ## What the tests prove -The bi-directional corner runs against deployed Flux and Argo CD, each configured as this guide -says. Its timing assertions describe those scenarios; they do not guarantee the ordering of -concurrent writers. - -| Covered | Where | -| --- | --- | -| Two API edits to one field seconds apart end at the later value with no further commit, and Flux catches up with both by itself | [Flux](../test/e2e/flux_bi_directional_e2e_test.go) | -| A Git-side push with an API edit on top lands both, and Flux applies the replayed commit without being reconciled by hand | Flux | -| A new `GitRepository` revision, from a commit outside the applied folder, is enough for the `Kustomization` to apply and revert drift | Flux | -| A refused edit with `onRefusal: Reapply` is reverted by its `Kustomization`, with no commit, and settles instead of repeating | [Refusal reapply](../test/e2e/refusal_reapply_e2e_test.go) | -| A `SourceMismatch` applier is never stamped, a push the Git host reports or the refresh finds is applied, without the corner | [Manager Flux Applier](../test/e2e/flux_applier_e2e_test.go), and [the Flux guide](flux.md#what-the-tests-prove) | -| A Git to cluster to Git round trip settles without a commit loop | Flux | -| A SOPS-encrypted Secret round-trips without a reverse commit | Flux | -| A Git revert removes the live object and the committed file | Flux | -| Argo CD bookkeeping never reaches Git, asserted as zero commits during a sync | [Argo CD](../test/e2e/argocd_bi_directional_e2e_test.go) | -| Two API edits to one field seconds apart keep the later value with no further commit, and Argo CD is never asked to catch up | Argo CD | -| A Git-side push is synced by a `ForeignCommit` request, pinned to that commit | Argo CD | -| A refused edit with `onRefusal: Reapply` is reverted by an Argo CD sync | Argo CD | -| An `Application` with automated sync on is `AutomatedSyncEnabled` and is never synced | Argo CD | +Each tool guide lists what its unit and end-to-end tests cover: +[Flux](bi-directional-flux.md#what-the-tests-prove) and +[Argo CD](bi-directional-argocd.md#what-the-tests-prove). The bi-directional corner runs against +deployed Flux and Argo CD, each configured as those guides say; `task test-e2e-bi-directional` runs +it, and the [corner specification](spec/e2e-bi-directional-corner.md) describes the setup. **No spec tests simultaneous writes to one field from both sides**, because the outcome depends on -arrival order the corner does not control. The replay mechanism is covered; a conflict policy is -not, because there is not one. - -Run `task test-e2e-bi-directional` for the corner, which is also a CI job. `task test-e2e` excludes -the corner and includes the refusal spec. `task argocd-ui` opens the installed Argo CD UI. The -[corner specification](spec/e2e-bi-directional-corner.md) describes -the setup. +an arrival order the corner does not control. The replay is covered; a conflict policy is not, +because there is not one. diff --git a/docs/components.md b/docs/components.md index 756d30f51..e9b3d62fc 100644 --- a/docs/components.md +++ b/docs/components.md @@ -23,7 +23,7 @@ the target watches supply *state*, the branch worker owns *Git*, and audit only ```mermaid flowchart TB subgraph CFG["Config plane (operator's own cluster)"] - CRD["api/v1alpha3
GitTarget · WatchRule · ClusterWatchRule
GitProvider · ClusterProvider · CommitRequest"] + CRD["api/v1alpha3
GitTarget · WatchRule · ClusterWatchRule
GitProvider · ClusterProvider · CommitRequest
FluxTrigger · ArgoCDTrigger"] CTRL["internal/controller
reconcilers"] ADM["internal/webhook
admission handlers"] AUTHZ["internal/authz
namespace admission"] @@ -76,10 +76,10 @@ The operator's own API surface and the reconcilers behind it. | Component | Path | Role | |---|---|---| -| CRD types | [`api/v1alpha3/`](../api/v1alpha3/) | the six kinds users apply | +| CRD types | [`api/v1alpha3/`](../api/v1alpha3/) | the eight kinds users apply | | Reconcilers | [`internal/controller/`](../internal/controller/) | one per kind, plus shared condition and status helpers | | Admission handlers | [`internal/webhook/`](../internal/webhook/) | the always-allow observer and `validate-operator-types`, which captures a `CommitRequest` submitter | -| Namespace admission | [`internal/authz/`](../internal/authz/) | the `ClusterProvider.spec.accessFrom` decision and the `spec.allowAnySourceNamespace` gate, in their own package because several call sites need the same answer | +| Namespace admission | [`internal/authz/`](../internal/authz/) | the `ClusterProvider.spec.accessFrom` decision and the `spec.allowCrossNamespace` gates, in their own package because several call sites need the same answer | | Rule store | [`internal/rulestore/`](../internal/rulestore/) | compiled `WatchRule` and `ClusterWatchRule` cache the data plane reads | ### Type and scope resolution @@ -124,6 +124,8 @@ write to that branch goes through its single event loop. | Document editor | [`internal/git/manifestedit/`](../internal/git/manifestedit/) | edits one YAML document in place inside a multi-document file | | Manifest report | [`internal/manifestreport/`](../internal/manifestreport/) | projects a live object into a comparable manifest | | Encryption and signing | [`internal/git/sops_encryptor.go`](../internal/git/sops_encryptor.go), [`internal/sshsig/`](../internal/sshsig/) | SOPS for sensitive types, SSH signatures for commits | +| Apply requests | [`internal/applier/`](../internal/applier/) | the Flux and Argo CD clients, the dispatcher that carries out each apply attempt a branch worker decides, and the trigger's `Configured` check | +| Push receiver | [`internal/webhook/receiver.go`](../internal/webhook/receiver.go) | `/git-push/` deliveries from a Git host, handed to the branch worker as a hint to read the remote | ### Attribution @@ -187,7 +189,9 @@ flowchart LR | Target watches | **objects** of claimed types | source cluster | raw `dynamic ... Watch()`, no informer and no cache ([`internal/watch/target_watch.go:969`](../internal/watch/target_watch.go#L969)) | Two more paths reach the operator without being watches at all. The audit webhook is a **push** from -kube-apiserver, and the admission webhooks are synchronous requests. Neither observes state. +kube-apiserver, and the admission webhooks are synchronous requests. Neither observes state. The +apply trigger reconciler reads each trigger's `Kustomization` or `Application` with a GET on its own +tick rather than a watch, so a cluster without Flux or Argo CD needs no informer for them. ### Why two type observers is correct diff --git a/docs/configuration.md b/docs/configuration.md index 4a66b2a79..9dcbf9776 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,6 +13,8 @@ The short version: - `ClusterWatchRule` does the same for cluster-scoped resources - `CommitRequest` optionally saves: it closes its author's commit window on its own timers, with its own message if it has one +- `FluxTrigger` and `ArgoCDTrigger` hold the Flux `Kustomization` or Argo CD `Application` that + applies a folder back, so GitOps Reverser decides when it applies The chart's optional `quickstart` values are a convenience layer that creates starter instances of those same resources. @@ -32,10 +34,11 @@ The usual flow is: 3. Create one or more `WatchRule` or `ClusterWatchRule` objects that point at that target. 4. Create a `CommitRequest` only when you want a save: your writes in one commit, on the request's timers rather than the target's, and optionally with your message. -5. When Flux or Argo CD also applies the folder back to the cluster, name its `Kustomization` or - `Application` on the `GitTarget` and give the `GitProvider` a push receiver, so GitOps Reverser - decides when Git is applied. See [who applies this folder](#who-applies-this-folder-specapplier) and the - [bi-directional guide](bi-directional.md). +5. Only when Flux or Argo CD also applies that same folder back to the cluster: hold its + `Kustomization` or `Application` with an apply trigger, name the trigger on the `GitTarget`, and + give the `GitProvider` a push receiver, so GitOps Reverser decides when Git is applied. See + [who applies this folder](#who-applies-this-folder-specapplytriggerref) and + [bi-directional GitOps](bi-directional.md). Flux and Argo CD applying other folders need nothing. That means one repository connection can back multiple targets, and one target can be fed by multiple watch rules. @@ -48,7 +51,7 @@ multiple watch rules. ## Field overview -Ready-made manifests for all six objects live in [`config/samples/`](../config/samples/). They +Ready-made manifests for all eight objects live in [`config/samples/`](../config/samples/). They cross-reference the same `example-provider` / `example-target` names, so the set can be applied together and edited down, rather than assembled from the fragments below. @@ -73,7 +76,8 @@ fields that need more than a line. |---|---|---| | `kubeConfig` | the operator's own cluster | Points at a remote source cluster instead of the one the operator runs in | | `accessFrom` | **no namespace** | Deny-by-default list of control-cluster namespaces that may reference this provider. Omitted, none may; an empty `selector: {}` admits every namespace | -| `allowAnySourceNamespace` | `false` | Whether rules may watch namespaces other than their own, `*` included | +| `allowCrossNamespace.sources` | `false` | Whether rules may watch namespaces other than their own, `*` included. See [cross-namespace grants](#what-a-namespace-may-reach-outside-itself-specallowcrossnamespace) | +| `allowCrossNamespace.appliers` | `false` | Whether triggers may name a `Kustomization` or `Application` in another namespace than their own | | `attribution.auditRoute` | the provider's own name | Audit route this provider joins. See [audit route](#audit-route-and-auditfactsreceived) | | `client.qps` | `20` | Client rate limit for this source cluster. Ignored when `kubeConfig` is omitted | | `client.burst` | `30` | Client burst for this source cluster. Ignored when `kubeConfig` is omitted | @@ -97,7 +101,7 @@ The destination fields are immutable: to move a target, delete it and create a n | `placement.useKustomize` | off | Keep the folder a kustomize folder. See [keeping the folder a kustomize folder](#keeping-the-folder-a-kustomize-folder-specplacementusekustomize) | | `serializeNamespace` | inferred per document | Whether a committed document carries its own `metadata.namespace`. An explicit `false` admits exactly one source namespace. See [whether documents carry their namespace](#whether-documents-carry-their-namespace-specserializenamespace) | | `prune.mode` | `OnEvent` | Which deletions reach Git. See [deletion policy](#deletion-policy-specprunemode) | -| `applier` | the applier applies on its own schedule | The Flux `Kustomization` (`kustomizationRef`) or Argo CD `Application` (`applicationRef`) that applies this folder, which GitOps Reverser then asks to apply, and for a `Kustomization` how long the branch stays quiet before it catches up (`quietPeriod`, `30s`). See [who applies this folder](#who-applies-this-folder-specapplier) | +| `applyTriggerRef` | the applier applies on its own schedule | The `FluxTrigger` or `ArgoCDTrigger`, in this namespace, holding the applier of this folder, which GitOps Reverser then asks to apply. See [who applies this folder](#who-applies-this-folder-specapplytriggerref) | | `onRefusal` | `Ignore` | What happens to an edit with no legal destination. See [reverting a refused edit](#reverting-a-refused-edit-speconrefusal) | | `encryption` | off | SOPS + age encryption for sensitive resources | | `suspend` | `false` | Stop writing without deleting the target. See [stopping a target from writing](#stopping-a-target-from-writing-specsuspend) | @@ -125,6 +129,25 @@ The same shape as a `WatchRule`, for cluster-scoped types. It selects no namespa | `rules[].apiVersions` | the preferred served version | API versions to match. `["*"]` watches every served version | | `rules[].objectSelector` | every object | Label selector the API server applies to the item's LIST and WATCH. See [selecting objects by label](#selecting-objects-by-label-rulesobjectselector) | +### `FluxTrigger` (namespaced): the Flux `Kustomization` GitOps Reverser asks to apply + +The references are immutable: to hold another `Kustomization`, delete the trigger and create a new one. + +| Field | Default | What it does | +|---|---|---| +| `clusterProviderRef` | `{"name":"default"}` | The `ClusterProvider` whose cluster holds the `Kustomization`. Its `accessFrom` must admit the trigger's namespace | +| `kustomizationRef` | **required** | The `Kustomization` that applies the folders of the `GitTarget`s naming this trigger. See [who applies this folder](#who-applies-this-folder-specapplytriggerref) | +| `quietPeriod` | `30s` | How long the branch must stay quiet after GitOps Reverser's own pushes before a `CatchUp` | + +### `ArgoCDTrigger` (namespaced): the Argo CD `Application` GitOps Reverser asks to sync + +The references are immutable: to hold another `Application`, delete the trigger and create a new one. + +| Field | Default | What it does | +|---|---|---| +| `clusterProviderRef` | `{"name":"default"}` | The `ClusterProvider` whose cluster holds the `Application`. Its `accessFrom` must admit the trigger's namespace | +| `applicationRef` | **required** | The `Application`, with automated sync off, that syncs the folders of the `GitTarget`s naming this trigger. See [who applies this folder](#who-applies-this-folder-specapplytriggerref) | + ### `CommitRequest` (namespaced): save now | Field | Default | What it does | @@ -489,22 +512,39 @@ with the shown label, may reference `prod-eu-1`. `names` and `selector` are ORed policy admits no control-cluster namespace. Which namespaces are read *from the source cluster* is bounded by the source connection's Kubernetes -RBAC: the credential's own RBAC is the hard maximum, and there is no second allow-list beside it. A -`WatchRule` may name a source namespace other than its own only when this provider also sets: +RBAC: the credential's own RBAC is the hard maximum, and there is no second allow-list beside it. + +### What a namespace may reach outside itself (`spec.allowCrossNamespace`) ```yaml - # Deny-by-default. While false, a WatchRule mirroring through this provider may - # watch only its OWN namespace. - allowAnySourceNamespace: true + allowCrossNamespace: # what an admitted namespace may reach outside itself + sources: false # a WatchRule may watch other namespaces, "*" included + appliers: false # a trigger may name a Kustomization or Application in another namespace ``` -That flag delegates the *choice* of source namespace to the `GitTarget`s this provider admits; it -grants nothing on its own, since the credential still has to be able to read what is chosen. It is -required for **every** cross-source-namespace request, including `sourceNamespace: "*"`. Setting it on an -**in-cluster** provider is a much sharper decision than on a remote one: there the config plane *is* -the watched cluster, so it deliberately bypasses live namespace RBAC and lets the owner of an -admitted `GitTarget` mirror another namespace's objects into a Git destination they control. That is -legitimate to grant on purpose, which is why it is explicit and defaults to false. +Both default to `false`, and each is a power of its own: + +- **`sources`** lets a namespace *read* other namespaces' objects into its own Git. A `WatchRule` may + name a source namespace other than its own only while it is `true`; while it is `false`, it + watches only its own namespace, and `sourceNamespace: "*"` is refused outright. It delegates the + *choice* of source namespace and grants nothing on its own, since the credential still has to be + able to read what is chosen. +- **`appliers`** lets a namespace *suspend and apply* another namespace's deployments: a + `FluxTrigger` or `ArgoCDTrigger` may name a `Kustomization` or `Application` outside its own + namespace, such as `flux-system`. While it is `false`, a trigger naming one elsewhere reports + `Configured=False` (`ApplierNamespaceNotAllowed`). The trigger's own namespace is always allowed; + in a remote cluster that is the namespace of the same name. + +**It is a delegation, not owner consent.** You trust every namespace `accessFrom` admits with +everything in reach of the provider's identity. Neither flag proves that a namespace owns the +objects it reaches, or the repository its `GitTarget` writes. Setting either on an **in-cluster** +provider is a much sharper decision than on a remote one: there the config plane *is* the watched +cluster, so it deliberately bypasses live namespace RBAC and lets the owner of an admitted object +act on another namespace through the operator's own credential. That is legitimate to grant on +purpose, which is why each is explicit and defaults to false. + +`allowAnySourceNamespace`, which `sources` replaces, is no longer read. Setting it to `true` is +refused; see [UPGRADING](UPGRADING.md). `spec.kubeConfig` and `GitTarget.spec.clusterProviderRef` are immutable: changing either would silently make an existing materialization mean a different source cluster. Rotate credential *contents* in the @@ -634,10 +674,10 @@ The important fields are: omit it and each document's namespace is inferred from the folder - `spec.prune`: which deletion paths may remove documents from this target's folder (see [Deletion policy](#deletion-policy-specprunemode)); omit it for the safe default -- `spec.applier`: the Flux `Kustomization` or Argo CD `Application` that applies this folder, so - GitOps Reverser decides when it applies (see - [Who applies this folder](#who-applies-this-folder-specapplier)); omit it when nothing applies the - folder back +- `spec.applyTriggerRef`: the `FluxTrigger` or `ArgoCDTrigger` holding the Flux `Kustomization` or + Argo CD `Application` that applies this folder, so GitOps Reverser decides when it applies (see + [Who applies this folder](#who-applies-this-folder-specapplytriggerref)); omit it when nothing + applies the folder back - `spec.onRefusal`: whether a refused edit asks the applier to reapply Git (see [Reverting a refused edit](#reverting-a-refused-edit-speconrefusal)) @@ -694,9 +734,9 @@ The most useful status fields are: - `status.retention`: how many documents `spec.prune.mode` is keeping, and under which mode. - `LayoutResolved`: what the last scan resolved about the folder's shape, with `status.placement` carrying the detail. See below. -- `ApplierConfigured`: true when the declared applier exists, applies this folder's repository and - branch, and can be asked to apply. Present only when `spec.applier` is set. -- `status.applier`: the last apply request GitOps Reverser made, and why. +- `ApplyTriggerReady`: true when the named trigger is `Ready` and its applier applies this folder's + repository and branch. Present only when `spec.applyTriggerRef` is set. The trigger's own + `status.operation` says what was last asked of the applier, and why. Use conditions for automation. @@ -721,7 +761,7 @@ summary reason, while the domain conditions name the axis to fix: | `ClusterProviderReady=False` or `SourceClusterReachable=False` | Source-cluster credentials and access. | | `GitPathAccepted=False` | Repository files under `spec.path`; the message includes the issue kind, path, and actor when known. | | `RenderMatchesLive=False` | Fields that no longer match the rendered source, often transformer-owned values. | -| `ApplierConfigured=False` | The applier named in `spec.applier`, its source, its automated sync (an `Application`), and the permission to write it. Commits still flow; apply requests may not. | +| `ApplyTriggerReady=False` | The trigger named in `spec.applyTriggerRef`, and its applier: its source, its automated sync (an `Application`), and the permission to write it. Commits still flow; apply requests may not. | A green `GitTarget` with a red `WatchRule` or `ClusterWatchRule` means the destination may be fine, but the selected source objects are not being watched or authorized. Inspect the rule's `Ready`, @@ -915,9 +955,10 @@ suppressed while it was set are not replayed, so what lands is what the cluster backlog of the values it passed through. **Suspend stops writes, not deployments.** A suspended target's applier is still asked to apply a -commit somebody else pushed, because with [`spec.applier`](#who-applies-this-folder-specapplier) -set nothing else asks it, and a paused mirror must not stall Git-side changes. It requests nothing -for a refusal, since that request exists to act on a live edit the target is no longer publishing. +commit somebody else pushed, because with +[`spec.applyTriggerRef`](#who-applies-this-folder-specapplytriggerref) set nothing else asks it, +and a paused mirror must not stall Git-side changes. It requests nothing for a refusal, since that +request exists to act on a live edit the target is no longer publishing. To re-read the folder now rather than on the periodic cadence, stamp the reconcile-request annotation with any value that changes: @@ -1093,161 +1134,193 @@ tick". [`gitopsreverser_git_fetches_total{reason="refresh"}`](interpreting-metri is what it costs. A refresh that finds a commit GitOps Reverser did not push is a foreign commit like any other: the -target's [applier](#who-applies-this-folder-specapplier) is asked to apply it once the branch is +target's [applier](#who-applies-this-folder-specapplytriggerref) is asked to apply it once the branch is quiet. That makes the refresh the backstop for deployments when no push webhook reaches the operator. It is not a latency mechanism: it buys bounded staleness with no inbound surface and no Git-host configuration. Seconds-fresh is the job of the [push receiver](#receiving-push-notifications-specreceiver). -### Who applies this folder (`spec.applier`) +### Who applies this folder (`spec.applyTriggerRef`) -When a GitOps tool applies this folder back to the cluster, `spec.applier` names the object that -does it, and GitOps Reverser becomes the one that decides when it applies. The tool keeps doing the -fetching, rendering, applying and pruning. The reason it has to be this way is the -[apply trigger design](design/apply-trigger.md); the [bi-directional guide](bi-directional.md) is -the model, [Using GitOps Reverser with Flux](flux.md) the whole Flux setup, and -[Using GitOps Reverser with Argo CD](argocd.md) the whole Argo CD setup. +Set this only when Flux or Argo CD applies the folder this `GitTarget` writes. A folder nothing +applies, and every folder Flux or Argo CD applies that GitOps Reverser does not write, needs no +trigger: leave the field out, and GitOps Reverser never touches Flux or Argo CD. -```yaml -spec: - applier: - kustomizationRef: - namespace: flux-system # default: this GitTarget's namespace - name: checkout - quietPeriod: 30s -``` +When the folder is shared, an **apply trigger** holds the object that applies it (its applier), and +GitOps Reverser decides when it applies: a `FluxTrigger` holds a Flux `Kustomization`, an +`ArgoCDTrigger` an Argo CD `Application`. The `GitTarget` names the trigger, in its own namespace. +[Bi-directional GitOps](bi-directional.md) is the model, and the complete setups, with the +lifecycle of an apply, every condition reason and the permissions, are +[bi-directional GitOps with Flux](bi-directional-flux.md) and +[bi-directional GitOps with Argo CD](bi-directional-argocd.md). The reason it has to be this way is +the [apply trigger design](design/apply-trigger.md). + +**A trigger acts only while an eligible `GitTarget` references it**: one in its namespace that names +it, is admitted by its `ClusterProvider`s, and writes the repository and branch the applier fetches. +Until one does, the trigger reads and writes nothing in the applier's cluster, and reports +`Configured=Unknown` and `Ready=True`, both `NotReferenced`, so the two may be created in either +order. A `FluxTrigger` that held its `Kustomization` hands it back once no eligible `GitTarget` +references it any more. ```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: FluxTrigger +metadata: + name: checkout + namespace: team-a +spec: + clusterProviderRef: + name: default # the cluster Flux runs in + kustomizationRef: + namespace: flux-system # default: the trigger's namespace + name: checkout + quietPeriod: 30s +--- +apiVersion: configbutler.ai/v1alpha3 +kind: GitTarget +metadata: + name: checkout + namespace: team-a spec: - applier: - applicationRef: - namespace: argocd # default: this GitTarget's namespace - name: checkout + # gitProviderRef, branch, path ... + applyTriggerRef: + kind: FluxTrigger # or ArgoCDTrigger + name: checkout ``` | Field | Default | What it does | | --- | --- | --- | -| `kustomizationRef` | one of the two | A Flux `Kustomization` whose `sourceRef` is a `GitRepository` on this target's repository and branch | -| `applicationRef` | one of the two | An Argo CD `Application` on a single source, with this target's repository and branch, and automated sync off | -| `quietPeriod` | `30s` | How long the branch must stay quiet after GitOps Reverser's own pushes before a `CatchUp` request. At most `1h`. Only with a `kustomizationRef`: rejected with an `applicationRef` | - -**The appliers.** A Flux `Kustomization` (`kustomizationRef`) and an Argo CD `Application` -(`applicationRef`); exactly one of the two. The `Application` runs with automated sync off -(`syncPolicy.automated.enabled: false`, or no `automated` block), and every sync it runs is one -GitOps Reverser starts, pinned to the commit it checked. It is never asked to catch up with GitOps -Reverser's own commits, since with automated sync off it applies nothing by itself. One with -automated sync on is `ApplierConfigured=False` with `AutomatedSyncEnabled`, and is never synced. - -A third is the aim, and the API does not accept it yet: **a webhook (`webhook`)** for any other tool, -a POST of `{"repository", "branch", "commit", "reason"}`, signed the way Flux's `generic-hmac` -`Receiver` checks. - -**Where the reference is looked up.** In the cluster `spec.applier.clusterProviderRef` names, with -the identity that [`ClusterProvider`](#clusterprovider) uses. Omitted, it is the `GitTarget`'s own -`spec.clusterProviderRef`, the source cluster. Name another provider when Flux or Argo CD runs in a -management cluster and applies to the source cluster from there; a provider without `spec.kubeConfig` is the -operator's own cluster. That provider's `spec.accessFrom` must admit the `GitTarget`'s namespace, -as for mirroring, or `ApplierConfigured` is `False` with `NamespaceNotAuthorized` (or -`ClusterProviderNotFound`) and nothing is asked. +| `FluxTrigger.spec.clusterProviderRef`, `ArgoCDTrigger.spec.clusterProviderRef` | `{name: default}` | The `ClusterProvider` whose cluster holds the applier, defaulted as a `GitTarget`'s is. Immutable | +| `FluxTrigger.spec.kustomizationRef` | required | A Flux `Kustomization` whose `sourceRef` is a `GitRepository` that follows a branch. Immutable | +| `FluxTrigger.spec.quietPeriod` | `30s` | How long the branch must stay quiet after GitOps Reverser's own pushes, or after a `GitTarget` first names the trigger, before a `CatchUp` request. At most `1h` | +| `ArgoCDTrigger.spec.applicationRef` | required | An Argo CD `Application` on a single source whose `targetRevision` names a branch, with automated sync off. Immutable | +| `GitTarget.spec.applyTriggerRef` | none | The trigger, by `kind` and `name`, in the `GitTarget`'s namespace | + +**One trigger, several folders.** Several `GitTarget`s may name one trigger: their folders are +applied by the same applier, and it is asked once for all of them. Each one's repository and branch +must be the ones the applier applies, or that `GitTarget`'s `ApplyTriggerReady` is `False` with +`SourceMismatch`; one that matches is enough for the trigger to act. One applier is held by one +trigger: a second trigger of the same kind in line for the same applier in the same cluster reports +`HeldByAnotherTrigger` and does nothing. A trigger no eligible `GitTarget` references is not in line, +so it keeps no other out. + +**Where the applier is looked up.** In the cluster the trigger's `spec.clusterProviderRef` names, +with the identity that [`ClusterProvider`](#clusterprovider) uses. Name a provider for a management +cluster when Flux or Argo CD runs there; a provider without `spec.kubeConfig` is the operator's own +cluster. That provider's `spec.accessFrom` must admit the trigger's namespace, as for mirroring, or +the trigger is `Configured=False` with `NamespaceNotAuthorized` (or `ClusterProviderNotFound`). An +applier in another namespace than the trigger's, such as `flux-system` in the example above, also +needs the provider's [`allowCrossNamespace.appliers`](#what-a-namespace-may-reach-outside-itself-specallowcrossnamespace), +or the trigger is `Configured=False` with `ApplierNamespaceNotAllowed`. For the operator's own +cluster, the chart grants the applier permissions with `applierAccess.flux: true` and +`applierAccess.argocd: true`. ```yaml +apiVersion: configbutler.ai/v1alpha3 +kind: FluxTrigger +metadata: + name: prod-eu-1-apps + namespace: team-a spec: clusterProviderRef: - name: prod-eu-1 # mirrored from here - applier: - clusterProviderRef: - name: management # Flux runs here - kustomizationRef: - namespace: flux-system - name: prod-eu-1-apps + name: management # Flux runs here, and applies to prod-eu-1 + kustomizationRef: + namespace: flux-system + name: prod-eu-1-apps ``` -For a `Kustomization`, the identity needs `get` and `patch` on `kustomizations` -(`kustomize.toolkit.fluxcd.io`) and `gitrepositories` (`source.toolkit.fluxcd.io`); for an -`Application`, `get` and `update` on `applications` (`argoproj.io`); and `create` on -`selfsubjectaccessreviews` to check it. For the operator's own cluster the chart grants them with -`applierAccess.flux: true` and `applierAccess.argocd: true`; for a remote cluster, grant the same to -the kubeconfig's identity. - **When a request is made.** Only on a quiet branch: no open commit window and no pending writes on the branch worker, or on any other worker writing the same repository and branch through another -`GitProvider`. That is the moment nothing live is missing from the tip. +`GitProvider`. | `reason` | When | Sent to | | --- | --- | --- | | `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. Open windows are closed and pending writes replayed and pushed first, so the request names that pushed commit | Every applier declared on the branch, once per applier | | `Refusal` | An eligible refused edit, with `spec.onRefusal: Reapply` | This target's applier | -| `CatchUp` | The branch has been quiet for `quietPeriod` since GitOps Reverser's own pushes, or since the operator started | Every `Kustomization` declared on the branch; never an `Application` | - -Flux applies on its own timers and on a controller restart, from whatever artifact the -`GitRepository` holds. `CatchUp` keeps that artifact at the tip, so such an apply applies values the -cluster already holds; an artifact left behind would revert edits that are already in Git. -`quietPeriod` keeps the catch-up out of an editing session: every new write restarts it. When several -`GitTarget`s name one `Kustomization`, it waits for the longest of their quiet periods, since its -apply reaches all their folders, and it is asked once for all of them. - -**What a request does to a `Kustomization`.** What `flux reconcile kustomization --with-source` does: stamp -`reconcile.fluxcd.io/requestedAt` on the `GitRepository` when its artifact does not hold the -requested commit; the new artifact wakes the `Kustomization` by itself. When the artifact already -holds the commit, only a `Refusal` stamps the `Kustomization`, to revert drift at an unchanged -commit; a `ForeignCommit` or `CatchUp` stamps nothing, since that artifact already woke it. Nothing -is stamped when, by the time of the patch, the commit is no longer the branch tip or an edit is -waiting to be published: the request is decided again at the next quiet moment. - -**What a request does to an `Application`.** It sets the `Application`'s `operation` to the sync -Argo CD's automated sync would start (`prune` from `syncPolicy.automated`, `syncOptions` and `retry` -from `syncPolicy`), with `revision` pinned to the requested commit and `initiatedBy` -`{username: gitops-reverser, automated: true}`. It never replaces an operation already running: it -waits for it to end, at most 5 minutes. The details are in [the Argo CD guide](argocd.md#what-gitops-reverser-asks-and-when). - -A request asks; it never claims the apply happened. It runs off the publication path, and a failed -request never fails a publication that succeeded. Read the `Kustomization`'s -`status.lastAppliedRevision`, or the `Application`'s `status.operationState`, for what it applied. - -**Every attempt checks again.** Just before it patches, GitOps Reverser reads each `GitTarget` again -and asks nothing for one that has been deleted or recreated, no longer names this applier on this -branch, is no longer admitted by its `ClusterProvider`, or, for a `Refusal`, no longer consents -(`onRefusal: Reapply`, not suspended). It never asks for something weaker in place of what it was -told; when no target still holds the request's reason it asks nothing. And nothing is asked unless -the `Kustomization`'s `GitRepository` fetches this repository and branch, or the `Application` syncs -it with automated sync off, whatever `ApplierConfigured` last said. - -**A request that fails is decided again from scratch.** The branch worker keeps what the -applier is owed until a request delivers it. After a failure it waits out that applier's own backoff -(2 seconds doubling to 5 minutes, at least 15 seconds when the applier cannot take a -request), then waits for a quiet branch like any first request, and asks for the tip as it is then. -A refusal that recovered in the meantime is not asked for again. - -**What it reports.** +| `Repair` | An apply reverted an API edit made after it was asked for; Git kept the edit | That applier | +| `CatchUp` | The branch has been quiet for `quietPeriod` since GitOps Reverser's own pushes, since the operator started, or since a `GitTarget` first named the trigger | Every `Kustomization` declared on the branch; never an `Application` | + +**What it reports.** The trigger reports its applier and the operation it is running. The branch +worker writes `status.operation` and `status.repairPending` at every move of the operation, so they are +current without anything reconciling: ```yaml +kind: FluxTrigger status: conditions: - - type: ApplierConfigured - status: "False" - reason: SourceMismatch - message: 'GitRepository flux-system/infra fetches branch "release", not main' - applier: - requestedCommit: 4f2c1ab9... + - type: Configured + status: "True" + reason: Succeeded + message: Kustomization flux-system/checkout applies GitRepository flux-system/platform, ... + - type: Held # FluxTrigger only + status: "True" + reason: Suspended # Suspended | Applying | Suspending | Releasing | NotReferenced | NotConfigured + - type: Ready + status: "True" + source: + gitRepository: {namespace: flux-system, name: platform} + url: https://github.com/acme/platform + branch: main + otherConsumers: # what else applies this GitRepository, held by no FluxTrigger + - {kind: HelmRelease, namespace: platform, name: ingress} + prefetch: false + operation: + phase: Succeeded # Requested | Running | Settling | Unresolved | Succeeded | Failed + reason: ForeignCommit # ForeignCommit | Refusal | Repair | CatchUp + commit: 4f2c1ab9... + token: "2026-10-20T10:14:03.123456789Z" + suspendToken: "2026-10-20T10:14:20.987654321Z" + appliedRevision: main@sha1:4f2c1ab9... requestedAt: "2026-10-20T10:14:03Z" - reason: ForeignCommit # ForeignCommit | Refusal | CatchUp + finishedAt: "2026-10-20T10:14:21Z" + repairPending: false ``` -`ApplierConfigured` is a gate, so `Ready` aggregates it. Its `False` reasons: - -| Reason | Means | +| Field | Means | | --- | --- | -| `ApplierNotFound` | No such `Kustomization` or `Application` in the applier's cluster, or Flux or Argo CD is not installed there | -| `SourceMismatch` | The `Kustomization` applies a different repository or branch, or a source that is not a `GitRepository`; or the `Application` syncs a different repository or revision, or more than one source | -| `AutomatedSyncEnabled` | The `Application` has automated sync on | -| `Forbidden` | The applier cluster's identity may not patch the `Kustomization` or its `GitRepository`, or update the `Application` | -| `NamespaceNotAuthorized` | `spec.applier.clusterProviderRef` names a provider whose `accessFrom` does not admit this namespace | -| `ClusterProviderNotFound` | `spec.applier.clusterProviderRef` names a provider that does not exist | - -It is checked on the target's reconcile tick rather than watched, so a cluster without the Flux or -Argo CD CRDs installed costs nothing. Each delivered request also records an `ApplyRequested` Event on the -`GitTarget`, and a request that could not be delivered records `ApplyRequestFailed` once, not again -each time the worker asks again. +| `operation.phase` | `Requested`, `Running`, `Settling` (Flux only: applied, not yet seen suspended again) and `Unresolved` (not completed in 5 minutes, still followed) are in flight; `Succeeded` and `Failed` have completed. [The life of an apply](bi-directional.md#the-life-of-an-apply) draws the transitions | +| `operation.token` | Identifies the request in the applier's own status: the `reconcile.fluxcd.io/requestedAt` value of the resume, or the `gitops-reverser request` entry in an Argo CD operation's `info` | +| `operation.suspendToken` | Flux only: the token of the suspend after the apply. Once it is handled, nothing that started before the suspend is still running | +| `operation.appliedRevision` | The revision the applier reports it applied. It can differ from `commit` when the branch moved between the check and the apply | +| `repairPending` | `true` while an apply reverted an API edit made after it was asked for: Git kept the edit, and the cluster runs the older value until the `Repair` | +| `source.otherConsumers` | Flux only: the `Kustomization`s and `HelmRelease`s that apply the same `GitRepository` and that no `FluxTrigger` holds. Information only; it never affects `Ready` | +| `source.prefetch` | Flux only: `true` while every consumer of the source is held, so GitOps Reverser may fetch the `GitRepository` ahead of the apply | + +A `Kustomization` resumed outside GitOps Reverser (`flux resume`) is suspended again and recorded as +an `Unresolved` operation with no `reason` and no `commit`, which completes once the suspend is +handled. + +**Conditions.** `Configured` is a gate, and the trigger's `Ready` aggregates it; a `FluxTrigger` is +also `Ready` only once `Held` is `True` (`Suspended`, or `Applying` while an apply of GitOps +Reverser's is in flight). Either kind is `Ready=False` (`OperationUnresolved`) while its operation +is `Unresolved`. An unreferenced trigger is `Ready=True` (`NotReferenced`). The +`GitTarget`s naming the trigger are reported as `NotBound` (`Unknown` while one cannot be judged +yet, `False` once each is refused for different reasons), or as the refusal they share, +`SourceMismatch` or `NamespaceNotAuthorized`, every `GitTarget` and its reason in the message. The +other `False` reasons are `ApplierNotFound`, `UnsupportedSource`, `AutomatedSyncEnabled` (Argo CD), +`SuspendOwnedElsewhere` (Flux), `Forbidden`, `NamespaceNotAuthorized` (the trigger's own namespace), +`ClusterProviderNotFound`, `ApplierNamespaceNotAllowed` and `HeldByAnotherTrigger`; each guide's +table says what they mean for its applier and how to fix them +([Flux](bi-directional-flux.md#the-fluxtrigger), [Argo CD](bi-directional-argocd.md#the-argocdtrigger)). +The `GitTarget` reports `ApplyTriggerReady`, and its `Ready` aggregates it: `True` when the trigger +is `Ready` and its applier applies this target's repository and branch, otherwise the trigger's own +`Ready` reason, `ApplyTriggerNotFound`, `SourceMismatch`, or `Unknown` (`NotReferenced`) until the +trigger has seen that the `GitTarget` names it. Each delivered request records an +`ApplyRequested` Event on the `GitTarget`, and one that could not be delivered records +`ApplyRequestFailed` once. + +The applier is read on the trigger's own reconcile tick rather than watched, so a cluster without +the Flux or Argo CD CRDs costs nothing: every 15 seconds while it is expected to clear on its own, +such as an applier created after its trigger, and otherwise every minute for a `FluxTrigger` (that +read is what finds a `flux resume`) and every 5 minutes for an `ArgoCDTrigger`. A `FluxTrigger` that +is `Suspending` is read every 2 seconds. Annotate the trigger with +`reconcile.configbutler.ai/requestedAt` to have it read now. + +**Deleting a `FluxTrigger` hands the `Kustomization` back to Flux,** through the finalizer +`configbutler.ai/flux-trigger`, unless another `FluxTrigger` is in line for it, and so does the last +eligible `GitTarget` that stops naming it. A trigger that never held its `Kustomization` carries no +finalizer. The [Flux guide](bi-directional-flux.md#how-the-trigger-holds-the-kustomization) has the +details. ### Reverting a refused edit (`spec.onRefusal`) @@ -1258,26 +1331,26 @@ something applies Git over it. `spec.onRefusal` decides whether GitOps Reverser ```yaml spec: path: apps/checkout - applier: - kustomizationRef: {namespace: flux-system, name: checkout} + applyTriggerRef: {kind: FluxTrigger, name: checkout} onRefusal: Reapply # default: Ignore ``` | Value | What happens | | --- | --- | | `Ignore` (default) | The refusal is reported and nothing else happens | -| `Reapply` | Once the branch is quiet, this target's [applier](#who-applies-this-folder-specapplier) is asked to apply the current tip, with reason `Refusal`. The refused edit is reverted when the applier applies it; the request asks, and the applier's own status says whether it did | +| `Reapply` | Once the branch is quiet, this target's [applier](#who-applies-this-folder-specapplytriggerref) is asked to apply the current tip, with reason `Refusal`. The refused edit is reverted when the applier applies it; the request asks, and the applier's own status says whether it did | -`Reapply` requires `spec.applier`; a target without one is rejected at admission. The request goes +`Reapply` requires `spec.applyTriggerRef`; a target without one is rejected at admission. The request goes to this target's applier alone, and it waits for a quiet branch, so an allowed edit still in a commit window is published before the apply and is not reverted with the refused one. An `Application` is asked to sync the tip. When a `Kustomization`'s `GitRepository` has to fetch to -reach the tip, every `Kustomization` on that `GitRepository` applies the new revision, as it would -for any push; give the folder its own `GitRepository` if the apply must reach it alone. The +reach the tip, the `Kustomization`s a `FluxTrigger` holds stay suspended and ignore the new artifact; +only the consumers no trigger holds, listed in the trigger's `status.source.otherConsumers`, apply +it, as they would for any push. The [bi-directional guide](bi-directional.md#choosing-speconrefusal) covers when to choose it. **It only fires for an edit to a document the folder already holds, and never for a write that -removes one.** That is the pruning fence. Re-applying corrects drift on an object Git manages, +removes one.** That is the pruning guard. Re-applying corrects drift on an object Git manages, which is the point. For an object Git does not manage, re-applying either does nothing (Flux prunes from its inventory and Argo CD from the resources it tracks, and a live-created object is in neither) or, if the object was managed and has since been removed from Git, prunes it. Hurrying @@ -1966,7 +2039,7 @@ Naming anything other than the rule's own namespace, **including `"*"`**, is aut things, both of which must hold: 1. the `GitTarget`'s namespace is admitted by its `ClusterProvider`'s `accessFrom`; and -2. that `ClusterProvider` sets `allowAnySourceNamespace: true`. +2. that `ClusterProvider` sets `allowCrossNamespace.sources: true`. There is no third condition on the `GitTarget`. What a target may read from its source cluster is bounded by that cluster's RBAC for the provider's credential: a request these two conditions permit @@ -1992,7 +2065,7 @@ spec: `"*"` is one cluster-wide list and one cluster-wide watch per matched type, not one of each per namespace. Connection count is bounded per matched type, but object count, traffic, and memory still grow with the cluster. Access is bounded by the source credential's RBAC -and by nothing else, which is why it is refused outright while `allowAnySourceNamespace` is false. +and by nothing else, which is why it is refused outright while `allowCrossNamespace.sources` is false. A `"*"` item and a named-namespace item for the same type **overlap**, so one `GitTarget` cannot hold both: the newer rule is refused (see [overlapping collections](#overlapping-collections)). Keep @@ -2002,7 +2075,7 @@ The outcome for all items is aggregated into one `SourceNamespaceAuthorized` con `kubectl get watchrules -o wide`. A **denied** explicit name refuses the whole `WatchRule` (`Ready=False`, `Stalled=True`, no streams) rather than silently trimming that item and mirroring part of what you asked for; the message names the failing item by index and by what it selects. -Authorization is re-evaluated on every reconcile, so withdrawing `allowAnySourceNamespace` revokes a +Authorization is re-evaluated on every reconcile, so withdrawing `allowCrossNamespace.sources` revokes a running rule rather than only affecting new ones. This changes only which namespace is **watched**. Git placement always follows each mirrored @@ -2541,7 +2614,9 @@ present the client certificate the audit listener requires, so the two cannot sh | Flag | Helm value | Default | What it does | | --- | --- | --- | --- | -| `--receiver-bind-address` | `receiver.bindAddress` | `0` (off); Helm: empty (off) | `host:port` the receiver listens on, such as `:9292`. `0` or empty serves no receiver | +| (none) | `servers.receiver.enabled` | `false` | Serve the receiver; the chart then passes the flag below | +| `--receiver-bind-address` | `servers.receiver.bindAddress` | `0` (off); Helm: `:9292` | `host:port` the receiver listens on. `0` or empty serves no receiver | +| (none) | `servers.receiver.port` | `9292` | The container port; must match the port in `bindAddress` | The chart creates a `-receiver` Service when the listener is on. Expose it to your Git host with an Ingress or a Gateway route of your own that terminates TLS, the way Flux's diff --git a/docs/definitions.md b/docs/definitions.md index a9a87fd8b..b1a69e467 100644 --- a/docs/definitions.md +++ b/docs/definitions.md @@ -40,7 +40,7 @@ Almost every naming mistake in this project's history came from one ambiguity: t without a side, or "namespace" without a direction. Fix the side first and the rest of the vocabulary follows. -**Control cluster.** The cluster the operator runs in, and where its own six kinds live. When a +**Control cluster.** The cluster the operator runs in, and where its own eight kinds live. When a document says "a namespace" with no qualifier, it means a control-cluster namespace. **Source cluster.** The cluster whose objects are mirrored into Git. A `ClusterProvider` names one. @@ -61,7 +61,7 @@ The two directions, which share the word "source" and must not share anything el `ClusterProvider` from a `GitTarget`. `spec.accessFrom` (the spelling is Flux's, on purpose). - **Source namespace** is the source-cluster direction: which source-cluster namespace a rule reads objects from. `WatchRule.spec.rules[].sourceNamespace`, gated by - `ClusterProvider.spec.allowAnySourceNamespace`. + `ClusterProvider.spec.allowCrossNamespace.sources`. A sentence about namespace policy names one of those two. Never "namespace access" unqualified. @@ -190,16 +190,46 @@ and the two are not interchangeable. ## Applying back -**Applier.** The object that applies a folder to its source cluster: a Flux `Kustomization`, an -Argo CD `Application`, or a webhook standing in for another tool. `GitTarget.spec.applier` names -it. Not "reconciler", which also names this operator's own controllers, and not "orchestrator", -which says nothing about what it does to the cluster. +**Applier.** The object that applies a folder to its source cluster: a Flux `Kustomization` or an +Argo CD `Application`. An apply trigger names it. Not +"reconciler", which also names this operator's own controllers, and not "orchestrator", which says +nothing about what it does to the cluster. + +**Apply trigger.** This operator's resource that holds one applier and starts every apply it makes: a +`FluxTrigger` (`spec.kustomizationRef`) or an `ArgoCDTrigger` (`spec.applicationRef`). A +`GitTarget` names one with `spec.applyTriggerRef` and reports it in its `ApplyTriggerReady` +condition. The trigger is never called the applier: the applier is the object it triggers. Not +"Argo trigger", which names a trigger of Argo Events. + +**Eligible.** A `GitTarget` is eligible for the trigger it names when it is in the trigger's +namespace, not being deleted, admitted by its `ClusterProvider`s, and writes the repository and +branch the applier fetches. A trigger acts only while an eligible `GitTarget` references it; one +nothing references is `NotReferenced`. Eligibility is judged from those inputs alone, never from +the `GitTarget`'s `Ready`. "Bound" names a trigger with at least one eligible `GitTarget`. + +**In line.** The triggers that may hold one applier: those an eligible `GitTarget` references, and a +`FluxTrigger` that holds the `Kustomization` or is handing it back. One of them holds it; the +others report `HeldByAnotherTrigger`. + +**Held.** How a `FluxTrigger` reports its `Kustomization`, from the trigger's side: `Suspended` +(suspended under GitOps Reverser's field manager, the last suspend handled, nothing applies), +`Applying` (resumed for an apply GitOps Reverser started), `Suspending` (a suspend was sent, and +Flux has not handled its token yet), `Releasing` (being handed back to Flux), `NotReferenced` +(nothing to hold) and `NotConfigured`. A suspend counts as done only once it is handled: by then no +reconcile that started before it is still running. Not "fenced", and not "resumed", which says what +happens to the `Kustomization` rather than what the trigger does. **Hand back** is releasing +`spec.suspend`, so Flux applies on its own again. **Apply request.** What GitOps Reverser sends an applier: apply this commit, for this reason -(`ForeignCommit`, `Refusal`, `CatchUp`). It asks; it never reports that the apply happened. +(`ForeignCommit`, `Refusal`, `Repair`, `CatchUp`). It asks; the operation it starts is followed +until the applier's status shows the apply completed. A reason is **pending** for an applier from +the moment the branch worker decides it until an attempt **carries** it; an operation that +succeeds **answers** it, and one that does not gives it back to pending. A pending `Repair` shows +on the trigger as `status.repairPending`. -**Quiet branch.** A branch whose worker holds no open commit window and no pending writes. Apply -requests go out only on a quiet branch, because only then is nothing live missing from the tip. +**Quiet branch.** A branch on which no branch worker writing that repository and branch holds an +open commit window or pending writes. Apply requests go out only on a quiet branch, because only +then is nothing live missing from the tip. **Receiver.** `GitProvider.spec.receiver`, the endpoint a Git host notifies on a push. The word and its `type` values are Flux's. A delivery to it is a hint to look, never a fact about the branch. @@ -410,3 +440,5 @@ outage. | sha, revision (for a commit hash) | commit | `revision` is Flux's word for a polymorphic identifier, and is reserved for one | | namespace access (bare) | access, or source namespace | two directions, two names | | reconciler, orchestrator (for Flux or Argo CD) | applier | "reconciler" also names our own controllers | +| owed, owes, debt, obligation | pending, decided, sent, waiting for | an accounting word names no state: say what is pending, for whom, and what ends it | +| fence, fenced (for what a trigger does to its applier) | suspend, held | the `Kustomization` is suspended, and the trigger holds it; say which, and whether the suspend is handled | diff --git a/docs/design/apply-trigger.md b/docs/design/apply-trigger.md index 764db5158..551888f05 100644 --- a/docs/design/apply-trigger.md +++ b/docs/design/apply-trigger.md @@ -1,18 +1,25 @@ # The apply trigger -> **design**: decided 2026-10-16, extended 2026-10-17; the Flux and Argo CD appliers (in any -> cluster), the push receiver, `onRefusal: Reapply` and the apply window are built, the webhook -> applier is the aim. Index: [`../INDEX.md`](../INDEX.md). -> How to use it: [`../flux.md`](../flux.md), [`../argocd.md`](../argocd.md) and -> [`../bi-directional.md`](../bi-directional.md). -> Fields: [`../configuration.md`](../configuration.md#who-applies-this-folder-specapplier). - -**GitOps Reverser decides when Git is applied to the cluster.** Flux and Argo CD keep fetching, -rendering, applying, pruning and reporting health. They stop applying at their own will: their -short timers, webhooks and automated sync are switched off, and GitOps Reverser asks them to apply -at moments it has checked. - -This page is why, what we learned building it, and what is still open. +> **design**: decided 2026-10-16, extended 2026-10-17 and 2026-10-19; built: the Flux applier (its +> `Kustomization` held suspended), the Argo CD applier (automated sync off), the trigger kinds that +> hold them and act only while an eligible `GitTarget` references them, the cross-namespace grant +> for appliers, the push receiver, `onRefusal: Reapply` and the apply window. The webhook applier is +> the aim. Index: [`../INDEX.md`](../INDEX.md). +> How to use it: [`../bi-directional.md`](../bi-directional.md), +> [`../bi-directional-flux.md`](../bi-directional-flux.md) and +> [`../bi-directional-argocd.md`](../bi-directional-argocd.md). +> Fields: [`../configuration.md`](../configuration.md#who-applies-this-folder-specapplytriggerref). + +**GitOps Reverser decides when Git is applied to the cluster, and the applier never applies on its +own.** Flux and Argo CD keep fetching, rendering, applying, pruning and reporting health. A Flux +`Kustomization` is held suspended while it is idle and resumed for each apply; an Argo CD +`Application` runs with automated sync off. Every apply is one GitOps Reverser started, at a moment +it checked. + +This page is why, what we learned building it, and what is still open. "Applier" keeps its meaning +from [`../definitions.md`](../definitions.md): the `Kustomization` or `Application` itself. The +**trigger** (`FluxTrigger`, `ArgoCDTrigger`) is GitOps Reverser's resource that holds one, and a +`GitTarget` names it with `spec.applyTriggerRef`. ## The problem @@ -32,14 +39,15 @@ edit and its push, the edit is live and absent from Git: in an open commit windo writes waiting to be pushed. Only the branch worker can see those. So only it can say when an apply is safe, and every trigger the applier runs by itself fires without that knowledge. -A request is sent only on a **quiet branch**: no open commit window and no pending writes. Three -things owe one: +An apply is started only on a **quiet branch**: no open commit window and no pending writes. Four +reasons make one due: -| Reason | When | Asked | +| Reason | When | Applied by | | --- | --- | --- | | `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. Open windows are closed early and pending writes replayed onto it and pushed first, so the applied commit carries the API side's answer | Every applier on the branch | -| `Refusal` | An edit had no legal destination in Git and the target opted in with `onRefusal: Reapply` | The refusing target's applier | -| `CatchUp` | The branch has been quiet for `quietPeriod` after GitOps Reverser's own pushes, or since it started | Every Flux applier on the branch | +| `Refusal` | An edit to a document the folder holds was refused at the write boundary, and the target opted in with `onRefusal: Reapply` and is not suspended. At most one request a minute per target; a refusal inside that minute is carried by a trailing one | The refusing target's applier | +| `Repair` | The apply window withheld a late apply over a newer API edit, so the cluster runs an older value than Git | The applier that wrote it | +| `CatchUp` | The branch has been quiet for `quietPeriod` after GitOps Reverser's own pushes, since it started, or since a `GitTarget` first named the trigger | Every Flux applier on the branch | ## What we learned @@ -50,47 +58,60 @@ foreign pushes to the applier. But Argo CD's timed refresh syncs every new tip t after each push, with `selfHeal: false` too, and Flux applies on every interval. Neither can be filtered from outside, so the applier's own triggers go, and GitOps Reverser makes the requests. -**Flux has no fetch-only mode.** A new `GitRepository` artifact makes every `Kustomization` reading -it apply at once, and `suspend: true` also blocks the requests. So a source that fetches on its own, -on a short interval or a webhook into a Flux `Receiver`, is a `Kustomization` that applies on its -own. The source stays on a long backstop interval, and `CatchUp` keeps its artifact current, because -Flux still applies on a restart from whatever artifact it holds, and a stale one would revert edits -already in Git. +**A long interval is not exclusive control.** The first Flux applier kept the `Kustomization` +unsuspended on a 24h interval and asked it to reconcile. Flux still applied when that interval +fired, when kustomize-controller restarted (from whatever artifact it held, which is why `CatchUp` +was once a safety rule), when a failed apply retried, when the `Kustomization`'s own spec changed, +and on every new artifact of its `GitRepository`, including one another consumer's request fetched. +None of those waits for a quiet branch. The third review called it best-effort coordination, and it +was. A suspended `Kustomization` does none of them, so it is held suspended, and resumed for one +apply at a time ([below](#the-flux-applier-suspended)). + +**Flux has no fetch-only mode, unless its consumers are suspended.** A new `GitRepository` artifact +makes every unsuspended `Kustomization` reading it apply at once. A suspended one ignores it +(measured: 0 of 16 runs attempted the new revision). So a source whose every consumer is held can +fetch on every push, and push webhooks can reach it again, through GitOps Reverser's receiver. **Not every commit GitOps Reverser authors is a no-op to apply.** A replayed `F + W` carries somebody else's `F`, and a refusal needs an apply to be reverted. So the decision is state on the -worker ("this applier is owed an apply"), set by observations and cleared by a delivered request. -A push notification decides nothing by itself; it is a hint: the worker reads the remote and -judges from that, which gets late, duplicate and rolled-back deliveries right. +worker ("an apply of this applier is due"), set by observations and cleared by an operation that +succeeded. A push notification decides nothing by itself; it is a hint: the worker reads the remote +and judges from that, which gets late, duplicate and rolled-back deliveries right. -**A decision has to hold when the patch is sent as well as when it is made.** The first review found +**A decision has to hold when the write is sent as well as when it is made.** The first review found that a request decided once was attempted for minutes against whatever the cluster had become: a target its `ClusterProvider` no longer admitted still drove the applier, and a source that followed -another branch was still stamped. Every attempt now reads each target again, and the Flux client -checks that the source fetches this repository and branch before it stamps anything. +another branch was still stamped. Every attempt now reads each target again, and the client checks +that the source fetches this repository and branch, and reads the artifact again, just before the +write that starts the apply. **A retry is a decision too, so only the worker may make one.** The second review found every remaining problem in the hand-off between the worker's decisions and the deliverer's retries: a retry skipped the quiet check, outlived a refusal that had recovered, ran for a recreated target, or waited on another applier's backoff. The fix was structural rather than four patches. The worker -owns every obligation until an attempt delivers it. The deliverer carries out one attempt and -reports back; it never retries, and never asks for something weaker than it was told. A retry is -decided like a first attempt: on a quiet branch, at the tip as it is then, from what still stands. - -**Three relations, three units.** One branch worker serves many `GitTarget`s, many `GitTarget`s can -name one `Kustomization`, and one `GitRepository` can feed many `Kustomization`s. Keeping each -concern on the right unit is what makes the 1:n cases fall out: +keeps every reason pending until an operation that succeeded answers it. The dispatcher carries out one +attempt and reports back; it never retries, and never asks for something weaker than it was told. A +retry is decided like a first attempt: on a quiet branch, at the tip as it is then. + +**An apply is an operation, and its end is observed, never guessed.** The third review found the +apply window closing early in three ways: kustomize-controller writes `lastAttemptedRevision` +before it builds anything (measured: about 0.8 s early per 200 objects); an artifact that held the +commit was taken as applied; and a 5-minute timer reported an apply delivered while it could still +run. Each apply is now one operation with phases, followed until the applier's own status proves it +ended. One past its budget is `Unresolved` and still followed, because a timer that closes the +window does not stop the apply. The operation is recorded on the trigger before the apply can start, +so a restart finds it. + +**Four relations, four units.** One branch worker serves many `GitTarget`s, many `GitTarget`s can +name one trigger, and one `GitRepository` can feed many `Kustomization`s. Keeping each concern on +the right unit is what makes the 1:n cases fall out: | Unit | Owns | Why | | --- | --- | --- | | The **branch** (repository and branch) | Quietness | A `Kustomization` applies the tip whichever folder an edit is in. Two `GitProvider`s naming one repository give one branch two workers, so a worker is quiet only when its siblings are too | -| The **applier** | Obligations and retries | One attempt out at a time, its own backoff, one attempt for every `GitTarget` naming it. A shared applier waits for the longest `quietPeriod` among them, because its apply reaches all their folders | -| The **`GitTarget`**, by uid | Configuration and consent | Consent to a `Refusal` is per target, and a target deleted and recreated under the same name inherits nothing owed to the old one | - -**A shared `GitRepository` cannot be isolated, so the promise was narrowed.** When a request has to -make the source fetch, every `Kustomization` on that source applies the new revision. That is how -Flux works. The branch is quiet, so they apply a tip with nothing live missing; the guide says a -folder needing strict isolation gets its own `GitRepository`. +| The **applier**, held by its trigger | Pending reasons, retries, the operation | One operation at a time, its own backoff, one for every `GitTarget` naming the trigger. The trigger's status records it, and its `quietPeriod` is declared once | +| The **source** (cluster and `GitRepository`) | One operation at a time across its appliers | The artifact is shared: a sibling's fetch landing between an operation's recheck and its reconcile would apply another commit | +| The **`GitTarget`**, by uid | Configuration, consent and eligibility | Consent to a `Refusal` is per target, and a target deleted and recreated under the same name inherits nothing due to the old one. A trigger acts only while an eligible `GitTarget` references it | ## The design, as built @@ -98,98 +119,403 @@ folder needing strict isolation gets its own `GitRepository`. flowchart LR subgraph WORKER["Branch worker (one per GitProvider and branch)"] OBS["Observations
push · fetch · notification"] - OWE["Owed, per applier
ForeignCommit · Refusal · CatchUp"] + DUE["Due, per applier
ForeignCommit · Refusal · Repair · CatchUp"] QUIET{"Quiet branch?
this worker and its siblings"} end EXEC["Dispatcher
one attempt, no retries"] + TRIG["Trigger status
operation · repairPending"] FLUX["Flux
GitRepository · Kustomization"] ARGO["Argo CD
Application"] - OBS --> OWE --> QUIET + OBS --> DUE --> QUIET QUIET -- "yes, and due" --> EXEC - EXEC -- "recheck targets, check source,
stamp requestedAt, wait for it" --> FLUX - EXEC -- "recheck targets, check source,
set a pinned sync, wait for it" --> ARGO - EXEC -. "report: Delivered · Current ·
Failed · Rejected · Dropped" .-> OWE + EXEC -- "record before the write" --> TRIG + EXEC -- "recheck, fetch, resume with a token,
follow, suspend again" --> FLUX + EXEC -- "recheck, set a pinned sync,
follow its token" --> ARGO + EXEC -. "report: Requested · Running · Settling ·
Unresolved · Succeeded · Failed · Current ·
RequestFailed · Rejected · Dropped · Stale" .-> DUE ``` - **Observing.** Every observation of the remote passes one point. The first after a start is a - baseline (owes `CatchUp`, never `ForeignCommit`, because an apply on startup would revert edits - made while the operator was down). A tip the worker pushed owes `CatchUp`; any other owes - `ForeignCommit`. When in doubt it is foreign, which costs one apply on a quiet branch. -- **Deciding.** At the end of every wake, if the branch is quiet, each applier that is owed something, - has no attempt out, and is past its backoff (and, for a `CatchUp`, its quiet period) gets one - attempt. The attempt carries away what it covers; each target in it carries its uid and its own - reason. + baseline (a `CatchUp` is due, never a `ForeignCommit`, because an apply on startup would revert + edits made while the operator was down). A tip the worker pushed makes a `CatchUp` due; any other + a `ForeignCommit`. When in doubt it is foreign, which costs one apply on a quiet branch. An + applier a `GitTarget` names for the first time gets what a first observation gives, a `CatchUp`: + a commit observed before it was named was due to nobody, and was taken, so it would otherwise not + be applied until some later push. For a `Kustomization` already at the tip that costs nothing + (`Current`). The worker lists its `GitTarget`s on every pass, cached, so a new name is seen at + once. +- **Deciding.** At the end of every wake, if the branch is quiet, each applier with an apply due, + no operation out, shares no source with one that has, and is past its backoff + (and, for a `CatchUp`, its quiet period) gets one attempt. The attempt carries away what it + covers; each target in it carries its uid and its own reason. The worker takes the operation's + token; nothing is on the trigger yet. - **Attempting.** The dispatcher re-reads each target. It asks only for those that are still the - same incarnation, still name this applier on this branch, are still admitted by their - `ClusterProvider`s (their own, and the applier's when `spec.applier.clusterProviderRef` names - another), and for a `Refusal` still consent. If no target still holds the request's - reason, it asks nothing (`Dropped`). Just before the stamp it asks the worker whether the decision - still holds (the commit is still the tip, nothing is waiting to be published); if not, it stamps - nothing (`Stale`, no failure). Then the applier's kind decides. Flux: stamp `reconcile.fluxcd.io/requestedAt` on the - `GitRepository` when its artifact lacks the commit; when the artifact holds it, only a `Refusal` - stamps the `Kustomization`, because the artifact already woke it for anything else. Argo CD: set - the `Application`'s sync operation, pinned to the commit ([below](#the-argo-cd-applier)). A - source on another repository or branch, or an `Application` with automated sync on, is - `Rejected`. A delivered attempt is reported only once the applier has finished the apply, or - after 5 minutes, so the attempt is out for exactly the apply window. -- **Settling.** A delivered attempt is done. A stale one gives back what it carried and is decided - again at the next quiet moment. Any other outcome gives back what the attempt carried, - a refusal only if it still stands, and starts that applier's backoff (2s doubling to 5m, at least - 15s after a rejection). `ApplyRequestFailed` is recorded for the first failure of a run. + same incarnation, still name a trigger that holds this applier on this branch, are still admitted + by their `ClusterProvider`s (their own, and their trigger's), and for a `Refusal` still consent. + If no target still holds the request's reason, it asks nothing (`Dropped`). Then the applier's + kind decides, and both kinds write the operation through to the trigger as `Requested` + (`StatusWriter.RecordApplyOperation`) just before the write that starts the apply. The record is + refused, and nothing is started, while the trigger is being deleted or its status holds another + operation in flight (an external resume the reconciler recorded, or one an earlier process left + in flight). A recorded attempt that then asks for nothing withdraws its record, which puts back + what the status held: the trigger never shows an operation nothing could have applied. The + writer matches operations by their token, never by time. A `FluxTrigger` that is not + `Held=True` (still adopting, releasing, or being deleted) is not resumed (`RequestFailed`). A source + on another repository or branch, a `Kustomization` whose `spec.suspend` another manager applies, + or an `Application` with automated sync on, is `Rejected`. A decision that no longer holds when + the write is due (the commit is no longer the tip, or something is waiting to be published) asks + nothing (`Stale`, no failure). +- **Following.** The dispatcher follows the operation and reports each move, so the attempt is out + for exactly the apply window. Flux: `Running`, `Settling` once suspended again, then `Succeeded` + or `Failed` ([below](#the-flux-applier-suspended)). Argo CD: the operation state carrying the + request's token ([below](#the-argo-cd-applier)). One not completed after 5 minutes is + `Unresolved`: a Flux `Kustomization` is suspended at once, and the operation is followed on, at an + interval doubling to a minute. The applier object being deleted, or its trigger being deleted, + completes it as `Failed`; no `GitTarget` naming the trigger any more abandons it, and a Flux + `Kustomization` is suspended, and that suspend handled, before it is let go. +- **Settling.** An operation that succeeded is done. A stale attempt gives back what it carried and + is decided again at the next quiet moment. Any other outcome, a failed operation included, gives + back what the attempt carried, a refusal only if it still stands, and starts that applier's + backoff (2s doubling to 5m, at least 15s after a rejection). `ApplyRequestFailed` or + `ApplyFailed` is recorded for the first failure of a run, and `ApplyUnresolved` when an operation + becomes `Unresolved`; `apply_operations_unresolved` counts those still out, and the trigger is + `Ready=False` (`OperationUnresolved`) until its operation completes. - **Learning about pushes.** A push webhook into GitOps Reverser's receiver (seconds), a rejected compare-and-swap push (at the next publication), or the periodic refresh (minutes). All three feed the same observation point. +## The Flux applier, suspended + +### Source facts + +Each was read in kustomize-controller **v1.9.6** (`internal/controller/`) and flux2 **@1ed703a3**, +both under `external-sources/flux/`, and each is applied by the fake the tests run against +(`internal/applier/fluxsim`, tested on its own). None should be re-derived. + +| Fact | Where | What it gives | +| --- | --- | --- | +| `finalizeStatus` is deferred and copies `reconcile.fluxcd.io/requestedAt` to `status.lastHandledReconcileAt` on **every** return: suspended, failed, dependency not ready, succeeded | `kustomization_controller.go:143`, `:1335–1340` | A token is handled exactly when the reconcile that read it ends, whatever the outcome | +| A suspended object returns right after the finalizer check, before resolving the source | `:189–192` | Suspended means nothing is fetched, built or applied | +| `spec.suspend` is read only when a reconcile starts | `:189` | A suspend does not stop an apply already running (measured too) | +| `lastAttemptedRevision` is written before the build | `:390` | It says nothing about an apply having finished | +| `observedGeneration` is set whenever `Ready` is True at the end, a suspended return with a `Ready` left from the last apply included | `:1345–1348` | No completion signal can be read from it | +| A failed apply requeues after `retryInterval`; a dependency not ready requeues with `Ready=False`, `DependencyNotReady` | `:311`, `:264–268` | Both are outcomes to read; a requeue into a suspended object does nothing | +| A reconcile is enqueued by a new generation **or** a new `requestedAt`; a new source revision enqueues it too | `kustomization_manager.go:139–173` | One write that sets both `suspend` and `requestedAt` enqueues a reconcile that reads both | +| A dependent needs its dependency's `observedGeneration == generation` and `Ready` | `:572`, `:618–624` | A dependency suspended after a failed apply holds its dependents back until it succeeds | +| controller-runtime runs at most one reconcile per object at a time | workqueue contract | A token handled by a later reconcile proves every earlier reconcile of that object has ended | +| `flux resume ks` is a merge patch of `spec.suspend: false`, then a wait on readiness | flux2 `cmd/flux/resume.go:183–245` | A resume is `flux resume` with a token added | + +### One operation, every reason + +Every reason runs the same operation, because a resume alone applies (measured: it reverted a +drifted object with no new commit): + +1. **Make the artifact hold the commit.** When it does not, stamp the `GitRepository` and wait for + it to handle that token. An artifact that ends on another commit is `Stale` when the decision no + longer holds, and a failure otherwise: the source cannot fetch what the worker checked. +2. **Recheck just before resuming:** read the `Kustomization` again (the same uid, the same + `sourceRef`, no other manager applying `spec.suspend`: one replaced or re-pointed during the + fetch is not resumed), read the artifact again, and ask the decision (`ApplyRequest.Current`). + This narrows [the residual race](#what-it-costs) as far as it can be narrowed. +3. **Record the operation** on the trigger, written through, as `Requested` with its token, and + ask the decision once more: the record can wait on the API server, and a decision that went + stale meanwhile withdraws the record and resumes nothing. +4. **Resume with a token.** One server-side apply under the field manager `gitops-reverser` sets + `spec.suspend: false` and `requestedAt: `, carrying the uid and `resourceVersion` of that + last read as preconditions, so anything that changed the object since is a conflict (read again, + up to five times) and a deleted one is not re-created. A resume whose answer is lost (a timeout, + an internal error) may have landed, so it is followed and suspended again as if it had. +5. **Wait for T1 handled.** A token somebody stamped over T1 (`flux reconcile`) stands in for it: + the reconcile that handles it started after the resume. A handled token does not prove a + reconcile ran resumed, though: a suspend landing between the resume and the reconcile (a + `flux suspend`) has the suspended return handle it, leaving the last apply's `Ready`. So the + outcome also needs proof that a reconcile ran since the resume: `Ready`'s `lastTransitionTime` + (a real reconcile marks it `Unknown` first) or the top `status.history` entry moved past what the + resume's apply returned. Then read the outcome: + + | Read after T1 is handled | Outcome | + | --- | --- | + | No reconcile ran since the resume | `Failed`, so the worker asks again | + | `Ready=True`, `lastAppliedRevision` holds the commit | `Succeeded` | + | `Ready=True`, the revision the `GitRepository` holds now, which is not the commit | `Succeeded`, logged and counted in `apply_revision_mismatches_total` (the residual race: the source moved on) | + | `Ready=True`, any other revision (kustomize-controller's cached source lagged the artifact checked) | `Failed`, so the worker asks again | + | `Ready=False`, `DependencyNotReady` | `Failed`, with a reason of its own | + | `Ready=False`, anything else, or no `Ready` | `Failed` | + +6. **Suspend again:** another server-side apply sets `spec.suspend: true` and + `requestedAt: ` (`Settling`, the token recorded as `suspendToken`). T2 handled proves that any + reconcile that started before the suspend has ended and the object is suspended now: a suspended + reconcile handles a token without applying, and reconciles of one object never overlap. Found + resumed again before T2 is handled, the `Kustomization` is suspended again. +7. **Complete** once T2 is handled, and keep the window open 10 s more. + +A `ForeignCommit` or `CatchUp` asks nothing (`Current`) when the `Kustomization` is suspended and +both its artifact and `lastAppliedRevision` hold the commit: nothing applied since, and nothing will +until it is resumed. A `Refusal` and a `Repair` always run the operation. A failed apply is suspended +like any other, which is what stops Flux's own `retryInterval` retry; the worker owns the retry. + +An `Unresolved` operation is suspended at once (the suspend does not stop the running apply, but it +stops whatever would follow), and that suspend's token then decides when it completes. There is no +automatic cancellation. + +`CatchUp` stays, as convergence rather than safety: a suspended `Kustomization` never applies a +stale artifact, but Flux's inventory learns of an object only from an apply of a commit that holds +it, and an object created through the API and never applied by Flux would not be pruned when Git +later deletes it. + +### Who owns `spec.suspend` + +- **A trigger acts only while an eligible `GitTarget` references it.** Eligible: in the trigger's + namespace, not being deleted, naming the trigger, admitted by its `ClusterProvider`s + (`ApplierAdmitted`), and writing the repository and branch the applier's source fetches. It is + judged from those inputs alone, never from the `GitTarget`'s `Ready`, which reads the trigger's + own `Ready` back. A trigger no `GitTarget` names is `Configured=Unknown` and `Ready=True`, both + `NotReferenced`: it reads and writes nothing in the applier's cluster and is not in line. With + several naming it, one eligible binds it; none eligible while one cannot be judged yet (its + `GitProvider` missing, a read failed) is `Configured=Unknown`, `NotBound`, progressing; each + refused is `Configured=False` with the reason they share, or `NotBound`, stalled. The trigger + reconciler watches what eligibility reads: `GitTarget`s (before and after an update), + `GitProvider`s, `ClusterProvider`s and namespace labels. A `GitTarget` reads `ApplyTriggerReady` + `Unknown` (`NotReferenced`) until the trigger has observed it, since naming a trigger does not + move its generation. +- **One trigger per applier.** Every trigger suspends under the same field manager, so two holding + one `Kustomization` would take each other's resume for an external one and release each other's + hold. Of the triggers of one kind in line for one applier in one cluster, one holds it; the others + are `Configured=False`, `HeldByAnotherTrigger`, do nothing, and release nothing, and the dispatcher + drops a request through them. In line: an eligible `GitTarget` references the trigger, or, for a + `FluxTrigger`, it carries its finalizer (it holds the `Kustomization`, or is handing it back), so + an older trigger with the wrong repository, or not admitted, keeps no younger one out. First comes + a `FluxTrigger` with an operation in flight, then one that carries its finalizer, then the oldest + by `creationTimestamp` (then namespace and name). So a `Kustomization` changes hands only once its + holder lets go of it, and never mid-apply or mid-release. The dispatcher's check before a resume, + and the reconciler's suspend of a resumed `Kustomization`, read the triggers uncached. A holder + that lets go with a trigger in line leaves the `Kustomization` suspended, its suspend handled, for + it, and the change re-runs the others. The same rule holds for `ArgoCDTrigger`s, which would + otherwise both start syncs, without the in-flight and finalizer rules: a sync somebody else + started is never replaced. +- **Adoption.** A `Configured` `FluxTrigger`, so one an eligible `GitTarget` references, puts its + finalizer in place and then suspends its `Kustomization` with a server-side apply under + `gitops-reverser`, and a token. A `Normal` `Adopted` Event records it. It is `Held=True` + (`Suspended`), and `Ready`, once that token is handled, and `Held=True` (`Applying`) while an + operation of a branch worker is in flight and followed. A trigger that never adopted has no + finalizer, and deletes at once. +- **The manifest omits `suspend`.** Whatever applies the `Kustomization` object (a parent + `Kustomization`, Helm, flux-operator) must not set the field, or it takes it back on its next + apply. A `managedFields` entry of another manager with operation `Apply` owning + `f:spec.f:suspend` is `Configured=False`, `SuspendOwnedElsewhere`, naming the manager, and + nothing is held. An `Update` entry (`flux resume`, `flux suspend`, `kubectl edit`) wrote the field + once, and the next forced apply takes it back. +- **Resumed by someone else.** The reconciler finds the `Kustomization` resumed with no operation of + a branch worker in flight (read uncached, since the worker records its operation a moment before + it resumes). It suspends it with a token, records a `Normal` `ResumedElsewhere` Event, and records + the apply as an `Unresolved` operation with that suspend token and no request token, which it + completes once the suspend is handled. It does this every time, with an Event each time: there is no + step-back state. A `FluxTrigger` that is `Ready` reads its `Kustomization` every minute, which + bounds how long such a resume runs unsuspended. The reconciler reads the operation before and + after the `Kustomization`, so an operation that completes between the two reads is no external + resume. +- **An operation nobody follows.** The reconciler leaves a branch worker's operation in flight + alone only while this process's dispatcher follows it (`Following`), or for a minute after it + first saw it unfollowed, so restart recovery gets its chance. After that it suspends the + `Kustomization` with a token, sends a `Warning` `OperationUnfollowed` Event, and completes the + operation `Failed` once the suspend is handled. A resumed `Kustomization` is never left with + nothing watching it. +- **Hand-back.** The trigger carries the finalizer `configbutler.ai/flux-trigger` from before its + first adoption write until it has handed the `Kustomization` back or over, also while its + configuration is invalid (then `Held=Unknown`, `NotConfigured`, and the message says the + `Kustomization` is left suspended under `gitops-reverser`). It hands it back when it is deleted, + and when no eligible `GitTarget` references it any more, conclusively: a reference that cannot be + judged yet keeps the hold, since a release is not undone. Until the hand-back is done the trigger + is `Held=False`, `Releasing`, and stays the holder in line. It suspends the `Kustomization` with a + token if an apply has it resumed, waits until that is handled, then releases `spec.suspend` with + an empty apply, so Flux owns it again (`Released`). An operation still recorded in flight that + this process does not follow then completes as `Failed`. A reference that returns before the + release is done holds the `Kustomization` again; one that returns after adopts it anew. Deleting + or re-creating a `GitTarget` therefore hands it back, and Flux applies the tip at once: what Flux + does without GitOps Reverser, accepted because with no `GitTarget` nobody carries live edits to + Git anyway. With another trigger in line that can take it over (`Configured=True`, or `False` + only for `HeldByAnotherTrigger`), it is not released but left suspended for that one: released, Flux would + apply it at once, unchecked against the branch, before the next trigger suspended it. On a real + API server our forced apply over a `flux suspend` *shares* `spec.suspend` with that `Update` + entry, so an empty apply alone would leave it `true`: the release first takes the field with a + forced `suspend: false`. A manager that *applies* the field keeps it, and `Released` names it. An + operation whose trigger is being deleted completes as `Failed` and writes nothing more to the + `Kustomization`; the dispatcher reads the trigger uncached before every step that may suspend. A + `Kustomization` that is gone lets the trigger go. A `ClusterProvider` that is gone does not end the + hand-back: it goes on through whatever client still reaches the cluster, which for a provider + without `kubeConfig` is the operator's own. A cluster no client reaches any more, a suspend or + release the identity may not make, or a hand-back still unfinished 10 minutes after `Held` turned + `Releasing` (each a `Warning` `ReleaseFailed` Event, which says where the `Kustomization` was left) + lets the trigger go. + +### Restart recovery + +The worker reads its triggers' `status.operation` before the loop handles its first item, so no +live event is mirrored before the windows of what it finds are open: + +- **An operation not completed** is followed to its end by its recorded tokens, asking the applier + for nothing (`ApplyRequest.Recovered`), with its apply window open from the recovery. For Flux, + the follow suspends the `Kustomization` again. A recorded token that was stamped over (by our own + suspend before the crash, or a `flux reconcile`) is judged on whether a reconcile ran since the + operation's `requestedAt`. A recorded request arriving before the dispatcher starts waits for it. + A recorded token that is nowhere on the `Kustomization` (neither its `requestedAt` nor the last + handled) means the process stopped between the record and the resume, or a suspend it never + reported followed: the `Kustomization` is suspended, and the operation ends `Failed`. +- **`repairPending`** is restored. +- A worker's own record is read back once, when this process first lists the trigger; later reads + can lag its own writes. A record with no request token (an external resume the reconciler + suspended) is followed whenever it is listed, once per suspend token, so its window opens while the + process runs too. + +The first observation is still a baseline that makes a `CatchUp` due, so a recovered operation +decides nothing about which apply is due. + +### Shared sources and prefetch + +- The trigger lists the consumers of its `GitRepository` in the applier's cluster: + `Kustomization`s with that `sourceRef`, and `HelmRelease`s whose chart comes from it, directly or + through a `HelmChart` their `chartRef` names (at whichever `HelmRelease` version the cluster + serves; consumers that cannot be determined are an error, so no prefetch). Those no `FluxTrigger` + on the same `ClusterProvider` holds, that is reports `Held=True`, are + `status.source.otherConsumers`, information only: it never affects `Ready`, and no `Warning` is + sent, because one `GitRepository` for `apps` and `infra` is a normal setup. +- `status.source.prefetch` is true only when that list was read and is empty, and the trigger holds + its own `Kustomization`. The dispatcher checks once more, on the cluster, that every consumer is + held just before it stamps the source, and stamps only the `GitRepository` whose consumers it + checked: the `Kustomization`'s live `sourceRef` must still name it. Then a push somebody else + made, observed while the branch is busy, has the source fetch at once, so the artifact is + ready when the apply is due. On a quiet branch the attempt sent at once fetches for itself. No + prefetch while any `FluxTrigger` on that source has an operation in flight, because a resumed + consumer applies whatever the artifact becomes. A listing that is forbidden counts as not holding. +- Appliers that share a source have one operation at a time, and take turns: the one whose last + turn is oldest goes first, so a busy sibling cannot starve another. This holds across the + workers of GitProviders that name the same repository and branch: a worker claims the source's + group before it sends, under the siblings' lock, and a sibling's operation of that source holds + a prefetch back too. + +### What the measurement found + +Measured on the e2e cluster (Flux v2.9.6, kustomize-controller and source-controller v1.9.6) with a +throwaway Gitea repository, 1 and 200 ConfigMaps, polling through `kubectl proxy` (±30 ms). The +"request" columns are the client this design replaced, which left the `Kustomization` unsuspended +and stamped its `requestedAt`; the "suspended" columns are the lifecycle above: + +| Median, ms from the first Flux write | 1 object, request | 1 object, suspended | 200 objects, request | 200 objects, suspended | +| --- | ---: | ---: | ---: | ---: | +| Source artifact holds the commit | 60 | 58 | 122 | 111 | +| Applied (`lastAppliedRevision`, `Ready`) | 149 | 58 + 115 | 928 | 111 + 852 | +| Suspend write | – | 10 | – | 12 | +| **Total** | **149** | **193** | **928** | **980** | + +The suspended lifecycle costs a fixed ~50 ms per apply, one serialized round trip, at any payload +size; build and apply dominate either way. The same runs found the behaviors the lifecycle is +built around: a suspended consumer ignores new artifacts; suspend does not stop a running apply; a +`requestedAt` on a suspended object is handled without an apply (which made the request-based +client report a `Refusal` done with nothing reverted); a resume alone applies; and a resume applies +whatever the artifact holds then. + +### Decisions + +Taken 2026-10-17: + +1. **Suspension is the only Flux mode,** as automated sync off is the only Argo CD mode. A second, + request-based mode would keep every hole of the 24h setup alive, and two completion protocols. +2. **A shared source is information only:** `status.source.otherConsumers`, prefetch off while it is + not empty, no `SharedSource` condition. +3. **An external resume is always suspended again,** with a `Normal` Event each time. +4. **`CatchUp` stays for Flux,** as convergence. The same inventory question exists for Argo CD, + which was decided without `CatchUp`. +5. **The resources are triggers,** with long kind names: `FluxTrigger` (`spec.kustomizationRef`) + and `ArgoCDTrigger` (`spec.applicationRef`), named by `GitTarget.spec.applyTriggerRef` in the + same namespace. Each is its own kind because suspension needs one owner object with a finalizer, + the operation and `repairPending` need a durable status, and a typed kind per applier needs no + union of references with CEL rules to keep them apart. The references are immutable: a re-pointed + trigger would leave the old applier held by nothing. + +Taken 2026-10-19, after the seventh review: + +1. **A trigger acts only while an eligible `GitTarget` references it** ([above](#who-owns-specsuspend)). + An idle trigger is `Ready=True` (`NotReferenced`): `Ready=False` would read as in progress to a + Flux health check, which falls back to `Ready`, and deadlock the `Kustomization` that creates + the `GitTarget` behind the one that creates the trigger. Deleting or re-creating a `GitTarget` + hands the `Kustomization` back, and Flux applies the tip at once; accepted. +2. **Cross-namespace appliers are a `ClusterProvider` grant,** `allowCrossNamespace.appliers`, + beside `allowCrossNamespace.sources`, which replaces `allowAnySourceNamespace`. The binding + compares URLs and proves no write access, so naming another namespace's `Kustomization` is a + power to delegate, not a consequence of being able to write the repository. Consent on the + applier itself (`ReferenceGrant` style) stays a possible follow-up. +3. **`allowAnySourceNamespace` is removed, not migrated.** It is never read, and kept in the schema + only so a rule on `spec` refuses `true` on any spec change. +4. **The trigger's status speaks from its own side:** `Held` with `Suspended`, `Applying`, + `Suspending`, `Releasing`, `NotReferenced`, `NotConfigured`; "fence" is retired for "suspend". + The operation phases keep `Running` and `Settling`. +5. **A holder keeps its `Kustomization` until it lets go:** a trigger that carries its finalizer + ranks before older ones in line, so one that becomes eligible later never displaces a holder + mid-release. +6. **A newly bound `Application` waits for the next push.** A first sync on binding would run its + hooks for a commit it may already have, which is why an `Application` gets no `CatchUp`. + ## The apply window -**A request is asynchronous, and that leaves a window.** Flux fetches and applies a few seconds -after the stamp. An API edit that lands in those seconds, to a field the `Kustomization` applies, is -reverted by the apply, because the commit Flux was asked for does not hold it yet. Making the -request synchronous would not help: waiting for the apply does not stop a user from editing during -it, and refusing edits would halt the API for everyone to protect a few seconds. +**An apply is asynchronous, and that leaves a window.** An API edit that lands while an apply runs, +to a field the applier sets, is reverted by the apply, because the commit it applies does not hold +the edit yet. Making the request synchronous would not help: waiting for the apply does not stop a +user from editing during it, and refusing edits would halt the API for everyone to protect a few +seconds. The quiet-branch rule confines the window to three moments: right after somebody else's push, right after a refusal the target asked to have reverted, and after `quietPeriod` of silence on a branch -whose artifact is behind. Inside it, **the API wins**, as it does for any object both sides changed: - -- **The window is known.** The dispatcher reports a delivered attempt only once the applier has - finished the apply: for Flux, `status.lastHandledReconcileAt` carries the stamp of a stamped - `Kustomization`, or the `Kustomization` attempted a new revision after a source stamp (or the - source handled the stamp with no new artifact); for Argo CD, the operation state carrying the - request's token has completed. At most 5 minutes; the worker keeps the window open 10 seconds - more, because the applier's writes reach it through the watch. +GitOps Reverser pushed to, or that a `GitTarget` newly names a trigger for. Inside it, **the API +wins**, as it does for any object both sides changed: + +- **The window is known.** It opens when the attempt is sent and stays open until its operation + completed: for Flux, the resume's token and the suspend after it handled; for Argo CD, the operation + state carrying the request's token completed. An `Unresolved` operation keeps it open until it + completes. Then the worker keeps it open 10 seconds more, because the applier's writes reach it + through the watch. An attempt sent within those 10 seconds carries the edits seen in the window + before it, and that window's end, so a late write of the earlier apply is still withheld, and an + attempt that asked for nothing does not end the earlier grace. A write of the new apply to an + object edited during the earlier one is then withheld too, since the managers cannot tell the two + applies apart: that costs one redundant `Repair`, never an edit. - **The late apply is withheld.** A write whose latest `managedFields` entries are the applier's own managers alone (`kustomize-controller`; `argocd-controller` or - `argocd-application-controller`), inside the window, to an object in the applier's folders that - the API side edited after the attempt was sent, is not mirrored. Git keeps the edit; for Flux the - `CatchUp` its push owes applies it back, for Argo CD the next sync does. The cluster runs the - reverted value until then. + `argocd-application-controller`) and newer than the API edit, inside the window, to an object in + the applier's folders that the API side edited after the attempt was sent, is not mirrored. Git + keeps the edit, and the withheld write makes a `Repair` due to its applier: an apply of the tip at + the next quiet moment, with no quiet period, for Flux and Argo CD alike. The cluster runs the + reverted value until then, and the trigger shows `repairPending`. - **Only proof withholds.** `managedFields` times resolve to the second. When the applier ties with another manager, the write could be the edit itself, and withholding an edit loses it - outright; it is mirrored as before. A DELETE carries nothing about its deleter and is never - withheld. Both are logged. + outright; it is mirrored as before. It is also remembered as an edit, even as the first write in + the window, so the applier's write alone a second later is still withheld. An applier entry in the + same second as the edit is mirrored the same way. One older than the edit is no write of the + applier's at all: an update that removes every field a manager owns deletes that manager's entry, + so undoing an edit (`kubectl label cm/x tmp-` after `tmp=1`) can leave the applier's entry from an + earlier apply as the newest. A late apply moves the applier's entry forward, so an older one is + the API's undo, mirrored as an edit. A DELETE carries nothing about its deleter and is never + withheld. All are logged. Every collision is logged with the object, the applier and the requested commit, and counted in `apply_window_collisions_total` (`withheld` or `ambiguous`). What remains: an edit reverted on the -cluster until the next apply (for Flux, up to `quietPeriod`), an ambiguous tie mirrored as the -applier's commit, and an applier's write that reaches the watch more than 10 seconds after it -reported done. +cluster until the repair, an ambiguous tie mirrored as the applier's commit, and an applier's write +that reaches the watch more than 10 seconds after its operation completed. ## The Argo CD applier -**GitOps Reverser takes full control of when Argo CD syncs.** The `Application` named by -`spec.applier.applicationRef` has automated sync off (`syncPolicy.automated.enabled: false`, or no +**GitOps Reverser takes full control of when Argo CD syncs.** The `Application` an `ArgoCDTrigger` +holds has automated sync off (`syncPolicy.automated.enabled: false`, or no `automated` block), so its timed refresh and its webhook only compare, and every sync it runs is one GitOps Reverser asked for, pinned to a commit the branch worker checked. This is the supported setup for bi-directional editing with Argo CD, not an option next to automated sync: an -`Application` with automated sync on is `ApplierConfigured=False` (`AutomatedSyncEnabled`) and is -never synced. The user guide is [`../argocd.md`](../argocd.md). +`Application` with automated sync on makes its trigger `Configured=False` (`AutomatedSyncEnabled`) +and is never synced. The user guide is [`../bi-directional-argocd.md`](../bi-directional-argocd.md). -Argo CD is the easier half. Flux has no fetch-only mode, so it needs a `CatchUp` to keep its -artifact from going stale. With automated sync off Argo CD applies nothing on its own, so it holds -nothing that can go stale, and **it gets no `CatchUp`**: `quietPeriod` is rejected with an -`applicationRef`, and the worker never owes an `Application` one. +Automated sync off is Argo CD's equivalent of a suspended `Kustomization`, and it needs no suspend +after an apply: an Argo CD operation that has completed is not followed by another one nobody +started. **It gets no `CatchUp`**: an `ArgoCDTrigger` has no `quietPeriod`, and the worker never +asks an `Application` for one. So an `Application` a `GitTarget` names for the first time is synced at the next push, a +refusal or a repair, not at once: a sync on binding would run its hooks for a commit it may already +have. Every rule the client relies on is Argo CD's own, read from its source (3.5.0): @@ -198,74 +524,94 @@ Every rule the client relies on is Argo CD's own, read from its source (3.5.0): | Automated sync is on when `syncPolicy.automated` is present and its `enabled` is unset or true (`IsAutomatedSyncEnabled`) | `AutomatedSyncEnabled` is exactly this rule; `automated: {enabled: false, prune: …}` is the off switch that keeps prune | | Automated sync builds `Operation{Sync{Revision, Prune: automated.prune, SyncOptions, Source}, InitiatedBy{Automated: true}, Retry: syncPolicy.retry or {Limit: 5}}` (`autoSync`) | A request builds the same, with `Revision` pinned to the checked commit | | A sync starts by setting `operation`, never while one is set, in an update that also clears `status.operationState` (`SetAppOperation`; the CRD has no status subresource) | A request updates the whole `Application`, guarded by the `resourceVersion` it read, and waits out a running operation instead of replacing it | -| The controller clears `operation` when the operation completes and leaves the result in `status.operationState`; a failed operation that will be retried is `Running` again, with a `finishedAt` | An attempt's window closes when the operation state carrying its token has completed (`Succeeded`, `Failed`, `Error`); a pending retry is still running | +| `NewOperationState` copies the operation, its `info` included, into `status.operationState`, and `operation` is cleared only once the operation completed | An update whose answer was lost, or one recovered after a restart, is followed by its token: in `operation` or `status.operationState` it landed. Neither means it was not made yet. The update carries the `resourceVersion` it was made from, so GitOps Reverser then writes the `Application` at the version it read, asking for a refresh (`argocd.argoproj.io/refresh`, which only compares). Once that write is made the update can never land, and the operation is `Failed` and asked again; a conflict means read again. No API-server deadline is assumed | +| The controller clears `operation` when the operation completes and leaves the result in `status.operationState`; a failed operation that will be retried is `Running` again, with a `finishedAt` | An operation completes when the operation state carrying its token has completed (`Succeeded`, `Failed`, `Error`); a pending retry is still running | +| The controller processes an operation only while `operation` is set (`processAppRefreshQueueItem`), so one whose `operation` somebody removed (the workaround for a stuck hook) stays `Running` for good | Our sync in that shape is `Failed` once `Unresolved`, when any controller pass in flight has written; a request finds such a state, unchanged across its whole wait, no operation in progress, and syncs over it | | A retry with `retry.refresh: true` drops the operation's revision and syncs the newest one | That one field is dropped from the copied retry: a retry stays pinned | | Every apply writes as field manager `argocd-controller`, client-side and server-side alike; a `Replace=true` sync creates and replaces with no field manager, so the API server takes it from the user agent, `argocd-application-controller` | The apply window knows both as Argo CD's managers | -| Sync windows are checked for every operation, by `InitiatedBy.Automated`, and a blocked operation stays `Running` until the window opens (`syncWindowPreventsSync`, `CanSync`) | A sync GitOps Reverser starts obeys the project's windows for automated syncs, and a blocked one runs later, still pinned: by then it can be older than the tip. The guide forbids sync windows on a named `Application`; `ApplierConfigured` does not check them yet | +| Sync windows are checked for every operation, by `InitiatedBy.Automated`, and a blocked operation stays `Running` until the window opens (`syncWindowPreventsSync`, `CanSync`) | A sync GitOps Reverser starts obeys the project's windows for automated syncs, and a blocked one runs later, still pinned: by then it can be older than the tip. The guide forbids sync windows on a held `Application`; `Configured` does not check them yet | | Hooks default to `BeforeHookCreation`: each sync deletes and recreates the hook's object (`DeletePolicies`) | A `WatchRule` covering a hook's kind mirrors that delete and create on every sync; the guide keeps hook kinds out of them | Three choices were decided (2026-10-17): 1. **Prune** comes from `syncPolicy.automated.prune` when `automated` is present with `enabled: false`. Without the block, a sync does not prune, and the guide says so. There is no - `spec.applier.prune`. + `prune` field on the trigger. 2. **The initiator** is `InitiatedBy{Username: "gitops-reverser", Automated: true}`. GitOps Reverser replaces automated sync, so a sync window that blocks automated syncs blocks its syncs too. 3. **Hooks run on every sync**, so on every `ForeignCommit`, which asks every applier on the branch. Narrowing it by changed paths waits for a measured complaint. -A hub Argo CD keeps `Application`s in a management cluster: `spec.applier.clusterProviderRef` +A hub Argo CD keeps `Application`s in a management cluster: the trigger's `spec.clusterProviderRef` reaches it as it reaches a Flux in another cluster. Out of scope: multi-source `Application`s (and -`sourceHydrator`), which are `SourceMismatch`; `ApplicationSet`s, which own the spec of the +`sourceHydrator`), which are `UnsupportedSource`; `ApplicationSet`s, which own the spec of the `Application`s they generate; and Argo CD's own sharding and HA. ## The aim beyond Flux and Argo CD -**A webhook applier** for any other tool: POST `{"repository", "branch", "commit", "reason"}`, -signed `X-Signature: sha256=` the way Flux's `generic-hmac` `Receiver` checks. It gets -every reason, `CatchUp` included, since nothing says the tool does not apply on its own. No -in-process plugin: it would run with every credential the operator holds. +**A webhook applier** for any other tool: a third trigger kind that POSTs `{"repository", "branch", +"commit", "reason"}`, signed `X-Signature: sha256=` the way Flux's `generic-hmac` +`Receiver` checks. It gets every reason, `CatchUp` included, since nothing says the tool does not +apply on its own. No in-process plugin: it would run with every credential the operator holds. ## What it costs -- **GitOps Reverser becomes the fast path to a deployment.** While it is down, a Git-side push - reaches the cluster through `flux reconcile`, a manual sync, or the long backstop interval. - Bi-directional editing already stops in that state. +- **While GitOps Reverser is down, a `Kustomization` waits suspended.** A Git-side push waits until + it is back, or for a manual `flux resume` or sync (after which GitOps Reverser suspends the + `Kustomization` again on start). This is the price of exclusivity, and the same trade Argo CD with + automated sync off makes. Bi-directional editing already stops in that state. A graceful stop + (`helm upgrade`, a node drain) suspends a `Kustomization` it had resumed for an apply, once and + without waiting for it to be handled (the dispatcher's `suspendOnStop`), and the next process + recovers that suspend as one of its own it did not report. After a crash, the apply that was + running finishes and the next process suspends the `Kustomization`; until then Flux may apply on + its own. The narrowing is a mitigation, not a guarantee: a crash, a suspend that fails, or an + apply already running leave a gap. +- **Flux tooling shows `Suspended`.** `flux get ks` reports it, and `flux reconcile ks` does nothing. +- **A dependency that failed stays behind.** A `Kustomization` suspended after a failed apply keeps + its generation unapplied, and its dependents wait until GitOps Reverser's retry succeeds. +- **The `Kustomization`'s manifest must omit `suspend`.** This is checked, not just documented. +- **The residual race.** Between the artifact recheck and the reconcile reading the artifact, + somebody can stamp the `GitRepository`. With prefetch held off during operations, that is a person + or the source's own interval. The outcome table catches it, and the apply window still withholds + a late write over a newer edit. - **One replica.** The receiver, the branch workers and the dispatcher run in one pod, and the chart refuses more than one replica. Leader election is not switched on; more than one replica needs ownership coordination first, and deliveries routed to the pod that owns the branch. -- **A request holds its applier while Flux applies.** The next request to that applier waits for - the running apply (up to 5 minutes), which is the point: nothing is stacked on an apply still - running. A Flux that never reports done costs those 5 minutes per request. -- **Every applier on the branch is asked on a `ForeignCommit`**, whether or not the push touched its +- **An apply holds its applier while it runs.** `wait: true`, health checks and a timeout lengthen + it, which is the point: nothing is stacked on an apply still running. One that never ends holds + its applier, which `apply_operations_unresolved` and an `ApplyUnresolved` Event show. +- **Every applier on the branch applies on a `ForeignCommit`**, whether or not the push touched its folder. Harmless for Flux; for Argo CD it runs hooks. Narrowing it needs the applier's own source paths, so it waits for a measured complaint. - **A sync somebody else started holds an Argo CD request**, for up to 5 minutes, and then fails it; the worker asks again after its backoff. +- **A trigger may hold any applier its `ClusterProvider` reaches**, with + `allowCrossNamespace.appliers`: every namespace the provider admits may suspend and apply a + `Kustomization`, or sync an `Application`, in another namespace whose source is the repository its + `GitProvider` names. Nothing proves the namespace owns it or may write to that repository. ## Where this goes next -The Flux applier PR (#419) carries what gitops-api, the first adopter, needs before it can drop its -own gate: both reviews' findings and the customer review's three blockers (stale stamps, a late -webhook cutting a save's window, a `Kustomization` in the management cluster via -`spec.applier.clusterProviderRef`), the hardened receiver, the apply window, and the Argo CD -applier with automated sync off as the standard. After it, in order, each one pull request: +What gitops-api, the first adopter, needs before it can drop its own gate is built: the hardened +receiver, the apply window, the Argo CD applier with automated sync off, and the Flux applier held +suspended. Next, in order, each one pull request: -1. **The webhook applier.** +1. **The webhook applier**, as a third trigger kind. 2. **More than one replica.** Leader election is not switched on, and the chart runs one replica. - Standbys need ownership coordination first (the HA plan in `docs/future/`), and deliveries then - need routing to the pod that owns the branch. + Standbys need ownership coordination first + ([the HA plan](../future/ha-gittarget-distribution-plan.md)), and deliveries then need routing to + the pod that owns the branch. Test gaps to close along the way: a remote source cluster and an applier in another cluster (a hub Argo CD among them) in e2e, `Forbidden` against real RBAC, the receiver and applier roles installed by the Helm chart, two `GitTarget`s sharing one `Kustomization` in e2e, an apply-window collision in -e2e, a `Replace=true` Argo CD sync's field manager measured rather than read from the source, and -`-race` in CI. +e2e, a prefetch observed end to end, a `Replace=true` Argo CD sync's field manager measured rather +than read from the source, and `-race` in CI. **What adopting it changes for gitops-api.** Its gate shrinks to "one save, one window", which protects commit messages and has nothing to do with Flux. Its push handler relays each push to the `GitProvider`'s receiver instead of stamping Flux. And publications trail an editing session by at -least `quietPeriod`, since Flux now applies GitOps Reverser's own commits only after the branch has +least `quietPeriod`, since Flux applies GitOps Reverser's own commits only after the branch has been quiet; `quietPeriod` is the knob for that trade. ## What proves it @@ -274,19 +620,29 @@ been quiet; `quietPeriod` is the knob for that trade. | --- | --- | | The decision: quiet branch, reasons, quiet period, baseline | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go) | | Retries are decided by the worker; recovered refusals, withdrawn consent and recreated targets are not retried; each applier keeps its own clock; one attempt for a shared applier; a busy sibling worker holds the branch | [`apply_attempt_test.go`](../../internal/git/apply_attempt_test.go) | -| Every attempt rechecks targets, asks for nothing weaker than it was told, and stamps no source on another branch | [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go), [`flux_test.go`](../../internal/applier/flux_test.go) | -| An applier in another cluster is reached through its own `ClusterProvider`, behind that provider's `accessFrom` | [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go), [`clusterprovider_admission_test.go`](../../internal/authz/clusterprovider_admission_test.go), [`gittarget_applier_test.go`](../../internal/controller/gittarget_applier_test.go) | -| An attempt stays out until Flux finished the apply; the applier's late write over a newer edit is withheld, and nothing that could be the edit is | [`flux_test.go`](../../internal/applier/flux_test.go), [`apply_window_test.go`](../../internal/git/apply_window_test.go), [`last_field_managers_test.go`](../../internal/watch/last_field_managers_test.go) | +| The operation is recorded before the write that starts it, moves through its phases on the trigger's status, an attempt that asked nothing restores the record before it, and a withheld rollback makes a `Repair` due | [`apply_operation_test.go`](../../internal/git/apply_operation_test.go), [`status_writer_test.go`](../../internal/applier/status_writer_test.go) | +| Every attempt rechecks targets, asks for nothing weaker than it was told, and resumes no `Kustomization` on another branch; a stop mid-apply suspends the `Kustomization` | [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go), [`flux_test.go`](../../internal/applier/flux_test.go) | +| The suspended lifecycle against the source facts: an apply is not finished when attempted; a token handled while suspended is no apply; a resume alone reverts a refusal; a suspend does not stop a running apply; the artifact is rechecked before the resume; a failed apply is suspended and Flux's requeue applies nothing; a shared source applies only the consumer asked | [`flux_lifecycle_test.go`](../../internal/applier/flux_lifecycle_test.go), [`flux_consumers_test.go`](../../internal/applier/flux_consumers_test.go), [`fluxsim`](../../internal/applier/fluxsim/fluxsim_test.go) | +| Adoption, `SuspendOwnedElsewhere`, an external resume suspended again with an Event every time, the hand-back on deletion (a `ClusterProvider` gone included), and the source's other consumers | [`fluxtrigger_hold_test.go`](../../internal/controller/fluxtrigger_hold_test.go) | +| The rule: a trigger nothing names reads nothing and is `Ready`; the first eligible reference adopts and the last releases, suspending an apply in flight first; a reference returning mid-release holds again; a re-created `GitTarget` releases and re-adopts; the binding over several `GitTarget`s; an older trigger not in line keeps none out, and a retiring holder stays first until its release is done; a holder left unconfigured says the `Kustomization` is suspended; `ApplierNamespaceNotAllowed` for both kinds | [`applytrigger_rule_test.go`](../../internal/controller/applytrigger_rule_test.go) | +| `allowAnySourceNamespace: true` is refused on create and on any spec change, a stored one included; `false` and its removal are accepted; a `WatchRule` that relied on it is refused naming the new field | [`removed_allow_any_source_namespace_test.go`](../../internal/controller/removed_allow_any_source_namespace_test.go), [`watchrule_source_namespace_test.go`](../../internal/controller/watchrule_source_namespace_test.go) | +| A restarted worker follows an operation left in flight with its window open, restores `repairPending`, and follows an apply started elsewhere; one recorded but never resumed ends `Failed` | [`apply_recovery_test.go`](../../internal/git/apply_recovery_test.go), [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go) | +| Appliers sharing a source have one operation at a time; a push is prefetched only while the branch is busy and no operation of the source is in flight | [`apply_groups_test.go`](../../internal/git/apply_groups_test.go), [`flux_consumers_test.go`](../../internal/applier/flux_consumers_test.go) | +| An applier in another cluster is reached through its own `ClusterProvider`, behind that provider's `accessFrom` | [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go), [`clusterprovider_admission_test.go`](../../internal/authz/clusterprovider_admission_test.go), [`applytrigger_controller_test.go`](../../internal/controller/applytrigger_controller_test.go) | +| The applier's late write over a newer edit is withheld, and nothing that could be the edit is: a tie, and an undo that leaves an older applier entry, are mirrored | [`apply_window_test.go`](../../internal/git/apply_window_test.go), [`last_field_managers_test.go`](../../internal/watch/last_field_managers_test.go) | +| A newly named `Kustomization` gets a `CatchUp`, and a newly named `Application` none | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go) | | The receiver's path names no provider, and every miss answers the same | [`receiver_test.go`](../../internal/webhook/receiver_test.go) | | The original race: two API edits end at the later value with no extra commit | [bi-directional Flux corner](../../test/e2e/flux_bi_directional_e2e_test.go) | -| `SourceMismatch` is never stamped and is delivered once fixed; a Gitea webhook and the refresh alone each get a push applied | [Manager Flux Applier](../../test/e2e/flux_applier_e2e_test.go) | -| A refused edit is reverted by its `Kustomization` | [Refusal reapply](../../test/e2e/refusal_reapply_e2e_test.go) | +| A trigger nothing names is `Ready` and holds nothing; `SourceMismatch` adopts nothing and never stamps the other branch's source; fixed, the `Kustomization` is adopted and Flux reaches the tip; dropping the reference hands it back and naming the trigger again adopts it; a Gitea webhook and the refresh alone each get a push applied; a failed apply is suspended and retried by the worker; a `flux resume` is suspended again; deleting the trigger hands the `Kustomization` back | [Manager Flux Applier](../../test/e2e/flux_applier_e2e_test.go) | +| A manager restarted during an apply suspends the `Kustomization` as it stops; the new one completes the same operation and leaves it suspended | [Flux Restart Recovery](../../test/e2e/flux_restart_recovery_e2e_test.go) | +| A refused edit is reverted by a resume of its `Kustomization` | [Refusal reapply](../../test/e2e/refusal_reapply_e2e_test.go) | | The Flux guide's example is accepted by the code | [`flux_guide_test.go`](../../internal/applier/flux_guide_test.go) | -| An Argo CD sync is the one automated sync builds, pinned to the commit, with a pinned retry; a running operation is waited out and never replaced; an operation set between the read and the update makes it fail; every `ApplierConfigured` reason, `AutomatedSyncEnabled` among them; when a sync counts as finished | [`argocd_test.go`](../../internal/applier/argocd_test.go), [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go) | -| An `Application` is never owed a `CatchUp`; Argo CD's late write over a newer edit is withheld by its own managers | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../../internal/git/apply_window_test.go) | -| Exactly one of `kustomizationRef` and `applicationRef`, and no `quietPeriod` with an `applicationRef` | [`gittarget_applier_schema_test.go`](../../internal/controller/gittarget_applier_schema_test.go) | +| An Argo CD sync is the one automated sync builds, pinned to the commit, with a pinned retry; a running operation is waited out and never replaced; an operation set between the read and the update makes it fail; one whose `.operation` was removed while it ran ends; every `Configured` reason, `AutomatedSyncEnabled` among them; when a sync counts as finished | [`argocd_test.go`](../../internal/applier/argocd_test.go), [`dispatcher_test.go`](../../internal/applier/dispatcher_test.go) | +| An `Application` is never asked for a `CatchUp`; Argo CD's late write over a newer edit is withheld by its own managers | [`apply_trigger_test.go`](../../internal/git/apply_trigger_test.go), [`apply_window_test.go`](../../internal/git/apply_window_test.go) | +| Each trigger kind holds one applier in one cluster, immutably; `quietPeriod` is bounded and defaulted on a `FluxTrigger` alone; a trigger nothing names is `Ready` with no finalizer, and a `GitTarget` naming it re-runs its check | [`applytrigger_schema_test.go`](../../internal/controller/applytrigger_schema_test.go) | | With Argo CD: two API edits keep the later value with no extra commit; a Git-side push is synced by a pinned `ForeignCommit`; a refused edit is reverted by a sync; an `Application` with automated sync on is `AutomatedSyncEnabled` and never synced | [bi-directional Argo CD corner](../../test/e2e/argocd_bi_directional_e2e_test.go) | | The Argo CD guide's example is accepted by the code | [`argocd_guide_test.go`](../../internal/applier/argocd_guide_test.go) | -The history of how this was decided, with the options weighed and the Git-side reference the code -cites, is in [`../finished/push-notification-and-reconcile-trigger.md`](../finished/push-notification-and-reconcile-trigger.md). +The history of how the first trigger was decided, with the options weighed and the Git-side +reference the code cites, is in +[`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md). diff --git a/docs/design/azure-devops-multi-ack.md b/docs/design/azure-devops-multi-ack.md index 88b255e09..eee4b2253 100644 --- a/docs/design/azure-devops-multi-ack.md +++ b/docs/design/azure-devops-multi-ack.md @@ -476,7 +476,7 @@ exists to avoid — but it is scoped to ADO, and it is provably enough for Flux' [the capability matrix](#the-capability-matrix), `PushAtomic` speaks `receive-pack`, which has no `multi_ack`, and the two guards that make it atomic live in the advertisement of that session and in the server-side compare-and-swap. A clone-fresh read path changes what fills the object store, not -how we write. The only obligations it inherits are the ones +how we write. The only requirements it inherits are the ones [the shallow-store section](#the-one-real-interaction-shallow-object-stores) names: leave `rootHash` present as the tip, and leave `HEAD` on the working branch — which is what the post-fetch branch setup does today anyway, since a fresh clone lands `HEAD` on the remote default branch. diff --git a/docs/design/branch-worker-event-model.md b/docs/design/branch-worker-event-model.md index b44158732..9c81c07c9 100644 --- a/docs/design/branch-worker-event-model.md +++ b/docs/design/branch-worker-event-model.md @@ -25,7 +25,7 @@ what this section asked for, in the order it set: ordered pending writes, one ma checkout, and one retry deadline; then the watch's admission boundary (5a), Git operation deadlines (6), intake pause and resume (5b), and the publication status (5c). None of it depends on a storage backend or on HA. A saturated branch pauses intake while its recovery loop stays alive; a process -restart still loses the in-memory obligations. +restart still loses the pending writes and unanswered saves held in memory. What comes next is the [transition boundary](#prepare-the-transition-boundary-for-later-durability): recording timers, attaches, withdrawals, and refreshes as transitions, then the HA plan's journal. @@ -41,8 +41,8 @@ recording timers, attaches, withdrawals, and refreshes as transitions, then the ignores the context past the dial. Timed-out work stays pending, and a push whose reply was lost is settled as published when the remote is found at the commits it sent. 3. **Kept.** The worker resolves publication, or establishes that the save cannot publish, before it - reports a terminal failure; a controller timeout alone cannot. `WaitingForPush` holds while a - recoverable publication obligation remains, and since step 5c its message says why. + reports a terminal failure; a controller timeout alone cannot. `WaitingForPush` holds while the + save's pending write can still be published, and since step 5c its message says why. 4. **Built.** The retry schedule takes an injectable clock; the stall tests run HTTP and SSH servers that stop answering, and the call returns on the worker goroutine, which stays the only owner of the checkout. @@ -81,7 +81,7 @@ described below. Serialization is useful independently of persistence. The proposed ownership rule is: > The worker decides each transition from explicit inputs and recorded observations. Its state -> and outstanding obligations are recoverable before the corresponding external work starts. +> and the work it has still to do are recoverable before the corresponding external work starts. Git operations can initially remain synchronous on this owner. Separating decision logic from effects does not require another goroutine or a workflow service. A later asynchronous executor @@ -187,11 +187,11 @@ once the cooldown had elapsed every later commit attempted the same unavailable deadline now governs those commits ([push-cooldown design](push-cooldown.md#7-options), option C). -The retry follows [parent recovery](../../internal/git/parent_recovery.go), which already kept an -obligation open and scheduled its next attempt (10 seconds, backing off to five minutes, shared -across targets on the worker). While the parent is missing, that schedule suppresses early -attempts. After the parent returns, the recovery obligation stays open until the work publishes, -but its missing-parent hold is removed. A publication that fails while the obligation is open +The retry follows [parent recovery](../../internal/git/parent_recovery.go), which already stayed +open until its held writes were published and scheduled its next attempt (10 seconds, backing off +to five minutes, shared across targets on the worker). While the parent is missing, that schedule +suppresses early attempts. After the parent returns, parent recovery stays open until the work +publishes, but its missing-parent hold is removed. A publication that fails while recovery is open therefore defers to recovery's next probe deadline: new commits wait for it, and recovery's timer makes the attempt. Review of #412 found the handoff gap (each new commit retried at once); `TestPublicationRetry_AFailureAfterTheParentReturnsWaitsForRecovery` pins the fix. @@ -288,14 +288,15 @@ barrier or per-stream watermark design can establish that stronger claim. A time ## Proposed recovery contract -Make durable acceptance the point at which the system owes a recoverable answer: +Make durable acceptance the point from which every caller is guaranteed an answer, even across a +restart: -> Once work is durably accepted, a restart preserves its ordering and save obligations. +> Once work is durably accepted, a restart preserves its ordering and the saves still to be answered. > Publication resumes when Git becomes available, within configured storage limits and subject > to the recorded write-policy decisions. Status can be rebuilt from durable outcomes. An accepted item may still be refused by policy or resolve without a commit. Record that outcome -instead of silently losing the item. A transient publication failure keeps the obligation open. +instead of silently losing the item. A transient publication failure keeps the item pending. Define explicit behavior for target deletion, repository replacement, and administrator cancellation; never reroute accepted work to a new object merely because its name matches. @@ -315,9 +316,9 @@ Represent intent, decisions, and external work separately so recovery knows what |---|---|---| | Command | `SaveRequested` | A caller asks to attach a save | | Fact | `SaveAttached` | The worker bound that request to a particular window | -| Effect intent | `PublishBatch` | A recorded obligation to attempt a push | +| Effect intent | `PublishBatch` | A recorded decision that a push must still be attempted | | Effect result | `PublicationConfirmed` | Evidence that a particular attempt published its work | -| Projection obligation | `ProjectSaveOutcome` | Kubernetes status still needs the recorded result | +| Pending projection | `ProjectSaveOutcome` | Kubernetes status still needs the recorded result | These names are illustrative. Use a versioned journal schema when choosing the implementation. Facts describe decisions already made; a duplicate command must find its previous decision. @@ -341,7 +342,7 @@ goroutine. The state machine applies recorded facts without reading the clock, c or contacting Git. Decision handling can request those observations as effects, then continue when their results have been recorded. -A transition must leave its next obligation recoverable. The HA plan defines the +A transition must leave the next step it requires recoverable. The HA plan defines the [atomic processing and acknowledgment boundary](../future/ha-gittarget-distribution-plan.md#atomic-handoff-and-acknowledgment) that preserves this across a crash. Redelivery must find the prior decision or resume unfinished work. Recording a push intent alone does not prove remote success. @@ -386,8 +387,8 @@ PublicationConfirmed(batch ID, attempt ID, per-write commit mapping) SaveResolved(request UID, outcome, branch, commit) ``` -Refusal, withdrawal, and no-window expiry have their own terminal facts. `CommitEmpty` is a -publication obligation even though its tree matches the parent. A foreign-window mismatch keeps +Refusal, withdrawal, and no-window expiry have their own terminal facts. `CommitEmpty` must still +be published even though its tree matches the parent. A foreign-window mismatch keeps its existing no-commit meaning. A remote-confirmed no-op and a local plan with no diff remain different observations. @@ -418,11 +419,11 @@ The current worker has six timer sources: | Push timer | Space successful publication cycles | | Retry | Retry a failed publication on its backoff, or probe for a missing parent | | Attach timeout | Settle a save still waiting for an eligible window | -| Refusal action | Recheck consent before a coalesced Refusal request is owed | +| Refusal action | Recheck consent before a coalesced Refusal request is due | | Applier | Send an apply attempt that came due on a quiet branch | The retry schedule has its own timer; the push timer is only the success cooldown. Parent -recovery has no timer of its own since #413: while its obligation is open, the retry deadline is +recovery has no timer of its own since #413: while recovery is open, the retry deadline is its probe. A durable design must persist that one deadline and what it is for. This scheduling does not limit the fetches other handlers can initiate. @@ -451,8 +452,8 @@ transition was already committed. Missed retries during downtime should produce attempt, followed by the normal backoff, rather than a burst of historical retry attempts. Not every input advances a phase. Duplicates and stale firings can be no-ops; a failed push records -an obligation and a next opportunity. The liveness rule is that every unresolved obligation has -an identified event that can service it, including when the branch receives no more writes. +the work still pending and when to try it next. The liveness rule is that every piece of pending +work has an identified event that can service it, including when the branch receives no more writes. ## Configuration and observations need explicit effective points @@ -520,7 +521,7 @@ Journal crashes, retention, encryption, and HA failure tests belong to the | Save's first push fails, then the branch goes quiet | Built (#412): retry publishes and resolves the same request without another edit | | Held save outlives the controller's safety window | Built (#412): withdrawal cannot falsely fail it; recovery remains scheduled | | Push fails under continuous arrivals | Built (#412): arrivals cannot bypass the retry budget | -| Parent returns, but publication still fails | Built (#413): new commits respect the one retry deadline while the obligation remains open | +| Parent returns, but publication still fails | Built (#413): new commits respect the one retry deadline while recovery remains open | | Retained-write rebuild fails before a new window commits | Built (#413): decided window and attached save stay pending for retry | | Intake fills during publication failure | Built (#413): accepted work stays; producers pause while the retry loop remains active | | Rejected UPDATE is redelivered from the same cursor | Built (#413): dedup does not suppress work that never entered the worker | @@ -528,8 +529,8 @@ Journal crashes, retention, encryption, and HA failure tests belong to the | Worker is canceled during a failed attempt | No retry is rearmed and no abandoned task retains checkout access | | Timer moves or fires twice | Stale and duplicate firings do not close another window | | Recorded decisions are replayed | Identical state returns without external effects | -| Fresh snapshot follows an accepted save | Coalescing preserves the save's membership and obligation | -| Parent disappears again during recovery | Earlier publication cannot clear newer retained obligations | +| Fresh snapshot follows an accepted save | Coalescing preserves the save's membership and its pending answer | +| Parent disappears again during recovery | Earlier publication cannot clear newer retained writes | | Policy changes during a window or push | Its specified effective point remains unchanged | | Resource update and save arrive on independent paths | Only the documented attachment ordering is claimed | @@ -560,8 +561,8 @@ a clean checkout can still have lost the commits behind retained work. The detailed design records continue to own adjacent policy: - [Parent contract](gittarget-parent-branch.md) and [hardening](gittarget-parent-hardening.md): - parent advertisement checks, empty-repository boundaries, bounded replay, and recovery - obligations remain intact. Git compare-and-swap protects the write branch and does not lock + parent advertisement checks, empty-repository boundaries, bounded replay, and parent recovery + remain intact. Git compare-and-swap protects the write branch and does not lock the parent after its advertisement. - [Parent observation](gittarget-parent-observation.md): deferred status work that adds no remote connections or publication permission. Incomplete ancestry evidence remains `Unknown`. diff --git a/docs/design/data-plane-triggering.md b/docs/design/data-plane-triggering.md index ea6a5bb4c..6b6b5633c 100644 --- a/docs/design/data-plane-triggering.md +++ b/docs/design/data-plane-triggering.md @@ -246,11 +246,11 @@ retained anyway. Shipped so far: the storm source is fixed and resyncs coalesce ([#312](https://github.com/ConfigButler/gitops-reverser/pull/312)); the ordering -hazard that coalescing introduced is fenced; and collection identity is settled, +hazard that coalescing introduced is fenced; collection identity is settled, with one stream per collection and the producing collection stamped on queued -work. The diff itself is unbuilt. +work; and the diff is built and applied. Removal semantics are not. -The writes stay a log throughout. Only the resync half of the queue is in +The writes stay ordered pending writes throughout. Only the resync half of the queue is in question, and the answer is to make the stream set incremental rather than to replace the queue. diff --git a/docs/design/event-pipeline-overview.md b/docs/design/event-pipeline-overview.md index 4e91b53ff..ce73d8add 100644 --- a/docs/design/event-pipeline-overview.md +++ b/docs/design/event-pipeline-overview.md @@ -367,7 +367,7 @@ and only the reply was lost, so the cycle settles as published instead of replay ## 6. Retry and intake -One retry deadline covers everything the worker still owes: 10 seconds, doubling to 5 minutes. +One retry deadline covers all of the worker's pending writes: 10 seconds, doubling to 5 minutes. While a parent branch is missing, the deadline's attempt is a single advertisement probe; otherwise it materializes and pushes. The deadline records the failure it is waiting out and when the outage began. The worker publishes both, with the intake pause, as its publication report: @@ -380,7 +380,7 @@ stateDiagram-v2 Healthy --> RetryPending: an attempt failed
scheduleRetry(cause) RetryPending --> RetryPending: new decisions wait
(no connection spent);
a resync is answered with the cause RetryPending --> Attempt: retry deadline fires - Attempt --> Healthy: published, nothing owed
clearRetry(): intake reopens + Attempt --> Healthy: published, nothing pending
clearRetry(): intake reopens Attempt --> RetryPending: failed again
backoff doubles state RetryPending { @@ -555,7 +555,7 @@ remains: ### High availability -- **One process owns a branch.** There is no failover, no handoff of a worker's obligations, and +- **One process owns a branch.** There is no failover, no handoff of a worker's pending writes, and no shared journal. Owner: HA plan; deferred. ## Where to read the code diff --git a/docs/design/gittarget-branch-worker-pending-writes.md b/docs/design/gittarget-branch-worker-pending-writes.md index 999150fb0..67045061a 100644 --- a/docs/design/gittarget-branch-worker-pending-writes.md +++ b/docs/design/gittarget-branch-worker-pending-writes.md @@ -44,7 +44,7 @@ This table records the original machinery; the built steps below describe what r |---|---|---| | Ordered pending writes | `pendingWrites`: each entry keeps its events, message, author, policy snapshot and attached save | A decision becomes a pending write only after its local commit succeeds, so a failed commit discards the decision: `dropFailedWindow`, five drop branches, and parent recovery's dropped `scopes`, which re-derive lost decisions through snapshots | | Projection of the pending writes | The checkout: a root commit plus one commit per entry, which replay already rebuilds from the pending writes | Three flags guess whether the projection is still valid (`worktreeDirty`, `replayRequired`, `rootParentStale()`); six `recoverRetainedWrites` call sites consult them, and `TestEveryLoopCommitPathRecoversADirtyWorktree` exists to catch a seventh that forgets; five functions rebuild the projection in slightly different ways | -| Deadline for an open obligation | Two clocks: `publicationRetry` and parent recovery's probe timer | `deferToRecovery`, the `parentProbeHold` atomic consulted in four places, `awaitingPush` | +| Deadline for work still pending | Two clocks: `publicationRetry` and parent recovery's probe timer | `deferToRecovery`, the `parentProbeHold` atomic consulted in four places, `awaitingPush` | | Outcomes | Saves resolve on push | `pcr.committed`, a flag that mirrors "this save rides a pending write" | The original window-loss defect followed from the first row: `finalizeOpenWindowWithReason` ran @@ -57,10 +57,11 @@ Three pieces of loop-owned state: - **`pendingWrites`**: decided entries, in order, that are not yet published. Entry kinds are the existing `PendingWrite` kinds (window, atomic, resync, request record, refusal touch). Accepted entries - survive saturation. There is no `owedSnapshot` entry replacing discarded accepted work. + survive saturation. There is no `owedSnapshot` entry (a placeholder that would request a snapshot + again) replacing discarded accepted work. - **`checkout`**: `{root, rootGen, applied}`. The checkout equals `root` plus the commits of `pendingWrites[:applied]`; `applied` unknown means the projection must be rebuilt. -- **`retry`**: the single obligation deadline. Built fields are `backoff`, `due`, and its timer; +- **`retry`**: the single deadline for the pending writes. Built fields are `backoff`, `due`, and its timer; cause, first-failure time, and last error are proposed diagnostic state in step 5. Three operations: @@ -91,7 +92,7 @@ Do not add a second authoritative phase field or another retry timer. `Recovering` is the due attempt while paused. A successful advertisement or fetch alone does not reopen admission: read access does not prove that a branch-protected remote accepts pushes. A -completed publication, or explicit terminal settlement of all accepted obligations, does. Keep +completed publication, or explicit terminal settlement of all accepted entries, does. Keep admission closed across partial replay progress and a transient drop below the high-water mark. Include already admitted FIFO payloads, the open window, and deferred work when deciding whether that backlog has settled. A healthy FIFO that briefly fills waits for capacity; it must not wait @@ -159,7 +160,7 @@ These findings gated steps 5a to 5c; the markers above say where each was fixed. A second review, at `fc02f51d`, reproduced four more. All are fixed in the same branch: -- **Recovery could stay open with nothing owed.** A resync held while the parent was missing could +- **Recovery could stay open with nothing pending.** A resync held while the parent was missing could find nothing to change once it was back. The last pending write settled without a push, so neither recovery nor its retry closed, and intake stayed paused. `advance` now ends the outage whenever nothing is pending (`TestParentRecovery_EndsWhenTheHeldWorkNeedsNoCommit`). @@ -227,8 +228,8 @@ handling. This step closes them, in [`pending_writes_loop.go`](../../internal/gi - Every commit re-reads its write's prune policy, as a replay does (`TestDecidedWrite_ADeferredDeleteObeysATightenedPrunePolicy`, a review finding). The read comes from the informer cache. -- Parent recovery never opens an obligation with nothing owed - (`TestParentRecovery_AFailedEmptySaveOwesNothing`, a review finding). +- Parent recovery never opens with nothing pending + (`TestParentRecovery_NothingPendingOpensNoRecovery`, a review finding). - `pcr.committed` is gone: `attached` means bound to a write, a window's or a save's own record. - `rebuildPendingWrites` folded into `replayOntoRemote`. Two replay functions remain on purpose: `replayOntoRemote` is the core the push cycle calls under its lock, and @@ -269,10 +270,10 @@ the way a live write's is. Two decisions, settled 2026-10-02: them and every save boundary are kept. The admission threshold limits payload growth; the FIFO still coalesces resyncs that are only queued. -With nothing dropped, nothing is owed a snapshot: parent recovery's `scopes`, `awaitingPush`, the -snapshot-request sequence and the controller's snapshot-request tracker are gone. The parent -recovery tests now pin that writes decided across one or two outages are kept and published, and -that a failure leaving nothing pending opens no obligation. +With nothing dropped, no snapshot has to be requested again: parent recovery's `scopes`, +`awaitingPush`, the snapshot-request sequence and the controller's snapshot-request tracker are +gone. The parent recovery tests now pin that writes decided across one or two outages are kept and +published, and that a failure leaving nothing pending opens no recovery. ### Step 4: a refused replay drops only its own entry (built) @@ -383,9 +384,9 @@ producer wait. The intake gate is [`intake.go`](../../internal/git/intake.go). size. The charge is serialized YAML, not heap: a queued object is held as an unstructured map of about six times that. Overshoot is limited to work accepted before the outage began. - **Pause is a latch.** A payload that would cross the budget is refused and pauses the branch; - every later payload is refused until the retry clears, which means a publication landed or - nothing is owed. Room under the budget again, a partial replay, or a remote that can be read but - refuses the push keep it paused. A healthy branch never pauses. + every later payload is refused until the retry clears, which means a publication landed or no + pending writes remain. Room under the budget again, a partial replay, or a remote that can be + read but refuses the push keep it paused. A healthy branch never pauses. - **Oversized items.** A snapshot larger than the whole budget is accepted on a healthy branch, whose pending writes drain at the next push, and the window cap finalizes around it as before. During an outage it can never fit: it is refused with a diagnostic naming its size and the budget, pauses @@ -552,7 +553,7 @@ separate jobs, and only one of them can wait: Settled 2026-10-02: 1. **Resyncs stay pending**, as described in [Resyncs among the pending writes](#resyncs-among-the-pending-writes). Capacity - closes admission; it never replaces accepted entries with an owed snapshot. + closes admission; it never replaces accepted entries with a request for a fresh snapshot. 2. **A partly failed write is cleaned locally** (step 1b). 3. **Follow repository validation and push rules.** This changes the Git write and watch paths, so the high-risk exception in `AGENTS.md` requires a local e2e pass before an implementation diff --git a/docs/design/gittarget-parent-branch.md b/docs/design/gittarget-parent-branch.md index d891bf59d..374ca6477 100644 --- a/docs/design/gittarget-parent-branch.md +++ b/docs/design/gittarget-parent-branch.md @@ -5,7 +5,7 @@ > records the contract as built and the reasoning behind it. Deferred: > [`gittarget-parent-empty-repository.md`](gittarget-parent-empty-repository.md) and > [`gittarget-parent-observation.md`](gittarget-parent-observation.md). -> Related: [`push-notification-and-reconcile-trigger.md`](../finished/push-notification-and-reconcile-trigger.md), +> Related: [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), > [`../definitions.md`](../definitions.md), and `architecture.md` (Ground rules; "A branch that does > not exist yet"). @@ -59,10 +59,10 @@ check is what protects the first write. | no | no, omitted, not empty (`HEAD` resolves to no branch, or only tags) | refuse the same way: `ParentBranchNotFound`, with a message to fix `HEAD` or set the field | | no | no, omitted, empty repository (no refs at all) | unborn `HEAD`; the first commit is a root commit on the write branch | -- **Recovery from a missing parent** is an obligation on the branch worker, not an observation. It - latches when an attempt fails on the parent and clears only when the work is published. Writes, - saves and resyncs decided while the parent is missing are kept as the worker's pending writes, - not dropped, and admission backpressure bounds them +- **Recovery from a missing parent** is held open by the branch worker until the held work lands, + whatever the parent's observed state. It latches when an attempt fails on the parent and clears + only when the work is published. Writes, saves and resyncs decided while the parent is missing + are kept as the worker's pending writes, not dropped, and admission backpressure bounds them ([`gittarget-branch-worker-pending-writes.md`](gittarget-branch-worker-pending-writes.md), step 3c). The worker probes with one advertisement after 10s, doubling to 5m, on its one retry schedule shared by every target on it; before a deadline nothing spends a connection on the parent, and reconciles never probe. diff --git a/docs/design/gittarget-parent-hardening.md b/docs/design/gittarget-parent-hardening.md index be0e1af0c..aa4827bbb 100644 --- a/docs/design/gittarget-parent-hardening.md +++ b/docs/design/gittarget-parent-hardening.md @@ -14,7 +14,7 @@ > it the dangling name (§4.3 test 1's `master`) never reaches the client: the observation's > branch is empty and the message says the remote has no default branch. > - Once the probe has found the parent it stops reading advertisements: later deadlines retry the -> retained writes and re-request the snapshots still owed. A failure caused by the parent arms +> retained writes and re-request the snapshots still missing from Git. A failure caused by the parent arms > the latch again. > > **The merge boundary.** #407 merges when this pass is done: the existing parent-branch feature @@ -30,14 +30,14 @@ Both are fixed, each with the reviewer's reproduction as its regression test (red before the fix). Both failures were reproduced through the worker event handlers against canonical `git http-backend`. -1. **P1: a second parent outage loses the obligation for newer dropped writes.** A recovery +1. **P1: a second parent outage forgets to republish newer dropped writes.** A recovery snapshot commits locally and enters `awaitingPush`. Before it is pushed, the parent disappears again and another event from the same scope is dropped. `noteParentUnavailable` keeps the scope's old `awaitingPush` entry. On recovery, `noteRecoveryPublished` clears that scope after publishing the older snapshot, and no new snapshot is requested. The reproduction publishes `cm1`, leaves `cm2` absent, closes recovery, and leaves `SnapshotRequestSeq` at 1. Invalidate - the old publication credit when new work is dropped, or track the obligation's generation. - Acceptance: publishing the old snapshot keeps the newer obligation open, requests a fresh + the old publication credit when new work is dropped, or track the generation of the dropped work. + Acceptance: publishing the old snapshot keeps recovery open for the newer write, requests a fresh snapshot, and eventually publishes `cm2` without another cluster edit. **Fixed:** a dropped write removes its scope's publication credit; pinned by `TestParentRecovery_ASecondOutageKeepsTheWorkDroppedInIt`. @@ -226,7 +226,7 @@ reconciles and refresh ticks occur. §4.4 defines it and tests it. - e2e does not assert which fetch reason found B (§4.6). - **Default-branch resolution follows go-git, not a stricter rule of our own** (§4.3). - **Recovery comes before cost.** §4.4 writes the recovery tests first. If the worker-side probe - grows larger than expected, ship the recovery obligation and its tests with a simpler + grows larger than expected, ship the recovery latch and its tests with a simpler controller-side backoff (§4.4, fallback). Never weaken recovery to save fetches. ## 4. Work items, in order @@ -481,7 +481,7 @@ Update the documentation in three places: - A failed `pushPending` stops its push timer (~1673), and nothing re-arms it until the next event. - With `--git-refresh-interval=0` there is no refresher at all. -- An observation flip is not an obligation: +- An observation flip does not show that the held work landed: - If the first recovery attempt after `Found` fails transiently, the observation stays `Found`, the timer is stopped, a probe gated on `Missing` no longer runs, and the work is stranded. - A controller that keys off "the stored Ready reason was `ParentBranchNotFound`" can miss a @@ -489,7 +489,7 @@ Update the documentation in three places: #### Design -1. **A recovery obligation on the worker.** Its lifetime is decoupled from observations. +1. **A recovery latch on the worker.** Its lifetime is decoupled from observations. - The worker latches `parentRecovery` the moment a cycle fails with `ErrParentBranchNotFound` or `ErrDefaultBranchUnresolved`. - The latch has two independent halves: @@ -536,7 +536,7 @@ Update the documentation in three places: - Drop the per-reconcile `parentWasMissing` force. - Ready reporting: - `ParentBranchNotFound` stays Ready=False/Stalled while the parent is missing. - - While the obligation is open but the parent has been found, report Ready=False with + - While recovery is open but the parent has been found, report Ready=False with `Reconciling=True`, reason `RecoveringParentBranch`. This keeps the not-Ready requeue cadence, so the controller sees the sequence within one requeue, and the state stays visible. Verify that the not-Ready requeue applies to that reason. @@ -586,7 +586,7 @@ Write the recovery tests first, against current code. They are the safety net fo #### Fallback, if step 2 or 3 grows too large -- Keep step 1 (the obligation) and step 4 (the request sequence). The obligation is the +- Keep step 1 (the recovery latch) and step 4 (the request sequence). The latch is the correctness part and is non-negotiable. - Replace the worker probe with a controller-side forced recheck under exponential backoff, still gated on the latch rather than on the Ready reason. @@ -704,7 +704,7 @@ Kept here so they are not re-litigated. Each was checked against the code. - **No pull before push.** A pull cannot close the race; the compare-and-swap plus bounded replay does. Every fix in this pass reads evidence the push or discovery already holds. -- **Recovery is an obligation, not an observation.** It stays open until the work is published, +- **Recovery tracks the held work, not an observation.** It stays open until the work is published, so a transient failure after the parent reappears cannot strand it (§4.4). It is tracked per `(target, collection)` scope. - **A configuration change is applied at a push admission boundary**, with one `{name, gen}` diff --git a/docs/design/gittarget-state-of-affairs.md b/docs/design/gittarget-state-of-affairs.md index b65afa98a..e4e4a9840 100644 --- a/docs/design/gittarget-state-of-affairs.md +++ b/docs/design/gittarget-state-of-affairs.md @@ -117,12 +117,13 @@ lands immediately before a crash can still be followed by a duplicate empty save ## What the refactor got right Keep the separation between deciding, materializing, and publishing. It lets a failed fetch keep a -save obligation without guessing what a later snapshot would have meant. Step 4's refusal isolation -also prevents one invalid folder from blocking another target on the same branch. +save waiting for its answer without guessing what a later snapshot would have meant. Step 4's +refusal isolation also prevents one invalid folder from blocking another target on the same branch. -The earlier `owedSnapshot` proposal and a separate parent retry timer are unnecessary for the -chosen capacity policy. Keep accepted entries and reject new admission explicitly. The next change -should repair the acceptance boundary, not replace the pending writes or introduce a generic workflow engine. +The earlier `owedSnapshot` proposal (a placeholder that would request a fresh snapshot in place of +discarded work) and a separate parent retry timer are unnecessary for the chosen capacity policy. +Keep accepted entries and reject new admission explicitly. The next change should repair the +acceptance boundary, not replace the pending writes or introduce a generic workflow engine. ## Recommended order diff --git a/docs/design/label-selection-follow-ups.md b/docs/design/label-selection-follow-ups.md index ef33deb78..4f9f573b3 100644 --- a/docs/design/label-selection-follow-ups.md +++ b/docs/design/label-selection-follow-ups.md @@ -17,7 +17,7 @@ For context only. None of these is open. |---|---| | The recovery hint for `unowned-delete-patch` said "rename or remove". For an object a patch hides, the refusal's path is the hidden base document, so the hint read as "delete your manifest". | The hint names the patch and its kustomization reference. | | A batch holding two events for one inherited object read the pre-batch render for the second one. A re-entry followed by an upsert refused the batch as hidden by an unowned patch. A re-entry followed by a delete left the object rendered. | The batch records what it did to each inherited object's presence and consults that record before the pre-batch store. See `rendersInBatch` in [`plan_flush.go`](../../internal/git/plan_flush.go). | -| Refusal memory and queued `onRefusal` commits were keyed by the selector-bearing collection. After a selector change, the old key survived recovery. A first attempt fixed only the resync side and broke live refusals: those still used the selector-bearing key, so an accepted snapshot never cleared them, even with an unchanged selector. | Every refusal key is built by `newRefusalKey`, which drops the selector. The full collection is still used for watch identity and status. See [`refusal_reapply.go`](../../internal/git/refusal_reapply.go). | +| Refusal memory and queued `onRefusal` requests were keyed by the selector-bearing collection. After a selector change, the old key survived recovery. A first attempt fixed only the resync side and broke live refusals: those still used the selector-bearing key, so an accepted snapshot never cleared them, even with an unchanged selector. | Every refusal key is built by `newRefusalKey`, which drops the selector. The full collection is still used for watch identity and status. See [`refusal_reapply.go`](../../internal/git/refusal_reapply.go). | | `targetWatchSet.stop` retired the producer gate before cancelling the stream. An in-flight snapshot enqueue blocked in `Client.Get` on the stream context could hold the owner loop. | Cancel first, then retire. The gate still admits nothing afterwards. | About the second row: a commit window keeps only the last event per path, and every pending write diff --git a/docs/design/open-asks-priority.md b/docs/design/open-asks-priority.md index c6a16845d..6d2c3381b 100644 --- a/docs/design/open-asks-priority.md +++ b/docs/design/open-asks-priority.md @@ -223,7 +223,7 @@ Three things came out of building it that the argument above did not contain. - **Deleting the inference exposed a second implementation of a rule, not just the rule.** "Omit `metadata.namespace`, the build context supplies it" was read off a *sibling's bytes*, so it only ever fired for an inferred placement. A **declared** path into the same kustomize directory wrote a - `namespace:` line every other document in that folder omits. The obligation belongs to the + `namespace:` line every other document in that folder omits. The rule belongs to the kustomization that governs the destination, not to whatever document happened to be next door, and moving it there fixed the declared path for free. - **And that rule was missing its safety half.** The old kustomize-root fallback asked only whether a diff --git a/docs/design/placement-visibility-and-declared-defaults.md b/docs/design/placement-visibility-and-declared-defaults.md index becc5e1a8..3c2744197 100644 --- a/docs/design/placement-visibility-and-declared-defaults.md +++ b/docs/design/placement-visibility-and-declared-defaults.md @@ -37,7 +37,7 @@ states the findings behind each, the options, and the call. | Default `placement.default` in the CRD? | **No**, on narrower grounds than the first draft claimed | A defaulted default consumes the slot ahead of the one step no template can express ("beside the folder's one kustomization"). Once F10 is fixed this is a legibility trade rather than a correctness wall | | Publish the effective layout in status? | **Yes, now** | It gives the clarity the CRD default was reaching for, without freezing anything, and the seam it needs already exists | | `{kindLower}` or a `toLower` function? | **`{kindLower}`** | A function syntax is a language; one variable answers the actual need | -| Is the kustomize root "still a little bit inference"? | **Yes, and it earns its keep anyway** | Its answer changes only when a `kustomization.yaml` changes, and ignoring it produces a file nothing renders. It gets the same visibility obligation as everything else | +| Is the kustomize root "still a little bit inference"? | **Yes, and it earns its keep anyway** | Its answer changes only when a `kustomization.yaml` changes, and ignoring it produces a file nothing renders. It has to be made as visible as everything else | | A declared path in a subdirectory of a kustomize folder | **Bug, fix it**: walk up to the nearest kustomization | Today it is registered only when render-root scoping happens to be in force, so a `byType` entry into a subdirectory silently produces a file nothing renders (F10) | | Two kustomize roots (ambiguous) | **Keep writing, make it loud** | The file is currently unreachable *and* uncounted. Refusing would protect nothing that mirroring does not | @@ -483,8 +483,8 @@ object changing. It differs from Option C in two ways that decide whether it sta | Changes when | any document is added or removed | a `kustomization.yaml` is added or removed | | Cost of ignoring it | a file in an unusual place | a file **nothing renders** | -The third row is why it stays; the first two are why it is safe enough to keep. But the visibility -obligation is identical, and that is what `renderRoot` plus `renderRootReason` discharge: the +The third row is why it stays; the first two are why it is safe enough to keep. But it has to be +made just as visible, and that is what `renderRoot` plus `renderRootReason` provide: the judgement becomes a field an operator can read, instead of a behavior they infer from where files appeared. diff --git a/docs/design/push-cooldown.md b/docs/design/push-cooldown.md index 7dd8940b1..f254e3ab5 100644 --- a/docs/design/push-cooldown.md +++ b/docs/design/push-cooldown.md @@ -5,7 +5,7 @@ > unchanged. Option D needs measurement; option E remains rejected without that evidence. > Index: [`../INDEX.md`](../INDEX.md). Reviewed at `373bf8d7` on 2026-10-03. > Related: [`../api-first-publication.md`](../api-first-publication.md), -> [`push-notification-and-reconcile-trigger.md`](../finished/push-notification-and-reconcile-trigger.md), +> [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), > [`../spec/commit-window-refactor.md`](../spec/commit-window-refactor.md) **The question.** `PushCooldown` predates the commit window. The window now groups an editing burst @@ -238,7 +238,7 @@ soon as two are. Dropping the success wait is therefore a **measured** decision, and it belongs in option D behind the numbers §9 asks for. Before #412, `pushPending` stopped its timer on failure and armed no replacement outside parent -recovery. A quiet branch retained its work indefinitely, while continued arrivals could provoke +recovery. A branch with no new arrivals retained its work indefinitely, while continued arrivals could provoke a failed push per commit because `lastPushAt` advances only on success. #412 adds the independent failure backoff. Step 3b then unified the publication and parent-recovery deadlines. While parent recovery is open, the due attempt services it. The following diagram describes publication retries; @@ -269,7 +269,7 @@ Ordered by how much they should worry you. 1. **Request amplification on shared branches.** The architecture explicitly supports several `GitTarget`s per branch, and multi-tenant installs are the case where the cooldown is load-bearing. Option B turns row 4's `2` into `6`. This contradicts the premise of - [`push-notification-and-reconcile-trigger.md`](../finished/push-notification-and-reconcile-trigger.md), + [`push-notification-and-reconcile-trigger.md`](push-notification-and-reconcile-trigger.md), which spent a whole change removing two requests per publication. 2. **Hosted Git rate limits.** With serialization the bound is one push per push-duration, which on a fast remote is several per second sustained. Measure throttling on the supported hosts before diff --git a/docs/finished/push-notification-and-reconcile-trigger.md b/docs/design/push-notification-and-reconcile-trigger.md similarity index 97% rename from docs/finished/push-notification-and-reconcile-trigger.md rename to docs/design/push-notification-and-reconcile-trigger.md index 035d5833c..28e76cbb2 100644 --- a/docs/finished/push-notification-and-reconcile-trigger.md +++ b/docs/design/push-notification-and-reconcile-trigger.md @@ -1,9 +1,10 @@ # Push notification and the apply trigger -> **finished**: superseded on 2026-10-16 by [`../design/apply-trigger.md`](../design/apply-trigger.md), -> which is the current design and what is still open. This page is the record of how the apply -> trigger was decided. Its Part 1 (how the branch worker reads the remote) is still cited from the -> Git code by its section numbers, which predate the last restructuring of this page. +> **built, and superseded** on 2026-10-16 by [`apply-trigger.md`](apply-trigger.md), which is the +> current design and what is still open. This page is the record of how the apply trigger was +> decided. It stays in `design/` because its Part 1 (how the branch worker reads the remote) is +> still cited from the Git code by its section numbers, which predate the last restructuring of +> this page. GitOps Reverser and the applier (Flux or Argo CD) share a branch. This page is how they talk: how GitOps Reverser learns that somebody else pushed, and how it decides when the applier applies. @@ -364,7 +365,7 @@ URL normalization is the receiver's (§4.1): case-insensitive host and path, no ### 4.1 The receiver `GitProvider.spec.receiver` (`type`, `secretRef`), named after Flux's `Receiver`, served on its own -listener (`--receiver-bind-address`, Helm `receiver.bindAddress`, off by default), on the elected +listener (`--receiver-bind-address`, Helm `servers.receiver`, off by default), on the elected replica only. It cannot share the audit listener, which requires a client certificate signed by the audit CA that no Git host can present. @@ -439,7 +440,7 @@ Both are in [interpreting-metrics.md](../interpreting-metrics.md#apply-requests- ### 4.4 Chart -- `receiver.bindAddress` and a `gitops-reverser-receiver` Service when it is set. Exposing it is the +- `servers.receiver` and a `gitops-reverser-receiver` Service when it is enabled. Exposing it is the installer's Ingress or Gateway route. - RBAC for the operator's own cluster: `get` and `patch` on `kustomize.toolkit.fluxcd.io` `kustomizations` and `source.toolkit.fluxcd.io` `gitrepositories`; `get` and `update` on diff --git a/docs/design/reconcile-triggering.md b/docs/design/reconcile-triggering.md index 81a0e717d..17a89254d 100644 --- a/docs/design/reconcile-triggering.md +++ b/docs/design/reconcile-triggering.md @@ -1,6 +1,8 @@ # Reconcile triggering — how our controllers wake up -> **design**: open, not yet built. Index: [`../INDEX.md`](../INDEX.md) +> **design**: open. §5 is superseded by [the apply trigger](apply-trigger.md); F1 is built for +> `GitTarget` (`reconcile.configbutler.ai/requestedAt`), and F4 is built. +> Index: [`../INDEX.md`](../INDEX.md) > Date: 2026-06-29. A controller is only as good as the events that wake it. Periodic requeue is a @@ -279,14 +281,19 @@ flowchart LR H -- yes --> R2[reconcile affected GitTargets] ``` -> **The middle step of that flow has since been ruled out, and the rest of this section stands.** -> [inbound push notification](../finished/push-notification-and-reconcile-trigger.md) §7 shows that +> **Superseded: pull-on-push is built, and not as this section planned.** The Git host's webhook +> reaches `GitProvider.spec.receiver`, and a delivery is only a hint: the branch worker reads the +> remote itself and classifies the move as its own push or somebody else's, which is also how the +> self-push dedupe of §5.3 is done. The refresher catches what no webhook reports. Neither Path A +> nor Path B in §5.4 was taken. The design of record is [the apply trigger](apply-trigger.md). +> +> **Why the receiver does not patch the annotation:** +> [inbound push notification](push-notification-and-reconcile-trigger.md) §7 shows that > `requestedAt` drives a full cluster-to-Git snapshot, so a receiver that patches it can publish pre-push cluster state > over the very change it was told about, before Flux or Argo CD has applied it. The receiver needs > a request shape of its own: invalidate the base, record the new tip, replay retained work, and -> never snapshot. §8.3 of that page is the wire contract. Read everything below (the signature -> validation, the self-push dedupe in §5.3, the reuse question in §5.4) as still current; read -> "patch the annotation" as the one step that was replaced. +> never snapshot. §8.3 of that page is the wire contract. Read everything below as the options that +> were weighed before it. ### 5.1 Can we reuse the Flux component? @@ -450,6 +457,7 @@ Path A applies here too. through our own watch stream (§5.5), reusing the same credentials Secret. Near-zero end-user config for Flux users. Fall back to our own inbound git-webhook (§5.1) for non-Flux users. Both feed the subtree-OID drift check with §5.3 self-push dedupe. + **Superseded:** built as the push receiver instead; see the note in §5. 6. **Replace CommitRequest polling (A4)** with the F5 nudge. ## 7. Open questions diff --git a/docs/design/source-scope-simplification.md b/docs/design/source-scope-simplification.md index 2276fbb49..62c1babc7 100644 --- a/docs/design/source-scope-simplification.md +++ b/docs/design/source-scope-simplification.md @@ -8,6 +8,11 @@ > `sourceNamespace: "*"` section below remains the definition of record; the "Decided for the wave" > reading is now the shipped one. > +> `allowAnySourceNamespace`, which this page introduced, has since been replaced by +> `ClusterProvider.spec.allowCrossNamespace.sources`, unchanged in meaning, so that the grant for +> appliers in other namespaces could sit beside it ([`apply-trigger.md`](apply-trigger.md#decisions)). +> Read the field's old name below as the new one. +> > Answers a design review's proposal to adopt Flux-style `serviceAccountName` impersonation for > source reads, by declining it. Evidence for the impersonation half is in > [`../facts/kubernetes-impersonation-and-flux-identity.md`](../facts/kubernetes-impersonation-and-flux-identity.md), @@ -125,8 +130,8 @@ flowchart LR > `allowedSourceNamespaces` into a concrete set, one stream per namespace. Recorded here because the > migration note and several comments still refer to it. > -> **Shipped**: one cluster-wide list and watch, rejected outright while `allowAnySourceNamespace` is -> false. [`configuration.md`](../configuration.md#watching-a-different-source-namespace), +> **Shipped**: one cluster-wide list and watch, rejected outright while +> `allowCrossNamespace.sources` is false. [`configuration.md`](../configuration.md#watching-a-different-source-namespace), > [`architecture.md`](../architecture.md) and > [`watchrule_types.go`](../../api/v1alpha3/watchrule_types.go) describe this reading; they changed > in the same commit as the behavior. diff --git a/docs/design/support-boundary/README.md b/docs/design/support-boundary/README.md index bb4d51d76..f94595e58 100644 --- a/docs/design/support-boundary/README.md +++ b/docs/design/support-boundary/README.md @@ -85,8 +85,9 @@ distinguishes those planned gates from the permanent construct boundary. the staged design for safely editing it. - [Git write preflight](../git-write-preflight.md): onboarding refusal and all-target admission checks, with target errors for late failures. This replaces the per-edit accounting proposal. -- [Admission consent](admission-consent.md) and [reconcile trigger](../../finished/push-notification-and-reconcile-trigger.md): - deferred extensions; preflight does not promise automatic rollback. +- [Admission consent](admission-consent.md): a deferred extension. Reverting a refused edit on the + cluster is opt-in, through `onRefusal: Reapply` and [the apply trigger](../apply-trigger.md); + preflight itself promises no automatic rollback. - Historical implementation records: [image/replica edit-through](finished/images-and-replicas-edit-through.md) and [higher-level KRM support](finished/higher-level-krm-documents.md). - [Fixture corpus](../../../test/fixtures/gitops-layouts/) — executable evidence; its diff --git a/docs/design/support-boundary/admission-consent.md b/docs/design/support-boundary/admission-consent.md index e48938350..3416d1c2f 100644 --- a/docs/design/support-boundary/admission-consent.md +++ b/docs/design/support-boundary/admission-consent.md @@ -10,13 +10,13 @@ > [gittarget-granularity-and-cross-environment-edits.md](gittarget-granularity-and-cross-environment-edits.md) > — the write boundary; fan-in = 1; base read-only by L1, > [render-attribution.md](render-attribution.md) §5 — attribution may be heuristic, verification may not, -> [the reconcile trigger options](../../finished/push-notification-and-reconcile-trigger.md) -> — **the sibling half: what reverts a refusal that lands anyway**, +> [the apply trigger](../apply-trigger.md) (`onRefusal: Reapply`) +> — **the sibling half: what reverts a refusal that lands anyway**, built, > [support-contract.md](support-contract.md) This is one half of a two-part design. This half is about turning a *refusal* into a *yes* at the moment of the edit; the other half — -[the reconcile trigger](../../finished/push-notification-and-reconcile-trigger.md) +[the apply trigger](../apply-trigger.md), with `onRefusal: Reapply` — is about making the *outcome* of a "no" prompt and visible. They compose, but they are separate topics and separate documents. @@ -195,8 +195,9 @@ Two limits retained from the preflight design: flush-time oracle re-verifies the *whole batch* against the declared intents — including the consented ones — and still refuses if the consented set does not actually converge. So a stale or mistaken admission-time "yes" cannot cause a bad write. Worst case it lands, the flush refuses it, - and the affected target enters error. The [reconcile trigger](../../finished/push-notification-and-reconcile-trigger.md) - is a separate proposed recovery mechanism; automatic rollback is not part of preflight. + and the affected target enters error. Reverting it on the cluster is the separate, opt-in + `onRefusal: Reapply` through [the apply trigger](../apply-trigger.md); preflight itself does no + automatic rollback. --- diff --git a/docs/design/support-boundary/argocd-bi-directional.md b/docs/design/support-boundary/argocd-bi-directional.md index 73da3813e..3687b16f0 100644 --- a/docs/design/support-boundary/argocd-bi-directional.md +++ b/docs/design/support-boundary/argocd-bi-directional.md @@ -9,20 +9,11 @@ amount of configuration reconciles them — because the moment a live write land Argo already has everything it needs to revert it, and there is no hook where an external system can say "wait, this one is legitimate." -The Argo CD configuration that supports a shared field changing from **both** -directions is: - -- **automated sync off** (`syncPolicy.automated.enabled: false`), with the - `Application` named by the `GitTarget`'s `spec.applier.applicationRef`, so GitOps - Reverser starts every sync, pinned to a commit it checked - ([the Argo CD guide](../../argocd.md)), plus -- **a push webhook** into GitOps Reverser's receiver for the Git → cluster - direction. - -Automated sync with `selfHeal: false` was the answer before GitOps Reverser could -drive an `Application`, and is still the best an unnamed `Application` can do; it -leaves the timed refresh syncing GitOps Reverser's own pushes over whatever is -pending ([why](#why-selfheal-false-is-not-enough)). +The configuration that supports a shared field changing from **both** directions is +automated sync **off**, with the `Application` held by an `ArgoCDTrigger`, so GitOps Reverser +starts every sync, pinned to a commit it checked. The user guide is +[bi-directional GitOps with Argo CD](../../bi-directional-argocd.md); this record is the evidence +for why nothing short of that works. Everything else that looks like a solution — `ignoreDifferences`, `managedFieldsManagers`, three-way merge, PreSync hooks, self-heal timing knobs — @@ -36,7 +27,7 @@ line, which is materially identical for these paths). The configuration it leads to is exercised end-to-end in [`../../../test/e2e/argocd_bi_directional_e2e_test.go`](../../../test/e2e/argocd_bi_directional_e2e_test.go); corner design in -[`../e2e-bi-directional-corner.md`](../../spec/e2e-bi-directional-corner.md). +[`e2e-bi-directional-corner.md`](../../spec/e2e-bi-directional-corner.md). ## How fast is self-heal, and why it cannot be slowed @@ -154,47 +145,23 @@ setting. Verdict: **cannot be used to be slow.** ## The configuration that works -Turn automated sync **off** and let GitOps Reverser start every sync. It is -`selfHeal: false` taken to its end: nothing in Argo CD applies on its own, so the -only syncs are ones started on a quiet branch, pinned to a commit with nothing live -missing from it. Argo CD's own rules make it safe to drive this way: a sync is -started by setting the `Application`'s `operation` as automated sync would -(`controller/appcontroller.go` `autoSync`), and never while another operation is set -(`util/argo/argo.go` `SetAppOperation`). With automated sync off, a refresh only -compares (`autoSync` returns at once when `IsAutomatedSyncEnabled` is false). - -### Why `selfHeal: false` is not enough - -The intermediate step, still useful for an `Application` no `GitTarget` names, is -automated sync with self-heal **off**: - -- **Cluster → Git.** With `selfHeal: false`, Argo does not revert live drift — it - marks the app `OutOfSync` and waits. GitOps Reverser captures the live change to - Git at its own pace. You can be as slow as you like; nothing overwrites the - cluster in the meantime. -- **Git → cluster.** A push webhook (which the corner wires up: - `argocd-server` `/api/webhook`, Gogs-format from Gitea) makes Argo apply a new - Git commit within seconds — *including a change to the same shared field from the - Git side*, which `ignoreDifferences` would have broken. - -```text - API edit ─▶ Cluster ──watch──▶ GitOps Reverser ──commit──▶ Git - ▲ │ - └──────────── apply ◀── Argo CD (selfHeal off) ◀─push webhook┘ -``` - -This is bi-directional on a shared field, with one hole: automated sync still -applies **every new commit**, including each one GitOps Reverser pushes, two to three -minutes later on the timed refresh (or seconds later with Argo CD's webhook), over -whatever is pending at that moment. Two API edits a few seconds apart are enough to -have the first commit synced over the second edit. Only switching automated sync -off and letting GitOps Reverser start the syncs closes it, which is what the e2e -corner now exercises. - -**What you give up** is self-heal's one unique service: reverting drift that -*should not* be captured (a rogue or accidental change). That loss is inherent — -you cannot ask one flag to both "erase unauthorized drift" and "keep authorized -drift," because Argo has no notion of authority. +Turn automated sync **off** and let GitOps Reverser start every sync. Nothing in Argo CD then +applies on its own, so the only syncs are ones started on a quiet branch, pinned to a commit with +nothing live missing from it. Argo CD's own rules make it safe to drive this way: a sync is started +by setting the `Application`'s `operation` as automated sync would (`controller/appcontroller.go` +`autoSync`), and never while another operation is set (`util/argo/argo.go` `SetAppOperation`). With +automated sync off, a refresh only compares: `autoSync` returns at once when +`IsAutomatedSyncEnabled` is false. + +Automated sync with `selfHeal: false` was the answer before GitOps Reverser could drive an +`Application`, and is still the best an `Application` without a trigger can do. It stops the live +revert, and leaves one hole: automated sync still applies **every new commit**, GitOps Reverser's +own included, two to three minutes later on the timed refresh, over whatever is pending at that +moment. + +**What you give up** is self-heal's one unique service: reverting drift that *should not* be +captured. That loss is inherent: one flag cannot both erase unauthorized drift and keep authorized +drift, because Argo CD has no notion of authority. ## If you need "revert unauthorized drift" as well @@ -218,13 +185,13 @@ forgotten. | PreSync hook (veto) | **No** | also aborts legitimate deploys; can't tell them apart | | self-heal timeout / backoff | **No** | throttles repeats; first drift reverted immediately | | `selfHeal: false` + push webhook | **Mostly** | nothing reverts drift, but every new commit, GitOps Reverser's own included, is synced over pending edits | -| automated sync off + `spec.applier.applicationRef` | **Yes** | GitOps Reverser starts every sync, on a quiet branch, pinned to a checked commit | +| automated sync off + an `ArgoCDTrigger` | **Yes** | GitOps Reverser starts every sync, on a quiet branch, pinned to a checked commit | | admission gate on Argo's writes | (recovers "revert unauthorized") | per-write authority decision; external to Argo | ## References -- User-facing guidance: [`../../bi-directional.md`](../../bi-directional.md) and - [`../../argocd.md`](../../argocd.md) -- E2E corner design: [`../e2e-bi-directional-corner.md`](../../spec/e2e-bi-directional-corner.md) +- User-facing guidance: [`bi-directional.md`](../../bi-directional.md) and + [`bi-directional-argocd.md`](../../bi-directional-argocd.md) +- E2E corner design: [`e2e-bi-directional-corner.md`](../../spec/e2e-bi-directional-corner.md) - Exercised behavior: [`../../../test/e2e/argocd_bi_directional_e2e_test.go`](../../../test/e2e/argocd_bi_directional_e2e_test.go) - Argo CD source (vendored): `external-sources/argo-cd` — cited inline by file:line. diff --git a/docs/design/support-boundary/expansion-boundary-and-corpus-organisation.md b/docs/design/support-boundary/expansion-boundary-and-corpus-organisation.md index 75525b512..811f6d01d 100644 --- a/docs/design/support-boundary/expansion-boundary-and-corpus-organisation.md +++ b/docs/design/support-boundary/expansion-boundary-and-corpus-organisation.md @@ -630,10 +630,10 @@ single-use and can."* That is a report, not a standard, and it is achievable. ## Residual open questions 1. ~~**Argo CD's triggered-hydration handshake is still unproven in e2e.**~~ - **Closed.** The `selfHeal: false` + push-webhook loop, with a shared field changing - from *both* sides, is exercised against a real Argo CD in the bi-directional corner - ([`argocd_bi_directional_e2e_test.go`](../../../test/e2e/argocd_bi_directional_e2e_test.go)), - alongside its `selfHeal: true` foil where the edit is lost. + **Closed.** Automated sync off, with an `ArgoCDTrigger` starting every sync and a shared + field changing from *both* sides, is exercised against a real Argo CD in the bi-directional + corner ([`argocd_bi_directional_e2e_test.go`](../../../test/e2e/argocd_bi_directional_e2e_test.go)), + alongside an `Application` with automated sync on, which GitOps Reverser never syncs. 2. **What exactly makes a chart a chart?** Decision 4 says "recognisable folder structure." `Chart.yaml` + `templates/` is the obvious test, but `mixed-and-hostile` plants a `templates/` directory with **no** `Chart.yaml` and unparseable content diff --git a/docs/design/support-boundary/render-fidelity-scenarios.md b/docs/design/support-boundary/render-fidelity-scenarios.md index 8eba67f22..5647cdb3a 100644 --- a/docs/design/support-boundary/render-fidelity-scenarios.md +++ b/docs/design/support-boundary/render-fidelity-scenarios.md @@ -120,7 +120,7 @@ resync reply. ## 5. End-to-end proof The dedicated fixture at `test/e2e/fixtures/render-fidelity/` is implemented and runs in the -regular `manager` E2E shard (not the Argo-only bi-directional lane). It uses a real Flux +regular `manager` E2E shard, outside the opt-in bi-directional corner. It uses a real Flux `Kustomization`: the repository Deployment contains `${REGION}`, while `postBuild.substitute` resolves it to `us-east` in the live object. Do not reuse a render-root-scoping fixture: this fixture owns the render-vs-live pair directly. diff --git a/docs/design/support-boundary/render-fidelity.md b/docs/design/support-boundary/render-fidelity.md index 42d32cf6e..077157698 100644 --- a/docs/design/support-boundary/render-fidelity.md +++ b/docs/design/support-boundary/render-fidelity.md @@ -356,7 +356,7 @@ divergence, and an incoming Git repair does not start one yet. The missing transition is intentional and visible: the controller does **not** yet observe an incoming Git revision, refresh the source, and begin a complete epoch. Therefore a human Git repair cannot by itself reopen a false gate. Adding that transition requires the retained-intent ordering barrier in -[the reconcile trigger options](../../finished/push-notification-and-reconcile-trigger.md), so a source refresh cannot +[the reconcile trigger options](../push-notification-and-reconcile-trigger.md), so a source refresh cannot discard or race an open live-edit window. The enforcement check already belongs in the branch worker, not status projection: the worker rejects a `WriteRequest` before it opens a commit window. The controller projects the same state as `Ready=False` / `Stalled=True` for `False`, and as @@ -450,7 +450,7 @@ It is recomputable only when a complete fresh epoch begins. Today that happens w installed or their scope set is replaced. Removing postBuild configuration or changing the source in Git does **not** yet start that epoch automatically, so it does not automatically reopen writes. The planned remote-revision transition must refresh safely behind the barrier in -[the reconcile trigger options](../../finished/push-notification-and-reconcile-trigger.md); +[the reconcile trigger options](../push-notification-and-reconcile-trigger.md); once that exists, a full clean replay can reopen the gate without manual acknowledgement. --- diff --git a/docs/design/target-watch-plan.md b/docs/design/target-watch-plan.md index fc73c2b9f..86441d8e3 100644 --- a/docs/design/target-watch-plan.md +++ b/docs/design/target-watch-plan.md @@ -495,7 +495,7 @@ change 3. `stop` canceling a stream without touching files is not an interim state. It is what [`configuration.md`](../configuration.md) promises today: a deselected namespace's documents stay in Git. The sweep converges the mirror afterwards, and -the plan owes it nothing in the meantime. A reader of the repository sees the +the plan has nothing left to do for it in the meantime. A reader of the repository sees the behavior the documentation already describes, reached more cheaply. Bundling change 3 would put an unresolved product decision ("The one decision @@ -526,7 +526,7 @@ it. ### What this left behind -Two small debts, recorded so they are paid rather than discovered: +Two small follow-ups, recorded so they are fixed rather than discovered: - `streamRevisions` fills in zero revisions when no shared fidelity gate is wired. That path exists only because the gate can be absent in test wiring, and diff --git a/docs/design/vocabulary-cleanup.md b/docs/design/vocabulary-cleanup.md index a1984c2c0..bf7ae1b3e 100644 --- a/docs/design/vocabulary-cleanup.md +++ b/docs/design/vocabulary-cleanup.md @@ -252,6 +252,10 @@ source-cluster one, which [`../definitions.md`](../definitions.md) now does. Thi case on the page of a definition being the right fix for an ambiguity, and it is worth noticing that the audit reached for a rename first. +`allowAnySourceNamespace` was later replaced by `allowCrossNamespace.sources`, when a second +cross-namespace grant, for appliers, needed a place beside it +([`apply-trigger.md`](apply-trigger.md#decisions)). + ## Decided: break it in place, and warn Nothing is open any more. The posture, decided by the maintainer: **there is no known production diff --git a/docs/design/watch-plane-status-convergence-failures.md b/docs/design/watch-plane-status-convergence-failures.md index 895e8f465..fc728a92a 100644 --- a/docs/design/watch-plane-status-convergence-failures.md +++ b/docs/design/watch-plane-status-convergence-failures.md @@ -31,7 +31,7 @@ running. It never was one. | --- | --- | --- | --- | | **A** | A `WatchRule` never reaches `Ready=True` within 90s while its streams are demonstrably running | 6× (CI 4×, local 2×) | **SOLVED (§2).** A status write that loses an optimistic-lock race is discarded, the reconcile is told it succeeded, and nothing re-enqueues it — the winning write is status-only and every `For()` filters those | | **B** | `status.retention.retainedDocuments` stays at its pre-sweep value after `prune.mode` is widened, on a target whose files were swept | 3× (CI twice, local once) | **Open**, narrowed to the roll-up's accept path; predicted to be the same defect as A, with a falsifier — see §3.5 | -| C | Argo CD `selfHeal` commit count moves 3 → 5 during a `Consistently` | 1× (CI, never re-run) | Unclassified, and probably unrelated | +| C | Argo CD `selfHeal` commit count moves 3 → 5 during a `Consistently` | 1× (CI, never re-run) | **Moot:** the spec is gone, and the Argo CD corner runs with automated sync off | | D | Encryption-secret recreation spec times out | 4× in one day | **Ambient, pre-existing** — and now the dominant CI blocker; see §5.1 | **A green run is not evidence, and this page has been wrong twice for forgetting that.** Run @@ -265,7 +265,7 @@ Six components in this chain logged what they REJECTED and said nothing about wh | Component | Before | Now | | --- | --- | --- | -| the render-fidelity condition | a constant string naming nothing | names each pending scope and the revision it owes | +| the render-fidelity condition | a constant string naming nothing | names each pending scope and the revision it is waiting for (`waiting for revision N`) | | `recordScope` revision mismatch | silent | recorded on the scope, surfaced in the condition | | `recordScope` `!found` guards | silent | logged, naming collection and revision | | zero-revision report | silent early return | logged as anomalous | @@ -401,7 +401,7 @@ one behavioural fix (§4.9) is four lines. | # | Change | Commit | | --- | --- | --- | -| 1 | The render-fidelity condition names its pending scopes and the revision each owes, instead of a constant string | `c87db68d` | +| 1 | The render-fidelity condition names its pending scopes and the revision each is waiting for, instead of a constant string | `c87db68d` | | 2 | Deleted `explainNotRunning` / `notRunningHypothesis` / `cellNames` — ~70 lines aimed at the wrong subsystem — and lowered the supersession log | `17a57352` | | 3 | The resync drain starts even when the enqueue was dropped, so a queue-full reply is read by the drain the contract promises will read it | `fc04b15b` | | 4 | All four fidelity refusal branches name the collection, the revision and the gate's message | `32b75e02` | @@ -442,7 +442,7 @@ The chain in §2.2 is instrumented at every hop, so a reproduction should be rea | --- | --- | | `status write lost a race` on the target, then a stale condition | A's mechanism, recurring — check the requeue that followed it | | `render scope result accepted` … `state:True`, condition still stale | the gate is fine; the failure is downstream in publication | -| `a render scope result was not applied` | the gate refused a report — read `reportedRevision` against the owed revision | +| `a render scope result was not applied` | the gate refused a report — read `reportedRevision` against the revision the scope is waiting for | | `stream carries no revision` | the plan pass and the gate disagree about the collection | | `a GitTarget reconcile request was dropped` | a load-bearing notification was crowded out | | `retention report accepted but published nothing` | a re-measurement produced the count already published — the bug is upstream of the roll-up | diff --git a/docs/flux.md b/docs/flux.md deleted file mode 100644 index 5565510b8..000000000 --- a/docs/flux.md +++ /dev/null @@ -1,378 +0,0 @@ -# Using GitOps Reverser with Flux - -When GitOps Reverser publishes a folder that Flux applies, GitOps Reverser decides when Flux -applies it. Flux still fetches, renders, applies, prunes and reports health; it applies when -GitOps Reverser asks, instead of on its own timers or on a Git-host webhook. This page is the -complete Flux setup: what to change on the `GitRepository` and the `Kustomization`, what to declare -on the `GitTarget`, where the Git host's webhook goes, how to check it works, and which tests prove -each part. - -The [bi-directional guide](bi-directional.md) explains the model behind this for any applier, and -the [configuration reference](configuration.md#who-applies-this-folder-specapplier) has every field -and condition reason. - -## What changes - -| Before | With GitOps Reverser | -| --- | --- | -| The `GitRepository` fetches every minute or so | It fetches when GitOps Reverser asks; its `interval` is a 24h backstop | -| The `Kustomization` applies on its interval | It applies when the new artifact arrives, or when GitOps Reverser asks; its `interval` is a 24h backstop | -| The Git host's push webhook goes to a Flux `Receiver` | It goes to GitOps Reverser's receiver; the Flux `Receiver` for this repository is removed | -| Nothing ties a folder to its `Kustomization` | The `GitTarget` names it in `spec.applier.kustomizationRef` | - -## Why Flux must not fetch on its own - -**A new `GitRepository` artifact makes every `Kustomization` that reads it apply, at once.** Flux -has no mode that fetches without applying, and `suspend: true` on the `Kustomization` also blocks -the requests GitOps Reverser makes. So a source that fetches on its own, on a short interval or on a -webhook into a Flux `Receiver`, is a `Kustomization` that applies on its own. That brings back the -two races GitOps Reverser exists to close: - -- **Its own pushes.** Two API edits a few seconds apart produce two commits. A fetch between them - applies the first commit over the second edit, and GitOps Reverser then publishes the revert as a - third commit. -- **Somebody else's push.** While an API edit is still in an open commit window, a Git-side push - arrives. Applied straight away, the push's tree reverts the edit, because the edit is not in Git - yet. GitOps Reverser instead closes the window, replays the edit onto the push, pushes the - result, and only then asks Flux to apply. - -Both apply something correct; they apply it at the wrong moment. Only GitOps Reverser can see the -edits that are live and not yet pushed, so only it can tell when an apply is safe. It sends a -request only on a **quiet branch**: no open commit window and no pending writes. - -The source still stays current. After GitOps Reverser's own pushes, once the branch has been quiet -for `spec.applier.quietPeriod`, it asks the `GitRepository` to fetch if its artifact is behind -(reason `CatchUp`). That matters because Flux still applies on a controller restart and on its long -interval, from whatever artifact it holds, and an artifact left behind would revert edits that are -already in Git. - -## A complete example - -One folder, `apps/checkout` on branch `main` of `github.com/acme/platform`, published by GitOps -Reverser and applied by the `Kustomization` `flux-system/checkout`. Every object below is checked by -a unit test against the code that validates and drives the applier (see -[what the tests prove](#what-the-tests-prove)), so this example cannot drift from what is accepted. - -The Flux side. The `GitRepository` follows exactly the branch the `GitTarget` writes, and both -intervals are long: - -```yaml -apiVersion: source.toolkit.fluxcd.io/v1 -kind: GitRepository -metadata: - name: platform - namespace: flux-system -spec: - url: https://github.com/acme/platform - ref: - branch: main - interval: 24h - secretRef: - name: platform-git ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: checkout - namespace: flux-system -spec: - sourceRef: - kind: GitRepository - name: platform - path: ./apps/checkout - prune: true - interval: 24h - retryInterval: 1m -``` - -The GitOps Reverser side. The `GitProvider` points at the same repository (any URL form of it) and -serves a push receiver; the `GitTarget` names the `Kustomization` as its applier: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: platform-receiver - namespace: team-a -stringData: - token: replace-with-a-long-random-string ---- -apiVersion: configbutler.ai/v1alpha3 -kind: GitProvider -metadata: - name: platform - namespace: team-a -spec: - url: git@github.com:acme/platform.git - allowedBranches: ["main"] - secretRef: - name: platform-git - receiver: - type: github - secretRef: - name: platform-receiver ---- -apiVersion: configbutler.ai/v1alpha3 -kind: GitTarget -metadata: - name: checkout - namespace: team-a -spec: - gitProviderRef: - name: platform - branch: main - path: apps/checkout - applier: - kustomizationRef: - namespace: flux-system - name: checkout - quietPeriod: 30s - onRefusal: Reapply -``` - -No Flux `Receiver` for `acme/platform` appears, on purpose: remove yours if you have one. - -## The `GitRepository` - -`ApplierConfigured` is `True` only when the `Kustomization`'s source fetches the branch the -`GitTarget` writes. Each rule below is checked when the target reconciles and again before every -request, so a source changed after the check is still never stamped for the wrong branch. - -| Rule | Reported when broken | -| --- | --- | -| `spec.url` is the `GitProvider`'s repository. The forms compare as one repository: `https://` and `ssh://`, `git@host:org/repo` (SCP style), a trailing `.git` or `/`, letter case, and the default port | `SourceMismatch`: `fetches , not ` | -| `spec.ref.branch` is exactly the `GitTarget`'s `spec.branch` | `SourceMismatch`: `fetches branch "release", not main` | -| `spec.ref` pins nothing stronger: no `commit`, `name`, `semver` or `tag`, even alongside the branch. Flux resolves those before the branch, so the artifact would never follow it | `SourceMismatch`: `pins spec.ref.tag "v1.0.0", so it never follows branch main` | -| `spec.ref` is set | `SourceMismatch`: `names no spec.ref.branch` | - -Set `spec.interval` long, `24h` in the example. It is Flux's backstop for the case where GitOps -Reverser is down; a short one is a source that fetches, and so applies, on its own. - -## The `Kustomization` - -- `spec.sourceRef` names a `GitRepository`. A `Kustomization` that applies from an `OCIRepository` - or a `Bucket` reports `SourceMismatch`: GitOps Reverser writes Git, and can only ask a Git source - to fetch. -- `spec.interval` is long, for the same reason as the source's. -- `spec.suspend` is not `true`. A suspended `Kustomization` ignores the requests. -- `spec.retryInterval` is how soon Flux retries an apply that **failed**, on its own. Keep it, since - a failed apply otherwise waits out the 24h interval, and know that this one retry is not gated on - a quiet branch. -- It may live in any namespace (`flux-system` is the usual one). The reference's namespace defaults - to the `GitTarget`'s own. A request reaches only a `Kustomization` whose source fetches this - `GitProvider`'s repository and branch, so naming one elsewhere grants nothing that write access to - the repository does not already give. -- Several `GitTarget`s may name one `Kustomization`, for example one whose `path` covers several - folders. It is asked once for all of them, each with its own reason, and waits for the longest of - their `quietPeriod`s, because its apply reaches every one of their folders. -- Two `GitProvider`s may name one repository, for example one per team. Each gets its own branch - worker, and the branch counts as quiet only when both are: a `Kustomization` applies the tip - whichever of them wrote it. - -## The `GitTarget` - -```yaml -spec: - applier: - kustomizationRef: - namespace: flux-system # default: this GitTarget's namespace - name: checkout - quietPeriod: 30s # default 30s, at most 1h - clusterProviderRef: # default: this GitTarget's spec.clusterProviderRef - name: management -``` - -`clusterProviderRef` is for Flux running in another cluster than the one it applies to, such as a -management cluster with a `Kustomization` per workload cluster. Leave it out when Flux runs in the -source cluster. - -`quietPeriod` is how long the branch must stay quiet after GitOps Reverser's own pushes before Flux -is asked to catch up with them. Every new write restarts it. A short one keeps Flux's artifact -current; a long one makes it less likely that an edit arrives while that catch-up apply runs. - -The target reports `ApplierConfigured`, and `Ready` includes it: - -| Reason | Means | Readiness | -| --- | --- | --- | -| `Succeeded` | The `Kustomization` exists, applies this repository and branch, and may be patched | ready | -| `ApplierNotFound` | No such `Kustomization` in the applier's cluster, or Flux is not installed there | progressing: Flux objects are often created after the `GitTarget` | -| `SourceMismatch` | One of the [`GitRepository` rules](#the-gitrepository) is broken, or the source is not a `GitRepository` | stalled: needs a human | -| `Forbidden` | The applier cluster's identity may not patch the `Kustomization` or its `GitRepository` | stalled: needs a human | -| `NamespaceNotAuthorized`, `ClusterProviderNotFound` | `spec.applier.clusterProviderRef` names a provider that does not admit this namespace, or does not exist | stalled: needs a human | - -## Permissions - -GitOps Reverser looks the `Kustomization` up in the cluster of `spec.applier.clusterProviderRef`, -by default the **source cluster**, with the identity of that `ClusterProvider`. That identity needs: - -| API group | Resource | Verbs | -| --- | --- | --- | -| `kustomize.toolkit.fluxcd.io` | `kustomizations` | `get`, `patch` | -| `source.toolkit.fluxcd.io` | `gitrepositories` | `get`, `patch` | -| `authorization.k8s.io` | `selfsubjectaccessreviews` | `create`, to check the two above | - -For the operator's own cluster, install the chart with `applierAccess.flux: true`, or apply -[`config/rbac/flux-applier-role.yaml`](../config/rbac/flux-applier-role.yaml) and its binding. For -a remote cluster, grant the same to the identity in the `ClusterProvider`'s kubeconfig. - -The chart's grant is cluster-wide for convenience. Requests only ever read and patch the -`Kustomization` and `GitRepository` a `GitTarget` names, so a `Role` in each namespace that holds -them, bound to the same identity, is enough when you would rather not grant cluster-wide. The access -review needs no grant of its own there: Kubernetes lets every authenticated identity create -`selfsubjectaccessreviews` (the `system:basic-user` role). - -## The Git host's webhook - -Point the Git host's push webhook at GitOps Reverser, so a push reaches the cluster in seconds, -after GitOps Reverser has replayed any pending edits onto it. - -1. Serve the receiver: set the chart's `receiver.bindAddress` (for example `":9292"`). The chart - adds a `-receiver` `Service`. The listener is plain HTTP; expose it to your Git host - with an Ingress or Gateway route that terminates TLS, the way Flux's `webhook-receiver` is - exposed. -2. Create a Secret with the webhook secret under the key `token`, in the `GitProvider`'s namespace. - It is the key a Flux `Receiver` Secret uses, so an existing one can be copied over. -3. Set `GitProvider.spec.receiver` with that Secret and the host's `type`: `github`, `gitlab`, - `gitea`, or `generic-hmac` for a relay of your own. The values are Flux `Receiver` types. -4. Read the path from `status.webhookPath` and register `https://` as a push - webhook with the same secret: - - ```bash - kubectl get gitprovider platform -n team-a -o jsonpath='{.status.webhookPath}' - # /git-push/3f9a…c41e - ``` - - The path is derived from the token, as Flux's `Receiver` path is, so it names no namespace or - provider. A wrong path and a wrong secret both answer `404`; the operator's log says which. - Rotating the token moves the path, so register the new one with the new secret. - -5. Delete the Flux `Receiver` for this repository and its webhook at the Git host. - -A delivery is only a hint. GitOps Reverser reads where the branch is from the remote and -decides from that, so a delivery for one of its own pushes, a duplicate, or a late one asks Flux for -nothing it should not. Without a receiver everything still works, later: a foreign push is found by -the next rejected push or by the periodic refresh (`git.refreshInterval`, `10m`, on the 5-minute -reconcile tick). - -## What GitOps Reverser asks, and when - -| Reason | When | Asked | -| --- | --- | --- | -| `ForeignCommit` | The branch gained a commit GitOps Reverser did not push. Pending edits are replayed onto it and pushed first | Every `Kustomization` declared on the branch | -| `Refusal` | An edit was refused and the target has `onRefusal: Reapply` | The refusing target's `Kustomization` | -| `CatchUp` | The branch has been quiet for `quietPeriod` after GitOps Reverser's own pushes, or since the operator started | Every `Kustomization` declared on the branch | - -A request is what `flux reconcile kustomization --with-source` does, minus the waiting. GitOps -Reverser merge-patches `reconcile.fluxcd.io/requestedAt` on the `GitRepository` when its artifact -does not hold the commit; the new artifact then wakes the `Kustomization`. When the artifact -already holds it, only a `Refusal` stamps the `Kustomization` (to revert drift at an unchanged -commit); a `ForeignCommit` or `CatchUp` stamps nothing, because the artifact that holds the commit -already woke the `Kustomization` when it arrived. - -Before it stamps, GitOps Reverser checks that its decision still holds: the commit is still -the branch tip, and no edit is waiting to be published. When either changed since the request was -decided, it stamps nothing and decides again at the next quiet moment. Without that check, a -request decided at one commit could stamp the `Kustomization` after a newer push and re-apply the -older commit over a published edit. - -**An edit made while Flux applies keeps its value.** The patch returns at once, and Flux fetches and -applies a few seconds later. Until Flux reports it finished (`status.lastHandledReconcileAt` for a -stamped `Kustomization`, a new `status.lastAttemptedRevision` after a source stamp, at most -5 minutes), GitOps Reverser treats the apply as running. An API edit made in those seconds, to a -field the `Kustomization` sets, is reverted on the cluster by that apply, because the commit Flux -was asked for does not hold it yet. GitOps Reverser recognizes that write by its field manager, -`kustomize-controller`, and does not mirror it: the API's value stays in Git, and the `CatchUp` its -push owes applies it back once the branch has been quiet for `quietPeriod`. Each such write is -logged and counted in `apply_window_collisions_total{outcome="withheld"}`. - -Only a write proven to be Flux's is withheld. When `kustomize-controller` and another manager wrote -the object in the same second, the write could be the edit itself, so it is mirrored as before and -counted as `ambiguous`. A delete is never withheld. - -The request runs off the branch worker, so a slow source cluster never holds a publication, and a -failed request never fails one. - -Before it patches, GitOps Reverser reads each `GitTarget` again: it must still be the same -object (a deleted and recreated one inherits nothing), still name this `Kustomization` on this -branch, still be admitted by its `ClusterProvider`, and, for a `Refusal`, still consent. When none -still holds the request's reason it asks nothing; it does not substitute a weaker request. - -**A request that fails is decided again from scratch.** GitOps Reverser keeps what each -`Kustomization` is owed until a request delivers it. After a failure, that `Kustomization` waits out -its own backoff (2 seconds doubling to 5 minutes, at least 15 seconds after a `SourceMismatch`), -then waits for a quiet branch like any first request, and asks for the tip as it is then. So a -`SourceMismatch` is delivered once it is fixed, and a refusal that recovered meanwhile is not asked -for again. - -## Reverting refused edits - -With `onRefusal: Reapply`, an edit GitOps Reverser cannot write (one that has no home in the folder, -such as a field a kustomize base owns) is reverted by asking the target's `Kustomization` to apply -the tip, once the branch is quiet. Only that `Kustomization` is asked. If its `GitRepository` has to -fetch to reach the tip, though, every `Kustomization` on that `GitRepository` applies the new -revision, as it would for any push. Give the folder its own `GitRepository` if the revert must reach -that folder alone. - -[Reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal) has the -eligibility rules. - -## Checking that it works - -```bash -# The applier is found, applies this repository and branch, and may be patched. -kubectl get gittarget checkout -n team-a \ - -o jsonpath='{.status.conditions[?(@.type=="ApplierConfigured")]}{"\n"}' - -# What GitOps Reverser last asked Flux for, and why. -kubectl get gittarget checkout -n team-a -o jsonpath='{.status.applier}{"\n"}' -# {"reason":"ForeignCommit","requestedAt":"2026-10-20T10:14:03Z","requestedCommit":"4f2c1ab9..."} - -# Each request, and each request that could not be delivered. -kubectl get events -n team-a --field-selector involvedObject.name=checkout,involvedObject.kind=GitTarget - -# What Flux actually applied: a request asks, it never reports. -kubectl get kustomization checkout -n flux-system -o jsonpath='{.status.lastAppliedRevision}{"\n"}' -``` - -`gitopsreverser_apply_requests_total{reason,outcome}` counts the requests; -[interpreting metrics](interpreting-metrics.md#apply-requests-and-the-push-receiver) reads it. - -## Troubleshooting - -| Symptom | Cause | Fix | -| --- | --- | --- | -| `ApplierConfigured=False`, `ApplierNotFound` | The `Kustomization` does not exist in the source cluster under that namespace and name, or Flux is not installed there | Create it, or fix `kustomizationRef`; an omitted namespace is the `GitTarget`'s | -| `ApplierConfigured=False`, `SourceMismatch` | The message names the broken [`GitRepository` rule](#the-gitrepository) | Fix the `GitRepository`; the request it was holding is delivered afterward | -| `ApplierConfigured=False`, `Forbidden` | The identity lacks one of the [permissions](#permissions) | Grant it, or set `applierAccess.flux: true` | -| A Warning `ApplyRequestFailed` Event | A request could not be delivered; the message says why | GitOps Reverser keeps asking on a quiet branch, at most every five minutes. It is recorded for the first failure of a run | -| Flux applies while edits are in flight | Something else still makes the source fetch: a short `interval`, a Flux `Receiver`, or someone running `flux reconcile` | Lengthen the intervals and remove the Flux `Receiver` | -| A Git-side push takes minutes to reach the cluster | No receiver, so the refresh finds it | Configure [the webhook](#the-git-hosts-webhook) | -| A refused edit is not reverted | The target is suspended, `onRefusal` is not `Reapply`, or the refusal is not eligible | See [reverting a refused edit](configuration.md#reverting-a-refused-edit-speconrefusal) | - -## While GitOps Reverser is down - -GitOps Reverser becomes the only fast path to a deployment. While it is down, a Git-side push -reaches the cluster through `flux reconcile kustomization --with-source`, which is safe then -because nothing is publishing, or through the 24h backstop. When it starts again, its first look at -the remote owes every `Kustomization` a `CatchUp` and no `ForeignCommit`: an apply on startup -could revert edits made while it was down, before its watches have caught them. - -## What the tests prove - -| Covered | Where | -| --- | --- | -| The [complete example](#a-complete-example) decodes strictly, reports `ApplierConfigured=True`, and its request stamps the `GitRepository`; the Flux objects follow this page's advice | [`flux_guide_test.go`](../internal/applier/flux_guide_test.go) | -| `ApplierNotFound`, then `SourceMismatch` for a source on another branch, reported from a running source cluster | [Manager Flux Applier](../test/e2e/flux_applier_e2e_test.go) | -| A `ForeignCommit` for a `SourceMismatch` applier never stamps the other branch's source, records `ApplyRequestFailed`, and is delivered once the source is fixed | Manager Flux Applier | -| A Gitea push webhook to the operator's receiver gets a Git-side push applied, with nothing else on the branch | Manager Flux Applier | -| With no receiver and no API edit, the refresh finds a Git-side push and Flux applies it | Manager Flux Applier | -| Two API edits seconds apart end at the later value with no extra commit, and Flux catches up by itself | [Bi-directional (Flux)](../test/e2e/flux_bi_directional_e2e_test.go) | -| A Git-side push with an API edit on top lands both, and Flux applies the replayed commit | Bi-directional (Flux) | -| A new `GitRepository` revision alone, from a commit outside the applied folder, makes the `Kustomization` apply, which is why the source must not fetch on its own | Bi-directional (Flux) | -| A refused edit with `onRefusal: Reapply` is reverted by its `Kustomization`, with no commit, and settles | [Refusal reapply](../test/e2e/refusal_reapply_e2e_test.go) | -| Every attempt checks the target again (incarnation, branch, admission, consent) and the source binding before it stamps, and never asks for something weaker | [`dispatcher_test.go`](../internal/applier/dispatcher_test.go) | -| A failed request is decided again on a quiet branch; a recovered refusal, withdrawn consent or recreated target is not retried; each `Kustomization` keeps its own backoff; a shared one is asked once; a sibling worker on the same repository holds the branch | [`apply_attempt_test.go`](../internal/git/apply_attempt_test.go) | - -`task test-e2e` runs the Manager specs, and `task test-e2e-bi-directional` runs the bi-directional -corner. Not covered end to end yet: a remote source cluster, `Forbidden` against cluster RBAC, and the -receiver installed by the Helm chart rather than the e2e manifests. diff --git a/docs/future/config-surface-for-a-structured-repository.md b/docs/future/config-surface-for-a-structured-repository.md index 139e67219..f04c15e8c 100644 --- a/docs/future/config-surface-for-a-structured-repository.md +++ b/docs/future/config-surface-for-a-structured-repository.md @@ -219,7 +219,7 @@ Do it in the same breaking wave as F6/F12, not separately. **the operator never discovers.** A GitTarget is told exactly one subtree. Argo and Flux scan; we deliberately do not. -That architectural call is right — and it makes onboarding a **step the product owes the user**, +That architectural call is right — and it makes onboarding a **step the product must provide**, performed by `manifest-analyzer --mode scan-repo`, which answers precisely the question the original doubt raises: *which folders can become GitTargets, what layout is each, which are supported and why not, and what GitTarget/WatchRule would express each one.* diff --git a/docs/future/git-api-three-way-comparison.md b/docs/future/git-api-three-way-comparison.md index 2bb23618c..02baa9ad7 100644 --- a/docs/future/git-api-three-way-comparison.md +++ b/docs/future/git-api-three-way-comparison.md @@ -478,9 +478,9 @@ needed for those forms, but neither every patch nor every update reveals an exac ## Why faster Git delivery helps but does not settle the merge -Faster notification reduces how long the engine plans against stale Git state. The -[apply trigger design](../design/apply-trigger.md) can improve -that latency independently of this investigation. +Faster notification reduces how long the engine plans against stale Git state. The push receiver +of the [apply trigger](../design/apply-trigger.md) shortens that latency, independently of this +investigation. The timestamp example already gives Reverser the Git change before Alice edits. The remaining gap is between the Git state and the state in the cluster. Faster application by Flux or Argo CD, and diff --git a/docs/future/ha-gittarget-distribution-plan.md b/docs/future/ha-gittarget-distribution-plan.md index 39d6a8978..ec7207c69 100644 --- a/docs/future/ha-gittarget-distribution-plan.md +++ b/docs/future/ha-gittarget-distribution-plan.md @@ -30,8 +30,9 @@ The first HA release uses active/passive ownership: The baseline durability contract is eventual state convergence under the configured write and prune policies. A missed DELETE cannot be inferred from a snapshot under `prune.mode: onEvent`; -an `always` target permits scoped absence-based pruning. The proposed extension preserves accepted -save obligations, including window membership, attribution, messages, deadlines, and outcomes. +an `always` target permits scoped absence-based pruning. The proposed extension preserves what an +accepted save still waits for, including window membership, attribution, messages, deadlines, and +outcomes. The event model defines those execution semantics; this plan owns how their records survive failure. Before implementing the extension, agree on retention, target replacement, and the response to @@ -194,11 +195,11 @@ Separate three durable milestones: | Milestone | What it proves | |---|---| | Accepted | Input is recoverable; its source cursor may advance | -| Processed | Decisions and remaining obligations are recoverable | +| Processed | Decisions and the work still pending are recoverable | | Completed | Publication or an explicit terminal outcome is proved | -Persist the consumed input position, resulting decisions, and new effect obligations together. -Otherwise a crash can consume a command but lose its push, or create an obligation twice. +Persist the consumed input position, resulting decisions, and the effects still to run together. +Otherwise a crash can consume a command but lose its push, or schedule the same effect twice. Stable effect identities permit redelivery. A durable outbox holds external operations and status updates still requiring execution; writing it does not prove those operations succeeded. @@ -287,13 +288,13 @@ acknowledge the journal or prove a request's historical execution. ## Surviving hours without Git Use the same pause/recover/resume behavior as the in-memory worker, with durable admission as the -acceptance boundary. Storage changes how long accepted obligations survive and how much backlog -fits; it does not change which outcomes the worker owes. +acceptance boundary. Storage changes how long accepted work survives and how much backlog +fits; it does not change which outcomes the worker must reach. | Boundary | During an outage | At capacity | After recovery | |---|---|---|---| | Current in-memory path | Retain accepted writes while the process lives | Refuse new admission; explicit producer pause is planned | Replay retained decisions, then resume watches | -| Future durable journal | Persist accepted inputs and workflow decisions; page the active working set | Stop durable admission before eviction or storage failure | Recover accepted obligations, then catch up sources | +| Future durable journal | Persist accepted inputs and workflow decisions; page the active working set | Stop durable admission before eviction or storage failure | Recover accepted pending work, then catch up sources | If Git is unavailable but the durable store has capacity, producers can keep appending without loading that backlog into the worker's RAM. When the store fills or cannot guarantee a durable @@ -313,7 +314,7 @@ of every later Kubernetes mutation: a prolonged ingestion stop can outlast watch Retained accepted work drains in branch order before a fresh source snapshot can supersede current state. An expired watch cursor records a continuity gap; the complete scoped snapshot repairs only what the current write/prune policy permits. Missed intermediate versions and expired delete -events are not recreated. An accepted save keeps its original obligation through this catch-up; +events are not recreated. An accepted save keeps its original pending write through this catch-up; an unaccepted save has no durable receipt. State these limits in status and operator documentation. Give recovery and lifecycle work a way to run under saturation, with its causal ordering intact. @@ -322,7 +323,7 @@ in-memory working set independently of journal size; keeping every persisted pay `pendingWrites` would preserve the current memory problem. Snapshot compaction must respect save boundaries. Replacing a request-bearing batch with the -latest object state loses its message, attribution, and publication obligation. Define explicitly +latest object state loses its message, attribution, and the publication it still waits for. Define explicitly which ordinary writes can coalesce, and preserve references needed by retained windows. The store's persistence, replication, retention, and eviction settings are part of the guarantee. @@ -361,7 +362,7 @@ Add a journal package beside the existing Redis store: - publish idempotent records with explicit admission receipts; - atomically couple watch cursor advancement to durable admission; -- persist worker decisions and effect obligations with their processing position; +- persist worker decisions and the effects still to run with their processing position; - route live observations, snapshots, saves, withdrawals, deadlines, and results through the recoverable boundary; - encode and encrypt sensitive payloads, enforce quotas, and page retained payloads to bound memory; @@ -495,7 +496,7 @@ scope has completed its authoritative replay. - No completed delivery is acknowledged without recoverable publication evidence or an explicit terminal outcome; pending status projection and required deduplication receipts remain durable. - A stale or partitioned owner cannot overwrite a newer Git ref. -- Replays and queue claims preserve accepted save obligations. Post-push-before-ack crashes use +- Replays and queue claims preserve accepted saves until they are answered. Post-push-before-ack crashes use the publication protocol; missing evidence produces explicit uncertainty rather than an unsupported success or duplicate-publication guarantee. - Sensitive resource content is never stored as plaintext in the durable queue. diff --git a/docs/interpreting-metrics.md b/docs/interpreting-metrics.md index e06a55003..34948b73f 100644 --- a/docs/interpreting-metrics.md +++ b/docs/interpreting-metrics.md @@ -81,8 +81,8 @@ without a kubeconfig. | --- | --- | --- | | `resource_condition` | gauge | `kind`, `resource_namespace`, `resource_name`, `type`, `status`, `reason` | -- `kind` is `GitTarget`, `WatchRule`, `ClusterWatchRule`, `GitProvider` or `ClusterProvider` — the - objects a human declares. +- `kind` is `GitTarget`, `WatchRule`, `ClusterWatchRule`, `GitProvider`, `ClusterProvider`, + `FluxTrigger` or `ArgoCDTrigger`: the objects a human declares. - `type` is the kstatus trio: `Ready` (the summary), `Reconciling` (working on it) and `Stalled` (given up, and nothing will retry without a human). - `status` is `True`, `False` or `Unknown`. @@ -204,6 +204,7 @@ boundary, the commit, the push. Background: | `git_fetches_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Every call that reads the remote through a `SmartFetch`. `reason` is `bootstrap` (repository preparation; **no production caller reaches it today**, so the series stays at zero) / `publication` (the head of a publication cycle) / `recovery` (re-establishing a base that could not be trusted, including a worktree a failed write left dirty that could not be undone locally) / `contention` (the reset onto the new tip after a push was rejected because somebody else moved the branch) / `push_failure_probe` (a push that failed without the remote saying anything, looking up where the branch is) / `forced_recheck` (a full re-read outside the publication cycle: a forced recheck, or the snapshot a resync judges against) / `refresh` (the periodic top-up of an idle branch, or a push receiver delivery on one, and only when the branch had actually moved). See the note below. | | `git_queue_refusals_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind` | Work refused at enqueue: the queue was full, intake was paused through an outage, or the worker was stopping. `kind` is `write` / `attach` / `resync` / `refresh`. The producer keeps a refused item and offers it again: the watch delivers a write again from its cursor, the controller re-sends a save, a resync is gathered again, and the next reconcile asks for a refresh. What it can cost, when its watch cursor expires before the write comes back, is the write's own commit and, under `prune.mode: OnEvent`, a deleted object's removal from Git. | | `git_commit_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A window or request that died between routing and pushing. `kind` is `window` / `atomic`; `reason` is `refused` (a Git path a human must fix) / `error`. Every increment is a window's events lost until the next resync. An unreachable remote does not count: a decided window waits for the publication retry instead. | +| `git_writes_dropped_total` | counter | `provider_namespace`, `provider_name`, `branch`, `kind`, `reason` | A live event (`kind="event"`) or atomic write (`kind="atomic"`) the branch worker dropped because its `GitTarget`'s render fidelity was not established. `reason` is `render_fidelity_pending`: a collection of the `GitTarget` has not finished its replay, which closes every collection of that target until it does. Or it is `render_fidelity_diverged`: a rendered value does not match the live object. Nothing replays a dropped write, so each one is an edit missing from Git until the object changes again or its collection replays. The worker logs the first drop under each gate status at Info, naming the collections the gate waits for. | | `git_commit_windows_total` | counter | `gittarget_namespace`, `gittarget_name`, `close_reason`, `timer_source` | One per closed commit window, whether or not it then produced a commit. `close_reason` is `idle_timeout` / `max_duration` / `attach_next` / `identity_change` / `buffer_limit` / `resync_before_apply` / `atomic_before_apply` / `foreign_commit` (the branch moved, or a Git host said it may have, so the window is published now) / `shutdown`; `timer_source` is `target` or `commit_request`. See [tuning commit windows](#tuning-commit-windows). | | `git_commit_window_duration_seconds` | histogram | `gittarget_namespace`, `gittarget_name` | How long each window collected: from the write that opened it to the start of its finalize. Its count equals `git_commit_windows_total` summed over `close_reason` and `timer_source`. | | `git_queue_depth` | gauge | `provider_namespace`, `provider_name`, `branch` | Pending + in-flight + committed-but-unpushed. Read at scrape time. | @@ -211,7 +212,7 @@ boundary, the commit, the push. Background: | `git_retained_writes` | gauge | `provider_namespace`, `provider_name`, `branch` | Decided writes waiting for a push. | | `git_intake_paused` | gauge | `provider_namespace`, `provider_name`, `branch` | 1 while the worker refuses new writes, saves and resyncs because an outage filled its budget; 0 otherwise. | | `git_oldest_retained_write_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the oldest write still waiting for a push was decided, as a Unix timestamp. No sample while nothing waits. | -| `git_next_retry_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the next attempt at owed work is due, as a Unix timestamp. No sample while nothing is owed. | +| `git_next_retry_timestamp_seconds` | gauge | `provider_namespace`, `provider_name`, `branch` | When the next attempt at the pending writes is due, as a Unix timestamp. No sample while no retry is scheduled. | | `git_materialization_failures_total` | counter | `provider_namespace`, `provider_name`, `branch`, `reason` | Attempts to commit decided writes that stopped before any push: `reason` is `unreachable` or `parent_unavailable`. Nothing is lost; the writes wait for the retry. A failure `git_pushes_total` cannot see, because no push cycle started. | | `git_branch_targets` | gauge | `provider_namespace`, `provider_name`, `branch`, `gittarget_namespace`, `gittarget_name`, `source_cluster` | Always 1. The **join** between the GitTarget-labeled half of the pipeline and the branch-labeled half. Published per configured GitTarget, whether or not its worker runs. | | `commit_requests_total` | counter | `outcome`, `gittarget_namespace`, `gittarget_name` | One per `CommitRequest` terminal **decision**. `outcome` is `committed` / `no_window` / `window_mismatch` / `already_present` / `failed`. A target that never resolved publishes the empty pair. See the counting note below. | @@ -347,7 +348,7 @@ red trains people to ignore it. Four classes: | **expected** | `routed`, `unchanged`, `bookmark`, `shutdown`, `written`, `deleted_live`, `deleted_sweep`, `retained`, `cached` | the pipeline working. `unchanged` is a document considered and found identical; `refused` is a document the writer declined to place, which is loss | | **degraded** | `author_kind="unresolved"`, `mode="list_fallback"`, `not_object`, `stream_error`, weak attribution tiers | working, on weaker evidence | | **recoverable** | `git_pushes_total{outcome="failed"}` (the writes are retained and retried); `route_failed` and any `git_queue_refusals_total` (the producer keeps the refused item and offers it again) | alert on it **sustained**: pushes with no successes, refusals on a branch whose intake is not paused. Never on one occurrence | -| **loss** | any `git_commit_failures_total`, `placement_refusals_total`, `git_documents_total{outcome="refused"}` | an observed change that did not reach Git | +| **loss** | any `git_commit_failures_total`, `git_writes_dropped_total`, `placement_refusals_total`, `git_documents_total{outcome="refused"}` | an observed change that did not reach Git | `route_failed` and a queue refusal **overlap**: a refused enqueue is one of the ways a route fails, so one refused event increments both. They are two views of one event, and summing them @@ -446,14 +447,15 @@ sum by (provider_namespace, provider_name, branch) ( rate(gitopsreverser_git_pushes_total[5m])) ``` -**Is work being refused, or thrown away?** Both should be flat zero on a healthy branch. A queue -refusal is a saturated worker or a paused intake, and the producer offers the work again; a commit -failure takes a whole window with it, and `reason="refused"` will not clear until someone fixes the -Git path: +**Is work being refused, or thrown away?** All three should be flat zero on a healthy branch. A +queue refusal is a saturated worker or a paused intake, and the producer offers the work again; a +commit failure takes a whole window with it, and `reason="refused"` will not clear until someone +fixes the Git path; a dropped write is an edit that met a closed render-fidelity gate: ```promql sum by (kind) (rate(gitopsreverser_git_queue_refusals_total[5m])) sum by (kind, reason) (rate(gitopsreverser_git_commit_failures_total[5m])) +sum by (kind, reason) (rate(gitopsreverser_git_writes_dropped_total[5m])) ``` **How much work is a type's ingestion doing?** A stream is single-threaded, so time it spends @@ -816,15 +818,17 @@ are unbounded, and both are in the log line at the write site. ## Apply requests and the push receiver -With `GitTarget.spec.applier` set, the branch worker decides when the Flux `Kustomization` or Argo -CD `Application` applies, and the push receiver tells it that somebody else pushed. See -[who applies this folder](configuration.md#who-applies-this-folder-specapplier). +With `GitTarget.spec.applyTriggerRef` set, the branch worker decides when the Flux `Kustomization` +or Argo CD `Application` the trigger holds applies, and the push receiver tells it that somebody +else pushed. See [who applies this folder](configuration.md#who-applies-this-folder-specapplytriggerref). | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | -| `apply_requests_total` | counter | `applier_kind`, `reason`, `outcome` | One per attempt to one applier. `applier_kind` is `kustomization` (a Flux `Kustomization`) or `application` (an Argo CD `Application`); `reason` is `ForeignCommit` / `Refusal` / `CatchUp` (never `CatchUp` for an `application`); `outcome` is `sent` (delivered, or a `CatchUp` that found Flux already current), `failed` (could not ask, including an applier that applies another repository or branch, an `Application` with automated sync on, or one whose running sync did not end within 5 minutes; the branch worker keeps the obligation and decides the retry on a quiet branch, so a failure that persists keeps counting at its backoff) or `dropped` (no `GitTarget` may still ask for that reason: each was deleted or recreated, moved to another branch or applier, is no longer admitted by its `ClusterProvider`, or withdrew its consent to a `Refusal`) or `stale` (asked nothing because, by the time of the write, the commit was no longer the branch tip or an edit was waiting to be published; the worker decides again at the next quiet moment) | -| `apply_window_collisions_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the request was sent: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value and the next apply restores it on the cluster) or `ambiguous` (the applier and another field manager wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit) | -| `receiver_deliveries_total` | counter | `type`, `outcome` | One per verified-or-refused delivery to `/git-push/`. `type` is the `GitProvider.spec.receiver.type`; `outcome` is `notified` (handed to the branch worker), `unmatched` (no worker for that branch), `ignored` (a ping, a tag, another branch's event type, or a `generic-hmac` body naming another repository), `rejected` (bad signature or an oversized body) or `invalid` (verified but unparseable) | +| `apply_requests_total` | counter | `applier_kind`, `reason`, `outcome` | One per attempt to one applier. `applier_kind` is `kustomization` (a Flux `Kustomization`) or `application` (an Argo CD `Application`); `reason` is `ForeignCommit` / `Refusal` / `Repair` / `CatchUp` (never `CatchUp` for an `application`); `outcome` is `sent` (the applier was asked, or it needed no request: a `ForeignCommit` or `CatchUp` found the `Kustomization` suspended with the commit applied already, or the remote no longer carries the branch, so there is no commit to apply; it says nothing about how the apply went), `failed` (could not ask, including an applier that applies another repository or branch, an `Application` with automated sync on, or one whose running sync did not end within 5 minutes; the branch worker keeps the reason pending and decides the retry on a quiet branch, so a failure that persists keeps counting at its backoff) or `dropped` (no `GitTarget` may still ask for that reason: each was deleted or recreated, moved to another branch or applier, is no longer admitted by its `ClusterProvider`, or withdrew its consent to a `Refusal`) or `stale` (asked nothing because, by the time of the write, the commit was no longer the branch tip or an edit was waiting to be published; the worker decides again at the next quiet moment) | +| `apply_revision_mismatches_total` | counter | `applier_kind` | One per apply operation that succeeded on another revision than the commit GitOps Reverser checked. A `Kustomization` applies whatever its `GitRepository` artifact holds when it reconciles; the artifact is read again just before the resume, so this is somebody fetching the source in between (a person, or the source's own interval). The apply window still withholds a late write over a newer edit; a steady rate means something else keeps fetching the source | +| `apply_window_collisions_total` | counter | `provider_namespace`, `provider_name`, `branch`, `outcome` | One per write by the applier, while an apply it was asked for runs, to an object the API side edited after the request was sent: the apply arriving late over a newer edit. `outcome` is `withheld` (proven to be the applier's write, by its field manager alone, with an entry newer than the edit: `kustomize-controller`, or `argocd-controller` / `argocd-application-controller`; so not mirrored: Git keeps the API's value, and a `Repair` of the applier follows, which restores it on the cluster) or `ambiguous` (the applier and another field manager, or the applier and the edit, wrote in the same second, so it was mirrored as ever; if it was the apply, the revert is in Git as the applier's commit). An applier entry older than the edit is not counted: it is the API's undo of an edit, mirrored | +| `apply_operations_unresolved` | gauge | `provider_namespace`, `provider_name`, `branch` | Apply operations on the branch that have not completed 5 minutes after they were requested. Each is still followed, holds its applier's apply window open, and holds back the applier's next request until it completes, so a value that stays above zero is an applier stuck mid-apply: read its status, and the `ApplyUnresolved` Event on the `GitTarget`s. Read at scrape time | +| `receiver_deliveries_total` | counter | `type`, `outcome` | One per verified-or-refused delivery to `/git-push/`. `type` is the `GitProvider.spec.receiver.type`; `outcome` is `notified` (handed to the branch worker), `unmatched` (no worker for that branch), `ignored` (a ping, an event other than a push, a tag or other non-branch ref, or a `generic-hmac` body naming another repository), `rejected` (a missing or wrong signature or token, or an oversized body) or `invalid` (verified but unparseable) | A delivery counted `notified` is only a hint: the worker reads the advertisement, and fetches under `git_fetches_total{reason="refresh"}` only when the branch actually moved. A `ForeignCommit` request @@ -839,9 +843,11 @@ sum by (reason) (rate(gitopsreverser_apply_requests_total{outcome="failed"}[15m] # A receiver a Git host is calling with the wrong secret. sum by (type) (rate(gitopsreverser_receiver_deliveries_total{outcome="rejected"}[15m])) > 0 # Edits landing while an apply runs, by branch. Withheld ones are reverted on the cluster until the -# next apply; ambiguous ones may have put the applier's revert in Git. +# Repair that follows; ambiguous ones may have put the applier's revert in Git. sum by (provider_namespace, provider_name, branch, outcome) ( increase(gitopsreverser_apply_window_collisions_total[1h])) > 0 +# An applier stuck mid-apply: nothing more is asked of it until its operation completes. +max by (provider_namespace, provider_name, branch) (gitopsreverser_apply_operations_unresolved) > 0 ``` ## Audit attribution (optional) diff --git a/docs/layout/new-file-placement-rules.md b/docs/layout/new-file-placement-rules.md index 3818e1f47..080474e9a 100644 --- a/docs/layout/new-file-placement-rules.md +++ b/docs/layout/new-file-placement-rules.md @@ -654,7 +654,7 @@ A document in a directory whose kustomization sets a `namespace:` transformer do [contextual-namespace.md](contextual-namespace.md)). A new document placed there must follow that convention, or it breaks the folder's own style, and this applies to **every** resolved path: a declared template pointing into a -governed directory is under the same obligation as the kustomize-root fallback. +governed directory follows the same rule as the kustomize-root fallback. It is conditional on the two namespaces **matching**. Omitting `metadata.namespace` hands the namespace to kustomize, so a transformer naming a *different* namespace would render the @@ -822,7 +822,7 @@ CRD to know where a resource will go; with C the answer lived in the repo plus a ladder, so the scan/dry-run output **must** state, per new resource, the chosen path and the cohort and ladder step that produced it. *It was never built, which is a fact worth keeping on this page: the feature shipped without the thing that made it defensible.* What exists -now is the smaller obligation the smaller ladder deserves — `placements_total{source}` per +now is the smaller signal the smaller ladder deserves — `placements_total{source}` per (GitTarget, type), plus the per-resource log line naming the resolved path. **P9 — Cohort lookup cost at scale.** Each new resource scanned the store, so a large repo @@ -1453,7 +1453,7 @@ catch-all layout. - never let a sensitive resource join a plaintext file, or a plaintext resource an encrypted one; refuse instead, with a bounded reason; - count every resolution by `{source, disposition}` and every refusal by `{reason}`, - labeled with the GitTarget and the type, which is the smaller obligation that + labeled with the GitTarget and the type, which is the smaller signal that replaces P8's never-built cohort trace. 5. Parse and validate path templates once per GitTarget reconcile. Sensitive resources must resolve to identity-complete paths and must be written through diff --git a/docs/rbac.md b/docs/rbac.md index 843dbca81..238c3b915 100644 --- a/docs/rbac.md +++ b/docs/rbac.md @@ -13,6 +13,15 @@ credential we own". | `-manager-role` | Exactly what the binary calls: its own CRs, `namespaces`, `customresourcedefinitions`, `apiservices`, and `secrets` with **`get`, `create`, `update`** | Generated from the kubebuilder markers into [`config/rbac/role.yaml`](../config/rbac/role.yaml). Never edit by hand. | | `-watch-any` **or** `-watch-selected` | The types a `WatchRule` may read, `get`/`list`/`watch` only | The chart's `rbac.watchTypes`; for kustomize, [`config/rbac/watch-any-role.yaml`](../config/rbac/watch-any-role.yaml). | +Two more ClusterRoles exist only when asked for, for a folder that is both written and applied +([`bi-directional.md`](bi-directional.md)): `-applier-flux` (chart value +`applierAccess.flux`; for kustomize, +[`config/rbac/flux-applier-role.yaml`](../config/rbac/flux-applier-role.yaml)) lets a +`FluxTrigger` suspend and resume its `Kustomization`, and `-applier-argocd` +(`applierAccess.argocd`; +[`config/rbac/argocd-applier-role.yaml`](../config/rbac/argocd-applier-role.yaml)) lets an +`ArgoCDTrigger` start its `Application`'s syncs. + They are separate because **RBAC is additive**. A wildcard read folded into the manager role would grant cluster-wide Secret `list`/`watch` no matter how narrow the Secret rule beside it is, and no chart value could take it back. Split out, the wildcard can be replaced. @@ -45,10 +54,10 @@ rbac: resources: ["deployments"] ``` -Verbs are always `get`, `list`, `watch` — the reverser never writes to a watched type, so a -`verbs:` key is rejected rather than letting you grant one by accident. So is `mode: -selected` with an empty list, which would leave the operator able to watch nothing, and an -unknown key under `rbac`, which is how you think you narrowed access when you did not. +Verbs are always `get`, `list`, `watch` (the applier grants above are separate): the reverser +never writes to a watched type, so a `verbs:` key is rejected rather than letting you grant one +by accident. So is `mode: selected` with an empty list, which would leave the operator able to +watch nothing, and an unknown key under `rbac`, which is how you think you narrowed access when you did not. Those rules live in the chart's [`values.schema.json`](../charts/gitops-reverser/values.schema.json), which Helm enforces on diff --git a/docs/security-model.md b/docs/security-model.md index f3a07dfd9..7d70bd4a3 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -38,9 +38,15 @@ To mirror live state into Git the controller must: `list` or a `watch`. - **Receive audit events.** The kube-apiserver audit webhook posts events to the audit ingress. Those carry object metadata and, for some resources, request/response bodies. +- **Drive an applier, only with an apply trigger.** Holding a Flux `Kustomization` takes `get` and + `patch` on it and its `GitRepository`, plus `list` on `Kustomization`s, `HelmRelease`s and + `HelmChart`s; holding an Argo CD `Application` takes `get` and `update` on it. For the operator's + own cluster the chart grants these in ClusterRoles of their own, only with `applierAccess.flux` + or `applierAccess.argocd`. -The controller never writes to a watched type. Its only write target is Git, plus the Secrets it -generates itself (the signing key, and the age key under `generateWhenMissing`). +Apart from the applier an apply trigger holds, the controller never writes to a watched type. Its +write targets are Git, that applier, and the Secrets it generates itself (the signing key, and the +age key under `generateWhenMissing`). ### Which namespaces a rule may read from @@ -48,10 +54,10 @@ RBAC above is the hard maximum; *within* it, scope is carried by the rule **kind - A **`WatchRule`** selects namespaced resources. Each `spec.rules[]` item watches the rule's own namespace by default. Naming any other source namespace — including `sourceNamespace: "*"` — - requires `ClusterProvider.spec.allowAnySourceNamespace`, a platform-admin delegation that is false - by default. That flag is the only namespace policy in this direction: what may actually be read is - bounded by the provider credential's own Kubernetes RBAC, and a `"*"` reaches every namespace that - credential can read, as one cluster-wide watch. + requires `ClusterProvider.spec.allowCrossNamespace.sources`, a platform-admin delegation that is + false by default. That flag is the only namespace policy in this direction: what may actually be + read is bounded by the provider credential's own Kubernetes RBAC, and a `"*"` reaches every + namespace that credential can read, as one cluster-wide watch. On an **in-cluster** provider that delegation deliberately bypasses live namespace RBAC: the owner of an admitted `GitTarget` can then mirror another namespace's objects — read through the @@ -106,7 +112,9 @@ namespace it already knows. Narrowing that further is | SOPS/age key material | Decrypts (and the public key encrypts) Secret data written to Git. | | Redis/Valkey queue | Buffers decoded audit events in transit; not an audit archive. | | Audit ingress (`/audit-webhook/default` for the default provider) | Accepts audit traffic; protected by mutual TLS via cert-manager. | +| Push receiver (`--receiver-bind-address`, off by default) | Plain HTTP for Git hosts' push webhooks. Each `GitProvider` route is a path derived from its token, and a delivery must carry a valid signature or token for it. A delivery is only a hint that makes the branch worker read the remote, never a fact about the branch. | | Generated Secret material | Signing keys and generated age keys live in cluster Secrets. | +| `ClusterProvider.spec.allowCrossNamespace.appliers` | Lets every namespace `accessFrom` admits suspend and apply a `Kustomization` or `Application` in another namespace, such as `flux-system`, through the provider's identity. A delegation, not owner consent: it does not prove the namespace owns the applier or its repository. A trigger only acts while an eligible `GitTarget` names it, and the binding check compares the `GitProvider` URL, not write access. | ## Secret data the controller writes to Git diff --git a/docs/spec/commitrequest-design.md b/docs/spec/commitrequest-design.md index cb17dd2c7..79d4694f9 100644 --- a/docs/spec/commitrequest-design.md +++ b/docs/spec/commitrequest-design.md @@ -173,7 +173,7 @@ that never left the worker would surface later, out of order. A decided write is kept when the remote cannot be reached, whether that happens at its commit (a rebuild of the checkout that needs a fetch) or at its push. The worker retries on its own schedule -(10s, doubling to 5m), with parent recovery owning attempts while its obligation remains active. A +(10s, doubling to 5m), with parent recovery owning attempts while it remains open. A request riding the write stays `WaitingForPush` until publication resolves it or worker shutdown fails it. An unavailable remote can leave it pending indefinitely, but retries no longer depend on another commit arriving. Only a failure of the write itself fails the request: a refused plan, or a diff --git a/docs/spec/e2e-bi-directional-corner.md b/docs/spec/e2e-bi-directional-corner.md index 00958606a..ddb801bc7 100644 --- a/docs/spec/e2e-bi-directional-corner.md +++ b/docs/spec/e2e-bi-directional-corner.md @@ -25,8 +25,8 @@ Prometheus, cert-manager) are delivered as Flux `HelmRelease`s from cannot be removed. Argo CD is *not* load-bearing: only bi-directional tests need it. Installing it in all four CI legs would tax every leg to serve two specs. -**Cohesion.** [`docs/bi-directional.md`](../bi-directional.md) currently says, under -"What is not complete yet": +**Cohesion.** [`docs/bi-directional.md`](../bi-directional.md) said, under +"What is not complete yet", when this corner was designed: > equivalent alignment patterns for Argo CD or other GitOps operators @@ -318,8 +318,9 @@ Per the existing per-file CRD-group convention, add [`test/e2e/icecream.go`](../../test/e2e/icecream.go) and register it in the suite's CRD pre-clean loop (`e2e_suite_test.go:225`). -The Flux spec's body does not change. It keeps its exact-commit-count discipline, -which works because it drives Flux as a manually triggered applier. +The Flux spec keeps its exact-commit-count discipline, which works because it +drives Flux as a manually triggered applier until its last phase hands the +`Kustomization` to a `FluxTrigger`. ### 5. Push-triggered reconciliation (the webhook) @@ -360,9 +361,9 @@ turns automated sync on, and there the webhook's refresh does auto-sync. ## Test design: `argocd_bi_directional_e2e_test.go` -Every spec runs the standard of [`docs/argocd.md`](../argocd.md): the `Application` -has automated sync off (`automated: {enabled: false, prune: true}`) and is named by -the `GitTarget`'s `spec.applier.applicationRef`, so every sync it runs is one GitOps +Every spec runs the standard of [`docs/bi-directional-argocd.md`](../bi-directional-argocd.md): the `Application` +has automated sync off (`automated: {enabled: false, prune: true}`) and is held by +the `ArgoCDTrigger` the `GitTarget` names, so every sync it runs is one GitOps Reverser started, pinned to a commit it checked. The earlier phases that ran automated sync (`selfHeal: true` losing the edit, `ignoreDifferences`, and `selfHeal: false` with the webhook) are gone with it: they characterized an Argo CD @@ -377,13 +378,16 @@ path would each stamp their own tracking-id on the same objects and raise The `Application` lives in the `argocd` namespace, targets `spec.destination.namespace = `, `spec.source.path = `, -`targetRevision: main`, `directory.recurse: true`. The `GitProvider` has a -`gitea` receiver, registered as a Gitea webhook next to Argo CD's own. +`targetRevision: main`, `directory.recurse: true`. The trigger lives in the test +namespace, so each describe first grants `allowCrossNamespace.appliers` on the +`default` `ClusterProvider`, as a user must for an `Application` in `argocd`. The +`GitProvider` has a `gitea` receiver, registered as a Gitea webhook next to Argo +CD's own. Ordering matters: `SetupRepo` leaves the Gitea repo **empty**, so the spec seeds `main` first, creates the `Application` (nothing to sync yet), and only then stands -up the reverser pipeline, whose `GitTarget` needs the `Application` to report -`ApplierConfigured=True`. +up the reverser pipeline, whose `GitTarget` needs its trigger's `Application` to +report `ApplyTriggerReady=True`. ### Spec A — a Git-side push is synced through a pinned `ForeignCommit` @@ -393,7 +397,8 @@ up the reverser pipeline, whose `GitTarget` needs the `Application` to report 3. Assert the `Application`'s operation state is GitOps Reverser's: initiated by `gitops-reverser` with `automated: true`, reason `ForeignCommit` in its info, `sync.revision` and `syncResult.revision` both the pushed commit, `Succeeded`, - with resources. The `GitTarget`'s `status.applier` names the same commit. + with resources. The trigger's `status.operation` names the same commit, as + `Succeeded`, written by the branch worker with no reconcile asked for. 4. Assert the **live** object carries `argocd.argoproj.io/tracking-id` (exact value format) and `kubectl.kubernetes.io/last-applied-configuration`, and the **committed file** carries neither, nor `managedFields` or `resourceVersion`. @@ -414,7 +419,7 @@ The race the apply trigger exists for. `container: WaffleBowl` and wait for that commit. 3. `Consistently` assert live stays `WaffleBowl`, the commit count is exactly baseline + 2, and the `Application`'s last `finishedAt` has not moved: GitOps - Reverser's own commits owe Argo CD no sync. + Reverser's own commits ask Argo CD for no sync. 4. Assert Argo CD, refreshed by its webhook, reports the `Application` `Synced`: there was nothing to catch up. @@ -427,8 +432,10 @@ env var no overlay declaration can express. 1. Seed the base and overlay, create the `Application` on the overlay (kustomize), and deploy it with one manual sync, before any `GitTarget` names it. 2. Drift `LOG_LEVEL` to `debug`. -3. Create the `GitTarget` on the overlay with `applicationRef` and - `onRefusal: Reapply`, and a `WatchRule` on `deployments`. +3. Create the `GitTarget` on the overlay, an `ArgoCDTrigger` holding the `Application`, + name the trigger in the `GitTarget`'s `spec.applyTriggerRef` with + `onRefusal: Reapply`, wait for `ApplyTriggerReady=True`, and create a `WatchRule` + on `deployments`. 4. Assert the write is refused at the boundary (`WriteBoundaryRefused`), and that GitOps Reverser's `Refusal` sync, pinned to the tip, reverts `LOG_LEVEL` to `info`, with no commit for it. @@ -436,13 +443,14 @@ env var no overlay declaration can express. ### Spec E — an `Application` with automated sync on is never synced 1. Re-apply the `Application` with automated sync on (an `automated` block with no - `enabled`), and request a `GitTarget` reconcile, since the applier is read on + `enabled`), and request a trigger reconcile, since the applier is read on reconcile rather than watched. -2. Assert `ApplierConfigured=False` with `AutomatedSyncEnabled`, and `Stalled`. +2. Assert the trigger's `Configured=False` with `AutomatedSyncEnabled`, and the + `GitTarget`'s `ApplyTriggerReady=False` with the same reason, and `Stalled`. 3. Push a Git-side change. Assert an `ApplyRequestFailed` Event: the `ForeignCommit` was decided and refused at the `Application`. 4. `Consistently` assert no operation initiated by `gitops-reverser` names the - pushed commit, and `status.applier` is unchanged. Argo CD's own automated sync + pushed commit, and the trigger's `status.operation` is unchanged. Argo CD's own automated sync applies the push; that is not GitOps Reverser's. ### Spec D — arming the landmine (phase 2) @@ -530,12 +538,15 @@ New matrix entry in `.github/workflows/ci.yml`: ```yaml - name: bi-directional script: "task test-e2e-bi-directional" + e2e_report_name: "bi-directional" needs_artifact: false coverage: "1" - e2e_ginkgo_procs: "2" - k3d_agent_count: "0" ``` +No `e2e_label_filter` and no `e2e_ginkgo_procs`: the task pins its own label filter +and Ginkgo procs (`BI_DIRECTIONAL_GINKGO_PROCS`, default `3`), so values here would +be ignored. + `coverage: "1"` — the leg uses the default `config-dir` install, which carries the `GOCOVERDIR` overlay, so Codecov unions its `e2e` upload with the other legs and the non-regression ratchet still holds. @@ -545,8 +556,8 @@ check on `main` (only *Project image*, *Unit tests*, *Lint* are), so adding a le carries no merge-gating risk. The two specs are pulled out of `full-core`, so that leg gets faster. -`--procs=2`: the two specs own separate Gitea repos and separate CRD groups, and -neither is `Serial`. +Three procs: the Flux, Argo CD and provenance spec files own separate Gitea repos, +namespaces and resource names, and none is `Serial`. ## Risks @@ -588,10 +599,10 @@ neither is `Serial`. ## References -- [`docs/bi-directional.md`](../bi-directional.md) — the user-facing guidance this closes gaps in -- [`docs/design/e2e-serial-registry.md`](e2e-serial-registry.md) — parallelism and shared cluster state -- [`docs/design/e2e-ci-runner-sharding-plan.md`](../finished/e2e-ci-runner-sharding-plan.md) — leg membership and rebalancing +- [`docs/bi-directional.md`](../bi-directional.md): the user-facing guidance this closes gaps in +- [`docs/spec/e2e-serial-registry.md`](e2e-serial-registry.md): parallelism and shared cluster state +- [`docs/finished/e2e-ci-runner-sharding-plan.md`](../finished/e2e-ci-runner-sharding-plan.md): leg membership and rebalancing - [`test/e2e/flux_bi_directional_e2e_test.go`](../../test/e2e/flux_bi_directional_e2e_test.go) and - [`test/e2e/argocd_bi_directional_e2e_test.go`](../../test/e2e/argocd_bi_directional_e2e_test.go) — + [`test/e2e/argocd_bi_directional_e2e_test.go`](../../test/e2e/argocd_bi_directional_e2e_test.go): the two specs in this corner -- [`internal/sanitize/types.go`](../../internal/sanitize/types.go) — the strip lists +- [`internal/sanitize/types.go`](../../internal/sanitize/types.go): the strip lists diff --git a/docs/spec/e2e-serial-registry.md b/docs/spec/e2e-serial-registry.md index 81ee34336..fdaa36e84 100644 --- a/docs/spec/e2e-serial-registry.md +++ b/docs/spec/e2e-serial-registry.md @@ -11,7 +11,7 @@ under bounded Ginkgo parallelism (Phase 2.5 of > The two audit-consumer specs, `bi_directional`, and `aggregated_apiserver` have > all been de-serialized (see the "De-serialized" section). `crd_lifecycle` was > de-serialized and then **re-serialized** (commit `3d249e3`) when a residual -> cluster-wide CRD-discovery churn re-flaked it. The current Serial set is the two +> cluster-wide CRD-discovery churn re-flaked it. The current Serial set is the three > singleton-controller specs, `crd_lifecycle`, and `playground`. ## Rule @@ -32,6 +32,7 @@ row below. | File | Container | Shared state it touches | Why name isolation can't fix it | |---|---|---|---| | [test/e2e/restart_reconcile_e2e_test.go](../../test/e2e/restart_reconcile_e2e_test.go) | `Restart Reconcile Safety` | Rollout-restarts the controller deployment. | The controller is a singleton; restarting it disrupts in-flight reconciles/commits for every other spec. | +| [test/e2e/flux_restart_recovery_e2e_test.go](../../test/e2e/flux_restart_recovery_e2e_test.go) | `Flux Restart Recovery` | Rollout-restarts the controller deployment while a Flux apply runs, and labels a node so that apply's Deployment can schedule. | The controller is a singleton, as for `Restart Reconcile Safety`; a node label is cluster-wide. | | [test/e2e/image_refresh_test.go](../../test/e2e/image_refresh_test.go) | `image refresh dependency chain` | Changes the controller image / redeploys the controller. | Same singleton controller; an image swap perturbs all concurrent specs. | | [test/e2e/crd_lifecycle_e2e_test.go](../../test/e2e/crd_lifecycle_e2e_test.go) | `Manager CRD Lifecycle` | Installs/deletes a cluster-scoped CRD and watches all `customresourcedefinitions` cluster-wide, asserting exact Git file presence/absence. | A concurrent CRD install/delete bumps the global discovery-catalog generation and re-resolves every GitTarget's watched-type tables, delaying this spec's reconcile (CRD file appears late / post-delete sweep lags — the 649↔673 flake). A cluster-wide catalog bump cannot be scoped by name. | | [test/e2e/tilt_playground_e2e_test.go](../../test/e2e/tilt_playground_e2e_test.go) | `playground` | Reusable manual-playground fixture: fixed (non-randomized) `tilt-playground` namespace and `playground` repo/provider names, preserved across runs. | A singleton by construction — its resource names are intentionally stable so `task playground-*` can re-attach, so per-run name isolation does not apply. | diff --git a/docs/spec/gitpath-foreign-content-stringency.md b/docs/spec/gitpath-foreign-content-stringency.md index b48e7f791..607d1be01 100644 --- a/docs/spec/gitpath-foreign-content-stringency.md +++ b/docs/spec/gitpath-foreign-content-stringency.md @@ -509,7 +509,7 @@ semantics rather than reinventing glob handling. extra states. If a real need ever appears the ratchet still permits adding one later, but it is explicitly out of scope here. - **D-foreign-5 — `WrapAsConfigMap` deferred — NOTED for later.** Out of scope for this doc. The only thing - this design owes it is to *keep the route open*, which refusing foreign content now does: a future + this design has to do for it is to *keep the route open*, which refusing foreign content now does: a future opt-in could wrap matched foreign files into a ConfigMap without a breaking change. It gets its own design (and its own narrow field) when the time comes. - **D-foreign-6 — shadowing is closed by a write-time invariant, not static analysis — DECIDED (§4.3).** diff --git a/docs/spec/status-conditions-guide.md b/docs/spec/status-conditions-guide.md index 36ada09ff..c85da1e18 100644 --- a/docs/spec/status-conditions-guide.md +++ b/docs/spec/status-conditions-guide.md @@ -112,10 +112,10 @@ a precedence stated in one place. Gates do not set them; gates *contribute* to a This is not a style preference. When each gate wrote the trio itself, the trio said whatever the last gate said — so a GitTarget whose Git path had been refused (terminal) was then handed to the source/provider projection, which stamped `Stalled=False, Reconciling=True` over the refusal because -a provider happened to be mid-check. To a human reading `Ready` little changed. To kstatus — which -never reads `Ready`, only the abnormal-true pair — the object flipped from `Failed` to `InProgress`, -so `kubectl wait` and every CI gate built on it waited out its timeout on an object that was never -going to converge. +a provider happened to be mid-check. To a human reading `Ready` little changed. kstatus reads the +abnormal-true pair first, and `Ready` only as a fallback when neither decides, so to it the object +flipped from `Failed` to `InProgress`, and `kubectl wait` and every CI gate built on it waited out +its timeout on an object that was never going to converge. ### Status writes are suppressed when nothing changed @@ -285,6 +285,31 @@ Canonical reads: existed so a `sourceNamespace: "*"` against a policy admitting nothing could not look healthy while mirroring nothing; `"*"` is now one cluster-wide watch, which cannot resolve to an empty set. +### Apply triggers + +`FluxTrigger` and `ArgoCDTrigger` use the trio with `Configured` as their gate, and a `FluxTrigger` +adds `Held` as a second one. + +Canonical reads: + +- **not referenced** (no `GitTarget` names the trigger): `Configured=Unknown` and `Ready=True`, both + reason `NotReferenced`, `Reconciling=False`, `Stalled=False`. As for a `GitTarget` with nothing to + mirror, doing nothing when nothing is asked is converged, and a Flux health check that waits on + the trigger must not wait for the `GitTarget` that will name it +- holding its applier: `Configured=True`, for a `FluxTrigger` `Held=True` (`Suspended` or + `Applying`), and `Ready=True` +- applier or `ClusterProvider` missing (`ApplierNotFound`, `ClusterProviderNotFound`): + `Ready=False`, `Reconciling=True`, `Stalled=False`, because Flux and Argo CD objects are often + created after the trigger. Every other `Configured=False` reason is `Stalled=True` +- an operation past its time (`status.operation.phase: Unresolved`): `Ready=False`, + `Reconciling=True`, reason `OperationUnresolved`. It clears when the operation completes +- a `FluxTrigger` whose suspend is not handled yet, or that is handing its `Kustomization` back: + `Held=False` (`Suspending`, `Releasing`), `Ready=False`, `Reconciling=True` + +A `GitTarget` naming a trigger gains `ApplyTriggerReady`, a gate on its own trio. `SourceMismatch`, +or a trigger that is itself `Stalled`, stalls the target; a missing trigger (`ApplyTriggerNotFound`) +or one that is not `Ready` yet keeps it `Reconciling`. + ### CommitRequest (one-shot) CommitRequest runs once to a terminal outcome. Best-practice 1 above would suggest a `Succeeded` summary, diff --git a/hack/crdfields/fields.yaml b/hack/crdfields/fields.yaml index 44f288b41..f0d8a53f9 100644 --- a/hack/crdfields/fields.yaml +++ b/hack/crdfields/fields.yaml @@ -17,7 +17,7 @@ preamble: | ## Field overview - Ready-made manifests for all six objects live in [`config/samples/`](../config/samples/). They + Ready-made manifests for all eight objects live in [`config/samples/`](../config/samples/). They cross-reference the same `example-provider` / `example-target` names, so the set can be applied together and edited down, rather than assembled from the fragments below. @@ -67,8 +67,16 @@ kinds: doc: >- Deny-by-default list of control-cluster namespaces that may reference this provider. Omitted, none may; an empty `selector: {}` admits every namespace - - path: allowAnySourceNamespace - doc: Whether rules may watch namespaces other than their own, `*` included + - path: allowCrossNamespace.sources + default: "`false`" + doc: >- + Whether rules may watch namespaces other than their own, `*` included. See + [cross-namespace grants](#what-a-namespace-may-reach-outside-itself-specallowcrossnamespace) + - path: allowCrossNamespace.appliers + default: "`false`" + doc: >- + Whether triggers may name a `Kustomization` or `Application` in another namespace than + their own - path: attribution.auditRoute default: the provider's own name doc: >- @@ -84,6 +92,8 @@ kinds: skip: the old spelling of `client.qps`, kept in the schema only so admission refuses it - path: burst skip: the old spelling of `client.burst`, kept in the schema only so admission refuses it + - path: allowAnySourceNamespace + skip: removed for `allowCrossNamespace.sources`, kept in the schema only so admission refuses `true` - kind: GitTarget title: "`GitTarget` (namespaced): which branch and folder" @@ -137,13 +147,12 @@ kinds: doc: >- Which deletions reach Git. See [deletion policy](#deletion-policy-specprunemode) - - path: applier + - path: applyTriggerRef default: the applier applies on its own schedule doc: >- - The Flux `Kustomization` (`kustomizationRef`) or Argo CD `Application` (`applicationRef`) - that applies this folder, which GitOps Reverser then asks to apply, and for a - `Kustomization` how long the branch stays quiet before it catches up (`quietPeriod`, - `30s`). See [who applies this folder](#who-applies-this-folder-specapplier) + The `FluxTrigger` or `ArgoCDTrigger`, in this namespace, holding the applier of this + folder, which GitOps Reverser then asks to apply. See + [who applies this folder](#who-applies-this-folder-specapplytriggerref) - path: onRefusal doc: >- What happens to an edit with no legal destination. See @@ -207,6 +216,34 @@ kinds: Label selector the API server applies to the item's LIST and WATCH. See [selecting objects by label](#selecting-objects-by-label-rulesobjectselector) + - kind: FluxTrigger + title: "`FluxTrigger` (namespaced): the Flux `Kustomization` GitOps Reverser asks to apply" + intro: "The references are immutable: to hold another `Kustomization`, delete the trigger and create a new one." + fields: + - path: clusterProviderRef + doc: >- + The `ClusterProvider` whose cluster holds the `Kustomization`. Its `accessFrom` must admit + the trigger's namespace + - path: kustomizationRef + doc: >- + The `Kustomization` that applies the folders of the `GitTarget`s naming this trigger. See + [who applies this folder](#who-applies-this-folder-specapplytriggerref) + - path: quietPeriod + doc: How long the branch must stay quiet after GitOps Reverser's own pushes before a `CatchUp` + + - kind: ArgoCDTrigger + title: "`ArgoCDTrigger` (namespaced): the Argo CD `Application` GitOps Reverser asks to sync" + intro: "The references are immutable: to hold another `Application`, delete the trigger and create a new one." + fields: + - path: clusterProviderRef + doc: >- + The `ClusterProvider` whose cluster holds the `Application`. Its `accessFrom` must admit + the trigger's namespace + - path: applicationRef + doc: >- + The `Application`, with automated sync off, that syncs the folders of the `GitTarget`s + naming this trigger. See [who applies this folder](#who-applies-this-folder-specapplytriggerref) + - kind: CommitRequest title: "`CommitRequest` (namespaced): save now" fields: diff --git a/hack/e2e/install-plain-manifests.sh b/hack/e2e/install-plain-manifests.sh index 2a1d00ca3..6cd6e24c3 100755 --- a/hack/e2e/install-plain-manifests.sh +++ b/hack/e2e/install-plain-manifests.sh @@ -35,7 +35,7 @@ trap 'rm -rf "${tmpdir}"' EXIT # cannot map if its CRD is created in the same apply. This mirrors the documented two-step user # flow (kubectl apply -f crds.yaml, then -f install.yaml), so the e2e exercises it for real. "${KUBECTL}" --context "${CTX}" apply -f dist/crds.yaml -"${KUBECTL}" --context "${CTX}" wait --for=condition=Established --timeout=60s -f dist/crds.yaml +CTX="${CTX}" KUBECTL="${KUBECTL}" bash hack/e2e/wait-crds-established.sh 60 -f dist/crds.yaml cp dist/install.yaml "${tmpdir}/install.yaml" diff --git a/hack/e2e/wait-crds-established.sh b/hack/e2e/wait-crds-established.sh new file mode 100755 index 000000000..d1128f140 --- /dev/null +++ b/hack/e2e/wait-crds-established.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Waits for CRDs that were JUST applied to be Established. +# +# Usage: wait-crds-established.sh ... or -f FILE> +# Env: CTX (required), KUBECTL (default kubectl) +# +# `kubectl wait --for=condition=Established` cannot be run straight after the apply that created +# the CRDs. Until the apiextensions controller writes its first conditions, a new CRD carries +# `status.conditions: null`, and kubectl's condition check (ConditionalWait.checkCondition, +# k8s.io/kubectl pkg/cmd/wait/condition.go) fails on that instead of waiting: +# +# .status.conditions accessor error: is of the type , expected []interface{} +# +# so the wait exits 1 within seconds, well inside its timeout. That aborted prepare-e2e on CI runs +# 34824995173 and 35235710425. A `--for=jsonpath=` wait fails the same way on the null list. This +# retries ONLY that error, inside one overall deadline; any other failure, including the timeout, +# fails as before. + +set -euo pipefail + +: "${CTX:?CTX is required}" +KUBECTL="${KUBECTL:-kubectl}" +timeout_seconds="${1:?timeout in seconds is required}" +shift +[ "$#" -gt 0 ] || { echo "wait-crds-established.sh: name the CRDs to wait for" >&2; exit 2; } + +deadline=$(( SECONDS + timeout_seconds )) +while :; do + remaining=$(( deadline - SECONDS )) + [ "${remaining}" -gt 0 ] || remaining=1 + if output="$("${KUBECTL}" --context "${CTX}" wait --for=condition=Established \ + --timeout="${remaining}s" "$@" 2>&1)"; then + printf '%s\n' "${output}" + exit 0 + fi + if ! grep -q '\.status\.conditions accessor error: ' <<<"${output}" || [ "${SECONDS}" -ge "${deadline}" ]; then + printf '%s\n' "${output}" >&2 + exit 1 + fi + echo "CRD status not written yet; waiting again" >&2 + sleep 1 +done diff --git a/internal/applier/applier.go b/internal/applier/applier.go index 942d7ef25..ea63a9766 100644 --- a/internal/applier/applier.go +++ b/internal/applier/applier.go @@ -7,12 +7,14 @@ commit is safe to apply. The worker decides WHEN; this package decides HOW, off so a slow or unreachable applier never holds a branch. Two kinds are served, behind one Client: a Flux Kustomization whose sourceRef is a GitRepository -(spec.applier.kustomizationRef, flux.go), and an Argo CD Application with automated sync off -(spec.applier.applicationRef, argocd.go). See docs/design/apply-trigger.md. +(a FluxTrigger's spec.kustomizationRef, flux.go), held suspended while idle and resumed for each +apply, and an Argo CD Application with automated sync off (an ArgoCDTrigger's +spec.applicationRef, argocd.go). Either way the applier applies only when GitOps Reverser asks. +See docs/design/apply-trigger.md, "The Flux applier, suspended". -A request asks and never reports. Nothing here claims an apply happened; the applier's own status -(lastAppliedRevision on a Kustomization, status.operationState on an Application) is the record of -that. +Each request starts one operation, which the dispatcher follows until the applier's own status +(the handled tokens and Ready of a Kustomization, status.operationState of an Application) says it +completed. */ package applier @@ -34,26 +36,84 @@ import ( "k8s.io/client-go/dynamic" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" ) -// Client asks one kind of applier, in one cluster, to apply. The dispatcher and the -// ApplierConfigured check pick one by the applier's kind (NewClient), and use nothing else of it. +// Client asks one kind of applier, in one cluster, to apply. The dispatcher (Dispatcher.client) and +// the trigger's Configured check (NewClient) pick one by the applier's kind, and use nothing else of +// it. type Client interface { - // Validate answers ApplierConfigured: the applier exists, applies want, and may be asked. - Validate(ctx context.Context, ref types.NamespacedName, want Binding) Verdict - // Request asks the applier to apply commit, after checking it still applies want and asking - // current just before it writes. A permanent error is a configuration no retry can fix; - // errStale is a decision that no longer held. - Request( - ctx context.Context, - ref types.NamespacedName, - want Binding, - commit string, - reason v1alpha3.ApplyReason, - current func() bool, - ) (Stamped, error) - // Handled reports whether the applier has finished the apply a Request asked for. - Handled(ctx context.Context, ref types.NamespacedName, s Stamped) (bool, error) + // Validate answers a trigger's Configured condition: the applier exists, applies a source GitOps + // Reverser can drive, and may be asked. It reports the source it read, whatever the verdict. + Validate(ctx context.Context, ref types.NamespacedName) Verdict + // Request starts one apply of ask.Commit, after checking the applier still applies ask.Want and + // asking ask.Current just before it writes. A permanent error is a configuration no retry can + // fix; errStale is a decision that no longer held. + Request(ctx context.Context, ref types.NamespacedName, ask Ask) (Started, error) + // Progress reports how far the applier is with the apply a Request started, and takes the + // operation's next step when it is the client's to take (a Flux Kustomization is suspended again + // after its apply). A read or write that fails is an error, and its Progress says nothing. + Progress(ctx context.Context, ref types.NamespacedName, s *Started) (Progress, error) +} + +// Ask is what one Request asks of an applier. +type Ask struct { + // Want is the repository and branch the applier must apply. + Want Binding + // Commit is the commit to apply; empty when the remote does not carry the branch. + Commit string + // Reason is why. + Reason v1alpha3.ApplyReason + // Token identifies the request in the applier's own status: the requestedAt a Flux + // Kustomization is resumed with, or the request entry in an Argo CD operation's info. + Token string + // Current reports whether the decision still holds; nil always does. + Current func() bool + // Record writes the operation on its trigger, and returns once it is written. It is called just + // before the write that starts the apply, so a restart finds every apply that may have started. + // Current is asked again after it, since it can wait on the status writer and the API server. + // Nil records nothing. + Record func(ctx context.Context) error + // Held reports, from a fresh read, whether the trigger holds its Flux Kustomization suspended + // (Held=True), and errs when it does not: a resume while the trigger's reconciler is still + // suspending it (an adoption, an external resume) would race that suspend. It is asked just + // before Record. Nil always holds; the Argo CD client never asks it. + Held func(ctx context.Context) error +} + +// recordOnce is Record, called the first time only: a request that writes more than once (an Argo +// CD update that lost a race) records its operation once. +func (a Ask) recordOnce() func(ctx context.Context) error { + recorded := a.Record == nil + return func(ctx context.Context) error { + if recorded { + return nil + } + if err := a.Record(ctx); err != nil { + return err + } + recorded = true + return nil + } +} + +// holds asks Current whether the decision still holds; a nil Current always does. +func (a Ask) holds() bool { + return a.Current == nil || a.Current() +} + +// Progress is where the operation a Request started stands, as the applier's status shows it. +type Progress struct { + // Outcome is git.ApplyRequested, git.ApplyRunning, git.ApplySettling, git.ApplySucceeded or + // git.ApplyFailed. + Outcome git.ApplyOutcome + // Message is why a Failed operation failed, or what a Succeeded one applied instead of the + // commit. + Message string + // SuspendToken is the requestedAt the suspend after a Flux apply carried, once written. + SuspendToken string + // AppliedRevision is the revision the applier reports it applied, once the operation completed. + AppliedRevision string } var ( @@ -83,29 +143,36 @@ type Outcome string // OutcomeCurrent asked for nothing: the applier needed no request (see each client's Request). const OutcomeCurrent Outcome = "Current" -// ApplierConfigured reasons (docs/configuration.md, "Who applies this folder"). Succeeded and -// Failed are the generic reasons, aliased from Flux's meta package as the conditions guide asks. +// Configured and ApplyTriggerReady reasons (docs/configuration.md, "Who applies this folder"). +// Succeeded and Failed are the generic reasons, aliased from Flux's meta package as the conditions +// guide asks. const ( - // ReasonSucceeded is ApplierConfigured=True: the applier exists, applies this repository and - // branch, and may be asked to apply. + // ReasonSucceeded is Configured=True: the applier exists, applies a source GitOps Reverser can + // drive, and may be asked to apply. ReasonSucceeded = fluxmeta.SucceededReason - // ReasonApplierNotFound is no such object in the source cluster, or its kind is not served there. + // ReasonApplierNotFound is no such object in the applier's cluster, or its kind is not served + // there. ReasonApplierNotFound = "ApplierNotFound" - // ReasonSourceMismatch is an applier that applies a different repository or branch. + // ReasonUnsupportedSource is an applier whose source GitOps Reverser cannot drive: a + // Kustomization that applies no GitRepository, or one pinned to anything but a branch, or an + // Application with several sources. + ReasonUnsupportedSource = "UnsupportedSource" + // ReasonSourceMismatch is ApplyTriggerReady=False on a GitTarget whose trigger's applier applies + // a different repository or branch than the GitTarget writes. ReasonSourceMismatch = "SourceMismatch" // ReasonAutomatedSyncEnabled is an Argo CD Application with automated sync on, which syncs on // its own and so is never asked to. ReasonAutomatedSyncEnabled = "AutomatedSyncEnabled" - // ReasonForbidden is a source-cluster identity that may not read or write the applier. + // ReasonForbidden is a ClusterProvider identity that may not read or write the applier. ReasonForbidden = "Forbidden" - // ReasonCheckFailed is a check that could not complete: the source cluster did not answer. + // ReasonCheckFailed is a check that could not complete: the applier's cluster did not answer. // Verdict.Err carries the cause; it says nothing about the applier itself. ReasonCheckFailed = fluxmeta.FailedReason ) -// Verdict is the answer Validate gives for the ApplierConfigured condition. +// Verdict is the answer Validate gives for a trigger's Configured condition. type Verdict struct { - // OK is ApplierConfigured=True. + // OK is Configured=True. OK bool // Reason is the condition reason. Reason string @@ -114,35 +181,81 @@ type Verdict struct { // Err is set only with ReasonCheckFailed: the read that failed. A caller may prefer to keep the // verdict it already published over replacing a known answer with "could not tell". Err error + // Source is what the applier applies, when Validate could read it. + Source *Source +} + +// Source is what an applier applies from: a GitRepository's spec.url and spec.ref.branch for a +// Kustomization, or an Application's spec.source repoURL and targetRevision. +type Source struct { + // GitRepository is the Kustomization's GitRepository (Flux only). + GitRepository types.NamespacedName + // URL is the repository. + URL string + // Revision is the branch a GitRepository follows, or an Application's targetRevision as + // written. + Revision string } -// Stamped is what one Request did, with what the applier reported before it, so Handled can tell -// when it has finished the apply it was asked for. -type Stamped struct { +// Started is the operation one Request started, may have started (Unconfirmed), or read back from +// the trigger's record (Recovered), for Progress to follow; with Outcome Current, nothing was asked. +type Started struct { // Outcome is what was asked, if anything. Outcome Outcome - // Token identifies the request: the reconcile.fluxcd.io/requestedAt value stamped on a Flux - // object, or the request entry in an Argo CD operation's info. Empty when nothing was asked. + // Token identifies the request: the requestedAt a Flux Kustomization was resumed with, or the + // request entry in an Argo CD operation's info. Empty when nothing was asked. Token string - // Source is the GitRepository the Kustomization applies from (Flux only). - Source types.NamespacedName - // PriorAttemptedRevision is the Kustomization's status.lastAttemptedRevision before the stamp - // (Flux only). - PriorAttemptedRevision string - // PriorArtifactRevision is the GitRepository's status.artifact.revision before the stamp - // (Flux only). - PriorArtifactRevision string + // Commit is the commit asked for. + Commit string + // SuspendToken is the requestedAt a Flux Kustomization was suspended again with, once it was. + SuspendToken string + // Overdue is set by the dispatcher once the operation is Unresolved. A Flux Kustomization is + // then suspended at once, rather than once its apply was seen to end. + Overdue bool + // Recovered is an operation read back from the trigger's record rather than started by this + // process: the write that would start it (a Flux resume, an Argo CD update) may never have been + // made, and the record says nothing of that. + Recovered bool + // Unconfirmed is the error of the write that started the operation (a Flux resume, an Argo CD + // update), when it failed without saying whether it reached the API server (a timeout, a lost + // connection); nil for one that was answered. Such an operation is followed, because the apply + // may be running, as a Recovered one is: a Flux one is suspended again, and an Argo CD one is + // followed by its token until it completed or can no longer be set. + Unconfirmed error + // unseen is a Recovered or Unconfirmed operation whose token the Kustomization did not carry + // when it was first read: never written, or stamped over before it was handled. + unseen bool + // RequestedAt is when a Recovered operation was asked for, as its record says. A reconcile + // that ran since is the only evidence of its apply this process can read. + RequestedAt time.Time + // verdict is the outcome of a Flux apply, read once the resume's token was handled and kept + // until the suspend after it is. + verdict *Progress + // resumedFrom is what a reconcile that builds moves on the Kustomization, as it stood when this + // process resumed it: a verdict that reads it unmoved reads an apply from before the resume. + resumedFrom *reconcileMark } // Binding is the repository and branch an applier must apply from: the GitProvider's spec.url and -// the GitTarget's spec.branch. Request checks it before it writes anything, not only Validate, -// because a published ApplierConfigured verdict can be older than the applier it describes. +// the GitTarget's spec.branch. Request checks it before it writes anything, because a trigger's +// published source can be older than the applier it describes. type Binding struct { RepositoryURL string Branch string } -// AccessCheck reports whether the source-cluster identity may perform attrs. +// BindingMismatch returns why the source a trigger last read is not want, or "" when it is. The +// GitTarget's ApplyTriggerReady condition reads it; Request checks the same against the applier +// itself before it writes. +func BindingMismatch(trigger v1alpha3.ApplyTrigger, want Binding) string { + url, branch, known := trigger.SourceBinding() + if !known { + return "" + } + return bindingMismatch(trigger, url, branch, want) +} + +// AccessCheck reports whether the ClusterProvider identity may perform attrs. type AccessCheck func(ctx context.Context, attrs authorizationv1.ResourceAttributes) (bool, error) // permanentError marks a request no retry can make succeed: the applier is configured in a way a @@ -157,6 +270,19 @@ func (e permanentError) Unwrap() error { return e.err } // errStale reports a request whose decision no longer held when the write was due. var errStale = errors.New("the decision no longer holds: the branch moved on or became busy") +// mayHaveLanded reports whether a write that failed with err may have been made all the same: the +// API server did not answer it (a timeout, a lost connection, the context ending while it waited), +// or answered that it cannot tell (a server timeout, an internal error). Any other answer refused +// the write. +func mayHaveLanded(err error) bool { + var status apierrors.APIStatus + if !errors.As(err, &status) { + return true + } + return apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) || + apierrors.IsUnexpectedServerError(err) +} + // isPermanent reports whether err is a permanentError. func isPermanent(err error) bool { var p permanentError @@ -190,18 +316,18 @@ func readFailure( case apimeta.IsNoMatchError(err) || (apierrors.IsNotFound(err) && !namesObject(err, ref.Name)): return Verdict{ Reason: ReasonApplierNotFound, - Message: fmt.Sprintf("the source cluster does not serve %s (%s); is %s installed there?", + Message: fmt.Sprintf("the applier's cluster does not serve %s (%s); is %s installed there?", kind, gvr.GroupVersion(), product), } case apierrors.IsNotFound(err): return Verdict{ Reason: ReasonApplierNotFound, - Message: fmt.Sprintf("%s %s not found in the source cluster", kind, ref), + Message: fmt.Sprintf("%s %s not found in the applier's cluster", kind, ref), } case apierrors.IsForbidden(err): return Verdict{ Reason: ReasonForbidden, - Message: fmt.Sprintf("the source-cluster identity may not get %s %s: %v", kind, ref, err), + Message: fmt.Sprintf("the ClusterProvider identity may not get %s %s: %v", kind, ref, err), } default: return Verdict{ @@ -246,7 +372,7 @@ func accessVerdict( if !allowed { return &Verdict{ Reason: ReasonForbidden, - Message: fmt.Sprintf("the source-cluster identity may not %s %s %s (%s)", + Message: fmt.Sprintf("the ClusterProvider identity may not %s %s %s (%s)", verb, kind, ref, gvr.GroupResource()), } } @@ -263,7 +389,7 @@ func pastTense(verb string) string { // SelfSubjectAccessCheck answers an AccessCheck with a SelfSubjectAccessReview created through // client, so it is answered for the identity client authenticates as. It goes through the dynamic -// client rather than a typed clientset because that client is the only one a source cluster has. +// client rather than a typed clientset because that client is the only one a remote cluster has. func SelfSubjectAccessCheck(client dynamic.Interface) AccessCheck { return func(ctx context.Context, attrs authorizationv1.ResourceAttributes) (bool, error) { review := &authorizationv1.SelfSubjectAccessReview{ diff --git a/internal/applier/argocd.go b/internal/applier/argocd.go index 92eaa3bc6..2ddf0fb33 100644 --- a/internal/applier/argocd.go +++ b/internal/applier/argocd.go @@ -4,6 +4,7 @@ package applier import ( "context" + "encoding/json" "fmt" "strings" "time" @@ -16,6 +17,7 @@ import ( "k8s.io/client-go/dynamic" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/repourl" ) @@ -73,10 +75,22 @@ const defaultArgoRetryLimit = 5 // subresource each of those writes conflicts with an update made from an older read. const maxArgoUpdateConflicts = 5 +// Argo CD's refresh request (AnnotationKeyRefresh, pkg/apis/application/v1alpha1): the controller +// compares the Application again and removes the annotation in the write that records what it +// found (persistReconciliationStatus, controller/appcontroller.go). A refresh only compares: with +// automated sync off, it starts nothing. +const ( + argoRefreshAnnotation = "argocd.argoproj.io/refresh" + argoRefreshNormal = "normal" +) + // Operation phases, from gitops-engine's pkg/sync/common. const ( argoPhaseRunning = "Running" argoPhaseTerminating = "Terminating" + argoPhaseSucceeded = "Succeeded" + argoPhaseFailed = "Failed" + argoPhaseError = "Error" ) // ArgoCD asks an Argo CD Application to sync, in one cluster. @@ -101,7 +115,7 @@ func NewArgoCD(client dynamic.Interface) *ArgoCD { client: client, Now: time.Now, CanI: SelfSubjectAccessCheck(client), - PollInterval: DefaultHandledPollInterval, + PollInterval: DefaultPollInterval, } } @@ -115,49 +129,57 @@ func NewArgoCD(client dynamic.Interface) *ArgoCD { // resourceVersion it read, so an operation set in between makes the update fail rather than be // overwritten; it then reads again and waits for that one too. // -// current is asked just before the update. When it reports false the decision no longer holds and -// nothing is set: errStale. +// current is asked before record and again after it, just before the update. When it reports +// false the decision no longer holds and nothing is set: errStale. record is called once before +// the first update, so a restart finds every sync that may have started; the update carries +// ask.Token whichever read it is made from. +// +// An update that failed without saying whether it was made (a timeout, a lost connection, a server +// timeout or internal error) may have set the operation all the same, and Argo CD may be syncing: +// it returns an Unconfirmed operation rather than an error, which Progress follows by its token, +// so the apply window stays open until it is known to have run or never to run (settleUnseen). // // An empty commit (the remote does not carry the branch) has nothing to pin, and asks nothing // (OutcomeCurrent): Argo CD reports the missing branch itself on its next refresh, and the branch -// coming back is an observation that owes a new request. -func (a *ArgoCD) Request( - ctx context.Context, - ref types.NamespacedName, - want Binding, - commit string, - reason v1alpha3.ApplyReason, - current func() bool, -) (Stamped, error) { +// coming back is an observation that leaves a new request pending. +func (a *ArgoCD) Request(ctx context.Context, ref types.NamespacedName, ask Ask) (Started, error) { var waitUntil time.Time + var abandonedState string + record := ask.recordOnce() for conflicts := 0; ; { - app, err := a.readControlled(ctx, ref, want) + app, err := a.readControlled(ctx, ref, ask.Want) if err != nil { - return Stamped{}, err + return Started{}, err } - if commit == "" { - return Stamped{Outcome: OutcomeCurrent}, nil + if ask.Commit == "" { + return Started{Outcome: OutcomeCurrent}, nil } - if operationInProgress(app) { - if waitUntil.IsZero() { - waitUntil = a.Now().Add(a.OperationWait) - } - if err := a.awaitOperation(ctx, ref, waitUntil); err != nil { - return Stamped{}, err + if mustWait(app, abandonedState) { + if abandonedState, err = a.awaitOperation(ctx, ref, app, &waitUntil); err != nil { + return Started{}, err } continue } - if current != nil && !current() { - return Stamped{}, errStale + if !ask.holds() { + return Started{}, errStale + } + if err := record(ctx); err != nil { + return Started{}, fmt.Errorf("record the operation on its trigger before syncing %s %s: %w", + kindApplication, ref, err) } - stamped, err := a.setOperation(ctx, ref, app, commit, reason) + // Recording can wait on the status writer and the API server, and the branch can move on + // meanwhile: the decision is asked again just before the update. + if !ask.holds() { + return Started{}, errStale + } + started, err := a.setOperation(ctx, ref, app, ask) if apierrors.IsConflict(err) && conflicts < maxArgoUpdateConflicts { // Somebody wrote the Application since it was read: the controller's status, or an // operation of its own. Read it again, and decide again from what it is now. conflicts++ continue } - return stamped, err + return started, err } } @@ -181,98 +203,237 @@ func (a *ArgoCD) readControlled( // setOperation updates app, as read, with the sync operation and no operation state. The update // carries the resourceVersion read, so it fails with a conflict when anything wrote the -// Application since. +// Application since. One that failed without an answer is Unconfirmed (Request). func (a *ArgoCD) setOperation( - ctx context.Context, ref types.NamespacedName, app *unstructured.Unstructured, commit string, - reason v1alpha3.ApplyReason, -) (Stamped, error) { - token := requestToken(a.Now()) + ctx context.Context, ref types.NamespacedName, app *unstructured.Unstructured, ask Ask, +) (Started, error) { + token := ask.Token update := app.DeepCopy() - if err := unstructured.SetNestedField(update.Object, syncOperation(app, commit, reason, token), + if err := unstructured.SetNestedField(update.Object, syncOperation(app, ask.Commit, ask.Reason, token), "operation"); err != nil { - return Stamped{}, fmt.Errorf("build the sync operation for %s %s: %w", kindApplication, ref, err) + return Started{}, fmt.Errorf("build the sync operation for %s %s: %w", kindApplication, ref, err) } unstructured.RemoveNestedField(update.Object, "status", "operationState") + started := Started{Outcome: OutcomeSyncRequested, Token: token, Commit: ask.Commit} _, err := a.client.Resource(applicationGVR()).Namespace(ref.Namespace). Update(ctx, update, metav1.UpdateOptions{}) - if err != nil { - return Stamped{}, fmt.Errorf("set the sync operation on %s %s: %w", kindApplication, ref, err) + switch { + case err == nil: + return started, nil + case mayHaveLanded(err): + started.Unconfirmed = fmt.Errorf("set the sync operation on %s %s: %w", kindApplication, ref, err) + return started, nil } - return Stamped{Outcome: OutcomeSyncRequested, Token: token}, nil + return Started{}, fmt.Errorf("set the sync operation on %s %s: %w", kindApplication, ref, err) } -// awaitOperation waits until the Application has no operation in progress, or fails once -// waitUntil passes. It decides nothing: Request reads the Application again after it. -func (a *ArgoCD) awaitOperation(ctx context.Context, ref types.NamespacedName, waitUntil time.Time) error { +// awaitOperation waits until the Application, read as app, has no operation in progress, or fails +// once waitUntil passes, which it sets OperationWait from now when it is zero, so a request that +// waits twice waits once in all. It decides nothing: Request reads the Application again after it. +// +// A Running or Terminating operationState with no .operation, the same at every read of the whole +// wait, is no operation in progress: Argo CD processes an operation only while .operation is set, +// and clears it only once the state completed, so one whose .operation somebody removed (the +// workaround for a stuck hook) never ends. awaitOperation then returns that state +// (abandonedOperation), for Request to sync over it. +func (a *ArgoCD) awaitOperation( + ctx context.Context, ref types.NamespacedName, app *unstructured.Unstructured, until *time.Time, +) (string, error) { + if until.IsZero() { + *until = a.Now().Add(a.OperationWait) + } + waitUntil := *until + abandoned, polled := abandonedOperation(app), false for { if !a.Now().Before(waitUntil) { - return fmt.Errorf("%s %s has an operation in progress; asking again once the branch worker decides to", - kindApplication, ref) + if abandoned != "" && polled { + return abandoned, nil + } + return "", fmt.Errorf("%s %s has an operation in progress; asking again once the branch worker "+ + "decides to", kindApplication, ref) } timer := time.NewTimer(min(a.PollInterval, waitUntil.Sub(a.Now()))) select { case <-ctx.Done(): timer.Stop() - return fmt.Errorf("wait for the operation in progress on %s %s: %w", kindApplication, ref, ctx.Err()) + return "", fmt.Errorf("wait for the operation in progress on %s %s: %w", kindApplication, ref, ctx.Err()) case <-timer.C: } app, err := a.client.Resource(applicationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("get %s %s: %w", kindApplication, ref, err) + return "", fmt.Errorf("get %s %s: %w", kindApplication, ref, err) } if !operationInProgress(app) { - return nil + return "", nil + } + if now := abandonedOperation(app); now != abandoned { + abandoned = "" } + polled = true } } -// Handled reports whether Argo CD has finished the sync s asked for, which is when the apply -// window closes. The operation state is this request's when its info carries the request's token; -// it has finished once its phase is Succeeded, Failed or Error. A failed sync closes the window -// too: Argo CD's own retries of it are part of the operation, which stays Running until they end. -// An operation that is gone before it ever ran (somebody cleared it) runs nothing, and is handled. -func (a *ArgoCD) Handled(ctx context.Context, ref types.NamespacedName, s Stamped) (bool, error) { +// mustWait reports whether Request must wait for app's operation: one is in progress, and it is not +// the abandoned state (abandonedOperation) the last wait found unchanged throughout. +func mustWait(app *unstructured.Unstructured, abandoned string) bool { + return operationInProgress(app) && (abandoned == "" || abandonedOperation(app) != abandoned) +} + +// abandonedOperation is the Application's operationState, encoded, when it is Running or +// Terminating with no .operation set: a state Argo CD processes no further. It is "" otherwise. +func abandonedOperation(app *unstructured.Unstructured) string { + if operation, found, _ := unstructured.NestedFieldNoCopy(app.Object, "operation"); found && operation != nil { + return "" + } + state, found, _ := unstructured.NestedMap(app.Object, "status", "operationState") + if phase, _, _ := unstructured.NestedString(state, "phase"); !found || + (phase != argoPhaseRunning && phase != argoPhaseTerminating) { + return "" + } + encoded, err := json.Marshal(state) + if err != nil { + return "" + } + return string(encoded) +} + +// Progress reports how far Argo CD is with the sync s asked for; the apply window closes once it +// completed. The operation state is this request's when its info carries the request's token. It +// is Running while its phase is Running (a failed sync waiting for Argo CD's own retry included, +// which is Running with a finishedAt) or Terminating, Succeeded once Succeeded, and Failed once +// Failed or Error. Until the controller starts it, the operation is still set with the token, and +// Requested. One Running or Terminating whose .operation somebody removed is Failed once Overdue: +// Argo CD processes it no further, and by then any controller pass in flight has written. An operation that is gone with no state of its own (somebody cleared it, or a later +// sync replaced its record) is Failed: nothing of it will run, and the worker decides again. +// +// The token is nowhere on an Unconfirmed or Recovered operation whose update was never made, and +// on one whose update is still on its way. The controller moves an operation it starts into +// status.operationState whole, its info included (NewOperationState, controller/sync.go), and +// clears operation only once the state completed (setOperationState, controller/appcontroller.go), +// so the token is not lost on the way. Such an operation is settled (settleUnseen). +func (a *ArgoCD) Progress(ctx context.Context, ref types.NamespacedName, s *Started) (Progress, error) { if s.Outcome != OutcomeSyncRequested { - return true, nil + return Progress{Outcome: git.ApplySucceeded}, nil } app, err := a.client.Resource(applicationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { - return false, fmt.Errorf("get %s %s: %w", kindApplication, ref, err) + return Progress{}, fmt.Errorf("get %s %s: %w", kindApplication, ref, err) + } + if progress, found := syncProgress(app, s.Token); found { + if progress.Outcome == git.ApplyRunning && s.Overdue && abandonedOperation(app) != "" { + return Progress{Outcome: git.ApplyFailed, Message: "the sync's .operation was removed while it ran, " + + "so Argo CD processes it no further: " + statusString(app, "operationState", "message")}, nil + } + return progress, nil + } + if s.Recovered || s.Unconfirmed != nil { + return a.settleUnseen(ctx, ref, app, s) } + return Progress{ + Outcome: git.ApplyFailed, + Message: "the sync operation is gone from the Application before it recorded an outcome", + }, nil +} + +// syncProgress reads the progress of the sync with token from the Application, and reports false +// when neither its operation nor its operation state carries the token. +func syncProgress(app *unstructured.Unstructured, token string) (Progress, bool) { if state, found, _ := unstructured.NestedMap(app.Object, "status", "operationState"); found && - operationToken(state["operation"]) == s.Token { + operationToken(state["operation"]) == token { phase, _, _ := unstructured.NestedString(state, "phase") - return phase != "" && phase != argoPhaseRunning && phase != argoPhaseTerminating, nil + message, _, _ := unstructured.NestedString(state, "message") + applied, _, _ := unstructured.NestedString(state, "syncResult", "revision") + switch phase { + case argoPhaseSucceeded: + return Progress{Outcome: git.ApplySucceeded, AppliedRevision: applied}, true + case argoPhaseFailed, argoPhaseError: + return Progress{ + Outcome: git.ApplyFailed, + Message: fmt.Sprintf("the sync ended %s: %s", phase, message), + AppliedRevision: applied, + }, true + default: + return Progress{Outcome: git.ApplyRunning}, true + } } - operation, found, _ := unstructured.NestedMap(app.Object, "operation") - return !found || operationToken(operation) != s.Token, nil + if operation, found, _ := unstructured.NestedMap(app.Object, "operation"); found && + operationToken(operation) == token { + return Progress{Outcome: git.ApplyRequested}, true + } + return Progress{}, false +} + +// settleUnseen decides an operation whose update may have been sent without an answer (Unconfirmed, +// or Recovered from a record written before the update), as app, read without its token, shows it. +// The update carried the resourceVersion it was made from, so it can land only while the +// Application is still at that version. settleUnseen moves the Application on from the version it +// read, with a write guarded by it that asks Argo CD for a refresh. Once that write is made, the +// update was not made before it and can never be made after it: Failed, and the worker asks again. +// No deadline of the API server's is assumed. A write that conflicts read a version the update may +// have landed after, and the next Progress reads again. A refresh already pending is left to the +// controller, whose write that consumes it moves the Application on as well. +func (a *ArgoCD) settleUnseen( + ctx context.Context, ref types.NamespacedName, app *unstructured.Unstructured, s *Started, +) (Progress, error) { + annotations := app.GetAnnotations() + if _, pending := annotations[argoRefreshAnnotation]; pending { + return Progress{Outcome: git.ApplyRequested}, nil + } + if annotations == nil { + annotations = map[string]string{} + } + annotations[argoRefreshAnnotation] = argoRefreshNormal + update := app.DeepCopy() + update.SetAnnotations(annotations) + _, err := a.client.Resource(applicationGVR()).Namespace(ref.Namespace).Update(ctx, update, metav1.UpdateOptions{}) + switch { + case apierrors.IsConflict(err): + return Progress{Outcome: git.ApplyRequested}, nil + case err != nil: + return Progress{}, fmt.Errorf("move %s %s on from the version an unanswered update was made from: %w", + kindApplication, ref, err) + } + message := fmt.Sprintf("the sync operation is not on the Application, and the update that would have set it "+ + "was never made (%v)", s.Unconfirmed) + if s.Unconfirmed == nil { + message = "recovered after a restart, and the update that would have set the sync operation was never made" + } + return Progress{Outcome: git.ApplyFailed, Message: message}, nil } -// Validate answers ApplierConfigured for an Application: that it exists in the cluster and its -// kind is served there, that it applies want from a single source, that automated sync is off, -// and that this identity may update it, since a request is an update. -func (a *ArgoCD) Validate(ctx context.Context, ref types.NamespacedName, want Binding) Verdict { +// Validate answers Configured for an ArgoCDTrigger's Application: that it exists in its cluster and +// its kind is served there, that it syncs a single source, that automated sync is off, and that this +// identity may update it, since a request is an update. Which repository and revision it syncs, it +// reports in the verdict's Source, for the GitTargets naming the trigger to check against what they +// write. +func (a *ArgoCD) Validate(ctx context.Context, ref types.NamespacedName) Verdict { app, err := a.client.Resource(applicationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { return readFailure("Argo CD", kindApplication, applicationGVR(), ref, err) } - if msg := applicationMismatch(app, ref, want); msg != "" { - return Verdict{Reason: ReasonSourceMismatch, Message: msg} + if msg := applicationUnsupported(app, ref); msg != "" { + return Verdict{Reason: ReasonUnsupportedSource, Message: msg} } + url, _, _ := unstructured.NestedString(app.Object, "spec", "source", "repoURL") + revision, _, _ := unstructured.NestedString(app.Object, "spec", "source", "targetRevision") + read := &Source{URL: url, Revision: revision} if automatedSyncEnabled(app) { return Verdict{ Reason: ReasonAutomatedSyncEnabled, Message: fmt.Sprintf("%s %s %s", kindApplication, ref, automatedSyncMessage), + Source: read, } } if verdict := accessVerdict(ctx, a.CanI, kindApplication, applicationGVR(), ref, "update"); verdict != nil { + verdict.Source = read return *verdict } return Verdict{ OK: true, Reason: ReasonSucceeded, - Message: fmt.Sprintf("%s %s syncs branch %s, with automated sync off", - kindApplication, ref, want.Branch), + Message: fmt.Sprintf("%s %s syncs revision %q of %s, with automated sync off", + kindApplication, ref, revision, url), + Source: read, } } @@ -369,18 +530,10 @@ func automatedPrune(app *unstructured.Unstructured) bool { // single source, or "" when it does. targetRevision must name the branch, as its name or as // refs/heads/: HEAD, a tag or a commit would never follow the branch GitOps Reverser writes. func applicationMismatch(app *unstructured.Unstructured, ref types.NamespacedName, want Binding) string { - if sources, _, _ := unstructured.NestedSlice(app.Object, "spec", "sources"); len(sources) > 0 { - return fmt.Sprintf("%s %s has spec.sources; GitOps Reverser syncs only a single-source Application", - kindApplication, ref) - } - if _, found, _ := unstructured.NestedFieldNoCopy(app.Object, "spec", "sourceHydrator"); found { - return fmt.Sprintf("%s %s has spec.sourceHydrator; GitOps Reverser syncs only an Application "+ - "whose spec.source it writes", kindApplication, ref) - } - source, found, _ := unstructured.NestedMap(app.Object, "spec", "source") - if !found { - return fmt.Sprintf("%s %s names no spec.source", kindApplication, ref) + if msg := applicationUnsupported(app, ref); msg != "" { + return msg } + source, _, _ := unstructured.NestedMap(app.Object, "spec", "source") url, _, _ := unstructured.NestedString(source, "repoURL") if !repourl.Same(url, want.RepositoryURL) { return fmt.Sprintf("%s %s syncs %s, not %s", kindApplication, ref, url, want.RepositoryURL) @@ -394,3 +547,20 @@ func applicationMismatch(app *unstructured.Unstructured, ref types.NamespacedNam } return "" } + +// applicationUnsupported returns why an Application has no single source GitOps Reverser can sync, +// or "" when it has one. +func applicationUnsupported(app *unstructured.Unstructured, ref types.NamespacedName) string { + if sources, _, _ := unstructured.NestedSlice(app.Object, "spec", "sources"); len(sources) > 0 { + return fmt.Sprintf("%s %s has spec.sources; GitOps Reverser syncs only a single-source Application", + kindApplication, ref) + } + if _, found, _ := unstructured.NestedFieldNoCopy(app.Object, "spec", "sourceHydrator"); found { + return fmt.Sprintf("%s %s has spec.sourceHydrator; GitOps Reverser syncs only an Application "+ + "whose spec.source it writes", kindApplication, ref) + } + if _, found, _ := unstructured.NestedMap(app.Object, "spec", "source"); !found { + return fmt.Sprintf("%s %s names no spec.source", kindApplication, ref) + } + return "" +} diff --git a/internal/applier/argocd_guide_test.go b/internal/applier/argocd_guide_test.go index 6dc75cf5d..db6396ee7 100644 --- a/internal/applier/argocd_guide_test.go +++ b/internal/applier/argocd_guide_test.go @@ -18,22 +18,25 @@ import ( // argoCDGuide is the Argo CD user guide. Its "A complete example" section is the configuration // this file holds to the code: a reader copies it, so it must be accepted as written. -const argoCDGuide = "../../docs/argocd.md" +const argoCDGuide = "../../docs/bi-directional-argocd.md" // TestArgoCDGuide_TheCompleteExampleIsAccepted reads the guide's complete example and checks it // against the code that validates and drives an Argo CD applier: the GitOps Reverser objects decode -// into the API types with no unknown field, ApplierConfigured comes out True for the Application as -// written, and a request sets a sync pinned to the commit. It also holds the example to the guide's +// into the API types with no unknown field, Configured comes out True for the Application as +// written, the trigger's source binds the GitTarget, and a request sets a sync pinned to the commit. It also holds the example to the guide's // own advice: automated sync off, prune kept, and a recursive directory source. func TestArgoCDGuide_TheCompleteExampleIsAccepted(t *testing.T) { objects := guideExampleObjects(t, argoCDGuide) var provider v1alpha3.GitProvider var target v1alpha3.GitTarget + var trigger v1alpha3.ArgoCDTrigger var secret corev1.Secret decodeStrict(t, objects, "GitProvider", &provider) decodeStrict(t, objects, "GitTarget", &target) + decodeStrict(t, objects, v1alpha3.ApplyTriggerKindArgoCD, &trigger) decodeStrict(t, objects, "Secret", &secret) + exampleReachesItsApplier(t, objects, &trigger) app := one(t, objects, kindApplication) // The GitOps Reverser objects are wired to each other. @@ -42,9 +45,10 @@ func TestArgoCDGuide_TheCompleteExampleIsAccepted(t *testing.T) { require.NotNil(t, provider.Spec.Receiver, "the example serves a push receiver") assert.Equal(t, secret.Name, provider.Spec.Receiver.SecretRef.Name) assert.NotEmpty(t, secret.StringData[v1alpha3.ReceiverTokenKey], "the receiver reads the token key") - require.NotNil(t, target.Spec.Applier) - assert.Equal(t, v1alpha3.ApplierKindApplication, target.Spec.Applier.Kind()) - assert.Nil(t, target.Spec.Applier.QuietPeriod, "quietPeriod is rejected with an applicationRef") + require.NotNil(t, target.Spec.ApplyTriggerRef) + assert.Equal(t, v1alpha3.ApplyTriggerReference{Kind: v1alpha3.ApplyTriggerKindArgoCD, Name: trigger.Name}, + *target.Spec.ApplyTriggerRef) + assert.Equal(t, target.Namespace, trigger.Namespace, "a GitTarget names a trigger in its own namespace") assert.Equal(t, v1alpha3.RefusalActionReapply, target.Spec.OnRefusal) // The Application follows the guide's advice. @@ -53,19 +57,25 @@ func TestArgoCDGuide_TheCompleteExampleIsAccepted(t *testing.T) { recurse, _, _ := unstructured.NestedBool(app.Object, "spec", "source", "directory", "recurse") assert.True(t, recurse, "GitOps Reverser writes nested folders") - // The applier, as written, is accepted and driven. - ref := types.NamespacedName{ - Namespace: target.Spec.Applier.ApplicationRef.Namespace, Name: target.Spec.Applier.ApplicationRef.Name, - } + // The applier, as written, is accepted, binds the GitTarget, and is driven. + ref := trigger.Applier() + assert.Equal(t, types.NamespacedName{Namespace: app.GetNamespace(), Name: app.GetName()}, ref) want := Binding{RepositoryURL: provider.Spec.URL, Branch: target.Spec.Branch} client := newSourceClient(app) argo := pinnedArgo(client) - verdict := argo.Validate(context.Background(), ref, want) - assert.True(t, verdict.OK, "ApplierConfigured for the guide's example: %s: %s", verdict.Reason, verdict.Message) + verdict := argo.Validate(context.Background(), ref) + assert.True(t, verdict.OK, "Configured for the guide's example: %s: %s", verdict.Reason, verdict.Message) + require.NotNil(t, verdict.Source) + trigger.Status.Source = &v1alpha3.ArgoCDTriggerSource{ + RepoURL: verdict.Source.URL, TargetRevision: verdict.Source.Revision, + } + assert.Empty(t, BindingMismatch(&trigger, want), "the trigger's Application syncs what the GitTarget writes") - stamped, err := argo.Request(context.Background(), ref, want, testCommit, v1alpha3.ApplyReasonForeignCommit, nil) + started, err := argo.Request(context.Background(), ref, Ask{ + Want: want, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, Token: requestToken(stampTime), + }) require.NoError(t, err) - assert.Equal(t, OutcomeSyncRequested, stamped.Outcome) + assert.Equal(t, OutcomeSyncRequested, started.Outcome) written, err := client.Resource(applicationGVR()).Namespace(ref.Namespace). Get(context.Background(), ref.Name, metav1.GetOptions{}) require.NoError(t, err) diff --git a/internal/applier/argocd_test.go b/internal/applier/argocd_test.go index ca806346e..fb79e2b6c 100644 --- a/internal/applier/argocd_test.go +++ b/internal/applier/argocd_test.go @@ -5,6 +5,7 @@ package applier import ( "context" "errors" + "strconv" "sync/atomic" "testing" "time" @@ -21,6 +22,7 @@ import ( k8stesting "k8s.io/client-go/testing" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" ) const argoNS = "argocd" @@ -144,11 +146,16 @@ func TestArgoCDRequest_TheOperationIsWhatAutomatedSyncWouldBuild(t *testing.T) { t.Run(tt.name, func(t *testing.T) { client := newArgoSource(applicationObj(testRepoURL, testBranch, tt.syncPolicy)) - stamped, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + started, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.NoError(t, err) - assert.Equal(t, Stamped{Outcome: OutcomeSyncRequested, Token: requestToken(stampTime)}, stamped) + assert.Equal(t, Started{Outcome: OutcomeSyncRequested, Token: requestToken(stampTime), Commit: testCommit}, + started) operation, found, _ := unstructured.NestedMap(readApp(t, client).Object, "operation") require.True(t, found) assert.Equal(t, map[string]any{ @@ -174,8 +181,9 @@ func TestArgoCDRequest_ClearsTheLastOperationState(t *testing.T) { _ = unstructured.SetNestedField(app.Object, "Synced", "status", "sync", "status") client := newArgoSource(app) - _, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonRefusal, nil) + _, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonRefusal, Token: requestToken(stampTime), + }) require.NoError(t, err) written := readApp(t, client) @@ -198,8 +206,12 @@ func TestArgoCDRequest_ARunningOperationIsNeverReplaced(t *testing.T) { t.Run("with no time to wait, the attempt fails and writes nothing", func(t *testing.T) { client := newArgoSource(running()) - _, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + _, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.ErrorContains(t, err, "has an operation in progress") assert.False(t, isPermanent(err), "the operation ends; asking again later can succeed") @@ -221,12 +233,16 @@ func TestArgoCDRequest_ARunningOperationIsNeverReplaced(t *testing.T) { argo := NewArgoCD(client) argo.OperationWait, argo.PollInterval = 5*time.Second, time.Millisecond - stamped, err := argo.Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + started, err := argo.Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.NoError(t, err) - assert.Equal(t, OutcomeSyncRequested, stamped.Outcome) - assert.Equal(t, stamped.Token, operationToken(readApp(t, client).Object["operation"])) + assert.Equal(t, OutcomeSyncRequested, started.Outcome) + assert.Equal(t, started.Token, operationToken(readApp(t, client).Object["operation"])) }) t.Run("an operation set between the read and the update makes the update fail, not replace it", @@ -242,22 +258,33 @@ func TestArgoCDRequest_ARunningOperationIsNeverReplaced(t *testing.T) { errors.New("the object has been modified")) }) - _, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + _, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.ErrorContains(t, err, "has an operation in progress") assert.Equal(t, theirOperation(), readApp(t, client).Object["operation"], "theirs is untouched") }) t.Run("a retry pending on a failed operation still counts as running", func(t *testing.T) { + // Argo CD keeps .operation set until the state completed (setOperationState), and processes + // an operation only while it is set: a retry waiting for its backoff keeps it. app := manualApp() + _ = unstructured.SetNestedMap(app.Object, theirOperation(), "operation") _ = unstructured.SetNestedMap(app.Object, map[string]any{ "phase": "Running", "finishedAt": "2026-10-20T10:14:00Z", "operation": theirOperation(), }, "status", "operationState") client := newArgoSource(app) - _, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + _, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.ErrorContains(t, err, "has an operation in progress") assert.Zero(t, updates(client)) @@ -278,14 +305,256 @@ func TestArgoCDRequest_AConflictWithAStatusWriteIsReadAgain(t *testing.T) { return false, nil, nil }) - stamped, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + started, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.NoError(t, err) - assert.Equal(t, OutcomeSyncRequested, stamped.Outcome) + assert.Equal(t, OutcomeSyncRequested, started.Outcome) assert.Equal(t, 2, updates(client)) } +// guardResourceVersions makes the fake's Application updates what the API server's are +// (guardedUpdate). The fake tracker checks no resourceVersion. +func guardResourceVersions(t *testing.T, client *dynamicfake.FakeDynamicClient) { + t.Helper() + client.PrependReactor("update", "applications", func(action k8stesting.Action) (bool, runtime.Object, error) { + update, _ := action.(k8stesting.UpdateActionImpl) + obj, err := guardedUpdate(client, update.GetObject().(*unstructured.Unstructured)) + return true, obj, err + }) +} + +// guardedUpdate stores an Application update as the API server would: one whose resourceVersion is +// not the stored one is a Conflict, and one that is made moves the version on. It writes the +// tracker directly, so a reactor can call it. +func guardedUpdate(client *dynamicfake.FakeDynamicClient, obj *unstructured.Unstructured) (runtime.Object, error) { + stored, err := client.Tracker().Get(applicationGVR(), obj.GetNamespace(), obj.GetName()) + if err != nil { + return nil, err + } + was := stored.(*unstructured.Unstructured).GetResourceVersion() + if obj.GetResourceVersion() != was { + return nil, apierrors.NewConflict(applicationGVR().GroupResource(), obj.GetName(), + errors.New("the object has been modified")) + } + version, _ := strconv.Atoi(was) + obj = obj.DeepCopy() + obj.SetResourceVersion(strconv.Itoa(version + 1)) + if err := client.Tracker().Update(applicationGVR(), obj, obj.GetNamespace()); err != nil { + return nil, err + } + return obj, nil +} + +// TestArgoCDRequest_AnUnansweredUpdateIsFollowed is review 5's first finding: an update whose +// answer was lost (a timeout, a reset connection, a 504) may have set the operation all the same, +// and a RequestFailed withdrew its record and closed the apply window while Argo CD synced. Now it +// is an Unconfirmed operation, followed by its token: one that was set is followed until the sync +// completes. One whose token is not seen is settled with a write guarded by the version read: once +// it is made, the update can never land, and the operation is Failed, so the worker asks again. No +// deadline of the API server's is assumed. An update the API server refused is still an error. +func TestArgoCDRequest_AnUnansweredUpdateIsFollowed(t *testing.T) { + request := func(t *testing.T, argo *ArgoCD) Started { + t.Helper() + started, err := argo.Request(context.Background(), platformApp, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) + require.NoError(t, err) + require.Error(t, started.Unconfirmed) + assert.Equal(t, []any{OutcomeSyncRequested, requestToken(stampTime), testCommit}, + []any{started.Outcome, started.Token, started.Commit}) + return started + } + progress := func(t *testing.T, argo *ArgoCD, started *Started) Progress { + t.Helper() + got, err := argo.Progress(context.Background(), platformApp, started) + require.NoError(t, err) + return got + } + // unanswered answers the request's update with answer, having made it when landed, and keeps it + // back otherwise: the returned func makes it then, as the API server would, at the version it was + // made from. + unanswered := func(client *dynamicfake.FakeDynamicClient, landed bool, answer error) func() error { + var answered atomic.Bool + var held *unstructured.Unstructured + client.PrependReactor("update", "applications", func(action k8stesting.Action) (bool, runtime.Object, error) { + if answered.Swap(true) { + return false, nil, nil + } + update, _ := action.(k8stesting.UpdateActionImpl) + held = update.GetObject().(*unstructured.Unstructured).DeepCopy() + if landed { + _, err := guardedUpdate(client, held) + require.NoError(t, err) + } + return true, nil, answer + }) + return func() error { + _, err := guardedUpdate(client, held) + return err + } + } + + t.Run("set, then the answer was lost: followed until the sync completes", func(t *testing.T) { + for name, answer := range map[string]error{ + "timed out": context.DeadlineExceeded, + "a 504": apierrors.NewTimeoutError("slow", 1), + "a 500": apierrors.NewInternalError(errors.New("etcd leader changed")), + "a lost answer": errors.New("connection reset by peer"), + } { + t.Run(name, func(t *testing.T) { + client := newArgoSource(manualApp()) + guardResourceVersions(t, client) + unanswered(client, true, answer) + argo := pinnedArgo(client) + started := request(t, argo) + ours, _, _ := unstructured.NestedMap(readApp(t, client).Object, "operation") + require.Equal(t, started.Token, operationToken(ours), "the update was made") + + assert.Equal(t, git.ApplyRequested, progress(t, argo, &started).Outcome, "set, not started") + app := readApp(t, client) + _ = unstructured.SetNestedMap(app.Object, map[string]any{"operation": ours, "phase": "Running"}, + "status", "operationState") + writeApp(t, client, app) + assert.Equal(t, git.ApplyRunning, progress(t, argo, &started).Outcome) + app = readApp(t, client) + unstructured.RemoveNestedField(app.Object, "operation") + _ = unstructured.SetNestedField(app.Object, "Succeeded", "status", "operationState", "phase") + writeApp(t, client, app) + assert.Equal(t, git.ApplySucceeded, progress(t, argo, &started).Outcome) + assert.NotContains(t, readApp(t, client).GetAnnotations(), argoRefreshAnnotation, + "an update seen to land is never settled") + }) + } + }) + + t.Run("not set: Failed once a write guarded by the version read lands, and it never can", func(t *testing.T) { + client := newArgoSource(manualApp()) + guardResourceVersions(t, client) + land := unanswered(client, false, apierrors.NewTimeoutError("slow", 1)) + argo := pinnedArgo(client) + started := request(t, argo) + + failed := progress(t, argo, &started) + + assert.Equal(t, git.ApplyFailed, failed.Outcome) + assert.Contains(t, failed.Message, "was never made") + assert.Equal(t, argoRefreshNormal, readApp(t, client).GetAnnotations()[argoRefreshAnnotation], + "the write asks Argo CD for a refresh, which only compares") + require.True(t, apierrors.IsConflict(land()), "the update can no longer land") + assert.NotContains(t, readApp(t, client).Object, "operation") + }) + + t.Run("not set when read, landed before the settling write: read again, and followed", func(t *testing.T) { + client := newArgoSource(manualApp()) + guardResourceVersions(t, client) + land := unanswered(client, false, context.DeadlineExceeded) + argo := pinnedArgo(client) + started := request(t, argo) + client.PrependReactor("get", "applications", func(k8stesting.Action) (bool, runtime.Object, error) { + stored, err := client.Tracker().Get(applicationGVR(), argoNS, "platform") + require.NoError(t, err) + require.NoError(t, land(), "the update lands right after Progress read the Application") + return true, stored, nil + }) + + assert.Equal(t, git.ApplyRequested, progress(t, argo, &started).Outcome, + "the settling write conflicts: the update may have landed") + client.ReactionChain = client.ReactionChain[1:] + assert.Equal(t, git.ApplyRequested, progress(t, argo, &started).Outcome, "set, not started") + assert.NotContains(t, readApp(t, client).GetAnnotations(), argoRefreshAnnotation) + }) + + t.Run("not set, with a refresh pending: left to the controller's write, which moves it on", func(t *testing.T) { + app := manualApp() + app.SetAnnotations(map[string]string{argoRefreshAnnotation: "hard"}) + client := newArgoSource(app) + guardResourceVersions(t, client) + unanswered(client, false, context.DeadlineExceeded) + argo := pinnedArgo(client) + started := request(t, argo) + before := updates(client) + + assert.Equal(t, git.ApplyRequested, progress(t, argo, &started).Outcome) + assert.Equal(t, before, updates(client), "a pending refresh is not overwritten") + app = readApp(t, client) + app.SetAnnotations(nil) + writeApp(t, client, app) // the controller consumed it + assert.Equal(t, git.ApplyFailed, progress(t, argo, &started).Outcome) + }) + + t.Run("refused: an error, and nothing to follow", func(t *testing.T) { + client := newArgoSource(manualApp()) + unanswered(client, false, apierrors.NewForbidden(applicationGVR().GroupResource(), "platform", + errors.New("no"))) + + _, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) + + require.ErrorContains(t, err, "set the sync operation") + assert.False(t, isPermanent(err)) + }) +} + +// TestArgoCD_ARecoveredSyncIsSettledFromItsRecordAlone is review 5's P1 after the fix above: a +// process that recorded the operation, sent the update, lost the answer and stopped leaves nothing +// but the record. The next one rebuilds the operation from that record alone, reads the Application +// without the token, and settles it as an Unconfirmed one is: the update it may have sent can no +// longer land, so the window it would have needed is never needed. +func TestArgoCD_ARecoveredSyncIsSettledFromItsRecordAlone(t *testing.T) { + client := newArgoSource(manualApp()) + guardResourceVersions(t, client) + app := readApp(t, client) + _ = unstructured.SetNestedField(app.Object, syncOperation(app, testCommit, v1alpha3.ApplyReasonForeignCommit, + requestToken(stampTime)), "operation") + sentBeforeTheRestart := app // made from version 1, its answer lost, still on its way + + started := recoveredStarted(git.ApplyRequest{ + Applier: git.ApplierRef{Kind: v1alpha3.ApplierKindApplication}, + Recovered: &v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationRequested, Reason: v1alpha3.ApplyReasonForeignCommit, + Commit: testCommit, Token: requestToken(stampTime), RequestedAt: metav1.NewTime(stampTime), + }, + }) + failed, err := pinnedArgo(client).Progress(context.Background(), platformApp, &started) + + require.NoError(t, err) + assert.Equal(t, git.ApplyFailed, failed.Outcome) + assert.Contains(t, failed.Message, "recovered after a restart") + _, err = guardedUpdate(client, sentBeforeTheRestart) + require.True(t, apierrors.IsConflict(err), "the update sent before the restart can no longer land") + assert.NotContains(t, readApp(t, client).Object, "operation") +} + +// TestArgoCDRequest_ADecisionThatGoesStaleWhileRecordedSyncsNothing is review 5's sixth finding: +// recording the operation can wait on the status writer and the API server, and the branch can +// move on meanwhile. The decision is asked again after the record, just before the update. +func TestArgoCDRequest_ADecisionThatGoesStaleWhileRecordedSyncsNothing(t *testing.T) { + client := newArgoSource(manualApp()) + holds, recorded := true, false + + _, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + Current: func() bool { return holds }, + Record: func(context.Context) error { + holds, recorded = false, true + return nil + }, + }) + + require.ErrorIs(t, err, errStale) + assert.True(t, recorded, "the worker withdraws what was recorded") + assert.Zero(t, updates(client), "nothing is set") +} + // TestArgoCDRequest_RefusesWhatIsNotUnderControl: a published verdict can be older than the // Application, so a request checks the source and automated sync again, and writes nothing on // either. A stale decision writes nothing either, and an empty commit has nothing to pin. @@ -313,8 +582,13 @@ func TestArgoCDRequest_RefusesWhatIsNotUnderControl(t *testing.T) { t.Run(tt.name, func(t *testing.T) { client := newArgoSource(tt.app) - stamped, err := pinnedArgo(client).Request(context.Background(), platformApp, testBinding, tt.commit, - v1alpha3.ApplyReasonForeignCommit, tt.current) + started, err := pinnedArgo(client).Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: tt.commit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + Current: tt.current, + }) switch { case tt.wantErr != "": @@ -324,14 +598,14 @@ func TestArgoCDRequest_RefusesWhatIsNotUnderControl(t *testing.T) { require.ErrorIs(t, err, errStale) default: require.NoError(t, err) - assert.Equal(t, tt.wantOutcome, stamped.Outcome) + assert.Equal(t, tt.wantOutcome, started.Outcome) } assert.Zero(t, updates(client), "nothing is written") }) } } -// TestArgoCDValidate_Reasons covers every ApplierConfigured answer for an Application. +// TestArgoCDValidate_Reasons covers every Configured answer for an Application. func TestArgoCDValidate_Reasons(t *testing.T) { sources := map[string]any{"sources": []any{map[string]any{"repoURL": testRepoURL}}} tests := []struct { @@ -345,7 +619,8 @@ func TestArgoCDValidate_Reasons(t *testing.T) { wantErr bool }{ {name: "configured", objs: []runtime.Object{manualApp()}, wantReason: ReasonSucceeded, - wantMessage: "Application argocd/platform syncs branch main, with automated sync off"}, + wantMessage: `Application argocd/platform syncs revision "main" of ` + testRepoURL + + ", with automated sync off"}, {name: "configured, as refs/heads/ names the branch", objs: []runtime.Object{applicationObj(testRepoURL, "refs/heads/main", nil)}, wantReason: ReasonSucceeded}, {name: "configured, with the off switch that keeps prune", @@ -364,33 +639,25 @@ func TestArgoCDValidate_Reasons(t *testing.T) { wantReason: ReasonApplierNotFound, wantMessage: "does not serve Application (argoproj.io/v1alpha1); is Argo CD installed there?", }, - {name: "another repository", - objs: []runtime.Object{applicationObj("https://github.com/acme/other", testBranch, nil)}, - wantReason: ReasonSourceMismatch, wantMessage: "syncs https://github.com/acme/other, not"}, - {name: "another branch", objs: []runtime.Object{applicationObj(testRepoURL, "release", nil)}, - wantReason: ReasonSourceMismatch, wantMessage: `targets revision "release", not branch main`}, - {name: "HEAD", objs: []runtime.Object{applicationObj(testRepoURL, "HEAD", nil)}, - wantReason: ReasonSourceMismatch, wantMessage: `targets revision "HEAD"`}, - {name: "no targetRevision, which is HEAD", objs: []runtime.Object{applicationObj(testRepoURL, "", nil)}, - wantReason: ReasonSourceMismatch, wantMessage: `targets revision "HEAD"`}, - {name: "a tag", objs: []runtime.Object{applicationObj(testRepoURL, "v1.0.0", nil)}, - wantReason: ReasonSourceMismatch, wantMessage: `targets revision "v1.0.0"`}, + {name: "another repository and revision, which only a GitTarget can judge", + objs: []runtime.Object{applicationObj("https://github.com/acme/other", "v1.0.0", nil)}, + wantReason: ReasonSucceeded, wantMessage: `syncs revision "v1.0.0" of https://github.com/acme/other`}, {name: "multi-source", objs: []runtime.Object{manualApp()}, mutate: func(app *unstructured.Unstructured) { unstructured.RemoveNestedField(app.Object, "spec", "source") _ = unstructured.SetNestedField(app.Object, sources["sources"], "spec", "sources") }, - wantReason: ReasonSourceMismatch, wantMessage: "syncs only a single-source Application"}, + wantReason: ReasonUnsupportedSource, wantMessage: "syncs only a single-source Application"}, {name: "a source hydrator", objs: []runtime.Object{manualApp()}, mutate: func(app *unstructured.Unstructured) { _ = unstructured.SetNestedMap(app.Object, map[string]any{}, "spec", "sourceHydrator") }, - wantReason: ReasonSourceMismatch, wantMessage: "has spec.sourceHydrator"}, + wantReason: ReasonUnsupportedSource, wantMessage: "has spec.sourceHydrator"}, { name: "no source", objs: []runtime.Object{manualApp()}, mutate: func(app *unstructured.Unstructured) { unstructured.RemoveNestedField(app.Object, "spec", "source") }, - wantReason: ReasonSourceMismatch, + wantReason: ReasonUnsupportedSource, wantMessage: "names no spec.source", }, {name: "automated sync on, as an empty block turns it on", @@ -433,7 +700,7 @@ func TestArgoCDValidate_Reasons(t *testing.T) { argo.CanI = tt.canI } - verdict := argo.Validate(context.Background(), platformApp, testBinding) + verdict := argo.Validate(context.Background(), platformApp) assert.Equal(t, tt.wantReason, verdict.Reason, verdict.Message) assert.Equal(t, tt.wantReason == ReasonSucceeded, verdict.OK) @@ -453,20 +720,26 @@ func TestArgoCDValidate_AsksForUpdate(t *testing.T) { return true, nil } - require.True(t, argo.Validate(context.Background(), platformApp, testBinding).OK) + require.True(t, argo.Validate(context.Background(), platformApp).OK) assert.Equal(t, []authorizationv1.ResourceAttributes{{ Namespace: argoNS, Verb: "update", Group: "argoproj.io", Version: "v1alpha1", Resource: "applications", Name: "platform", }}, asked) } -// TestArgoCD_Handled pins when an Argo CD apply window closes: when this request's operation has -// completed, whatever its phase, and not while it runs or retries, nor on another operation's state. -func TestArgoCD_Handled(t *testing.T) { +// TestArgoCD_Progress pins how an Argo CD operation is followed: Requested while this request's +// operation is set and not started, Running while its state runs (or waits for Argo CD's own retry), +// and complete once Succeeded, Failed or Error, which is when the apply window closes. An operation +// cleared before it recorded an outcome ran nothing, and is Failed so the worker asks again. +func TestArgoCD_Progress(t *testing.T) { client := newArgoSource(manualApp()) argo := pinnedArgo(client) - stamped, err := argo.Request(context.Background(), platformApp, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + started, err := argo.Request(context.Background(), platformApp, Ask{ + Want: testBinding, + Commit: testCommit, + Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) require.NoError(t, err) ours, _, _ := unstructured.NestedMap(readApp(t, client).Object, "operation") @@ -475,42 +748,128 @@ func TestArgoCD_Handled(t *testing.T) { app := readApp(t, client) _ = unstructured.SetNestedMap(app.Object, map[string]any{ "operation": operation, "phase": phase, "finishedAt": "2026-10-20T10:15:00Z", + "message": "one or more objects failed to apply", }, "status", "operationState") if !keepOperation { unstructured.RemoveNestedField(app.Object, "operation") } writeApp(t, client, app) } - handled := func(t *testing.T) bool { + progress := func(t *testing.T) Progress { t.Helper() - ok, err := argo.Handled(context.Background(), platformApp, stamped) + got, err := argo.Progress(context.Background(), platformApp, &started) require.NoError(t, err) - return ok + return got } - assert.False(t, handled(t), "set, not started") + assert.Equal(t, git.ApplyRequested, progress(t).Outcome, "set, not started") setState(t, theirOperation(), "Succeeded", true) - assert.False(t, handled(t), "another operation's state is not ours, and ours is still set") + assert.Equal(t, git.ApplyRequested, progress(t).Outcome, + "another operation's state is not ours, and ours is still set") setState(t, ours, "Running", true) - assert.False(t, handled(t), "running, or a failed sync waiting for its retry (Running with a finishedAt)") - setState(t, ours, "Failed", false) - assert.True(t, handled(t), "a failed sync closes the window") + assert.Equal(t, git.ApplyRunning, progress(t).Outcome, + "running, or a failed sync waiting for its retry (Running with a finishedAt)") + setState(t, ours, "Terminating", true) + assert.Equal(t, git.ApplyRunning, progress(t).Outcome) + for _, phase := range []string{"Failed", "Error"} { + setState(t, ours, phase, false) + failed := progress(t) + assert.Equal(t, git.ApplyFailed, failed.Outcome, phase) + assert.Equal(t, "the sync ended "+phase+": one or more objects failed to apply", failed.Message) + } setState(t, ours, "Succeeded", false) - assert.True(t, handled(t)) + assert.Equal(t, Progress{Outcome: git.ApplySucceeded}, progress(t)) setState(t, theirOperation(), "Succeeded", false) - assert.True(t, handled(t), "ours was cleared before it ran: nothing of it will apply") + assert.Equal(t, git.ApplyFailed, progress(t).Outcome, "ours was cleared before it recorded an outcome") - assert.True(t, func() bool { - ok, err := argo.Handled(context.Background(), platformApp, Stamped{Outcome: OutcomeCurrent}) + current, err := argo.Progress(context.Background(), platformApp, &Started{Outcome: OutcomeCurrent}) + require.NoError(t, err) + assert.Equal(t, git.ApplySucceeded, current.Outcome, "nothing asked is nothing to wait for") +} + +// TestArgoCD_ASyncWhoseOperationWasRemovedEnds is review 7's A1. Removing .operation is the +// community workaround for a stuck hook: Argo CD then processes the Application no further, and +// its operationState stays Running. GitOps Reverser followed such a sync forever, and refused every +// later request as "in progress". Now our sync, Running with .operation gone, is Failed once it is +// overdue (by then any controller pass in flight has written), and a request finds such a state +// not in progress once it stayed unchanged for the whole wait. +func TestArgoCD_ASyncWhoseOperationWasRemovedEnds(t *testing.T) { + orphaned := func(t *testing.T, client *dynamicfake.FakeDynamicClient, operation map[string]any) { + t.Helper() + app := readApp(t, client) + _ = unstructured.SetNestedMap(app.Object, map[string]any{ + "operation": operation, "phase": "Running", "message": "waiting for the PreSync hook", + }, "status", "operationState") + unstructured.RemoveNestedField(app.Object, "operation") + writeApp(t, client, app) + } + + t.Run("ours is Failed once overdue", func(t *testing.T) { + client := newArgoSource(manualApp()) + argo := pinnedArgo(client) + started, err := argo.Request(context.Background(), platformApp, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) + require.NoError(t, err) + ours, _, _ := unstructured.NestedMap(readApp(t, client).Object, "operation") + orphaned(t, client, ours) + + running, err := argo.Progress(context.Background(), platformApp, &started) + require.NoError(t, err) + assert.Equal(t, git.ApplyRunning, running.Outcome, "a controller pass may still write it") + started.Overdue = true + failed, err := argo.Progress(context.Background(), platformApp, &started) require.NoError(t, err) - return ok - }(), "nothing asked is nothing to wait for") + assert.Equal(t, git.ApplyFailed, failed.Outcome) + assert.Contains(t, failed.Message, ".operation was removed") + }) + + t.Run("a request waits it out, then syncs", func(t *testing.T) { + client := newArgoSource(manualApp()) + orphaned(t, client, theirOperation()) + argo := NewArgoCD(client) + argo.CanI = allowAll + argo.OperationWait, argo.PollInterval = 50*time.Millisecond, time.Millisecond + + started, err := argo.Request(context.Background(), platformApp, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, + Token: requestToken(stampTime), + }) + + require.NoError(t, err, "a state nothing processes any more is not in progress") + assert.Equal(t, OutcomeSyncRequested, started.Outcome) + assert.Equal(t, started.Token, operationToken(readApp(t, client).Object["operation"])) + }) } -// TestNewClient_PicksTheClientByKind: the dispatcher and the ApplierConfigured check use the kind -// spec.applier names. +// TestNewClient_PicksTheClientByKind: the dispatcher and the Configured check use the kind of +// applier the trigger holds. func TestNewClient_PicksTheClientByKind(t *testing.T) { client := newArgoSource() assert.IsType(t, &ArgoCD{}, NewClient(v1alpha3.ApplierKindApplication, client)) assert.IsType(t, &Flux{}, NewClient(v1alpha3.ApplierKindKustomization, client)) } + +// TestArgoCDTrigger_BindingMismatchReadsTheTargetRevision: an Application binds a GitTarget when its +// targetRevision names the GitTarget's branch, as its name or as refs/heads/ and the name. HEAD, an +// empty targetRevision (which is HEAD), a tag and another branch never follow it. +func TestArgoCDTrigger_BindingMismatchReadsTheTargetRevision(t *testing.T) { + for revision, want := range map[string]string{ + testBranch: "", + "refs/heads/" + testBranch: "", + "release": `applies branch "release", not main`, + "HEAD": `applies branch "HEAD"`, + "": `applies branch "HEAD"`, + "v1.0.0": `applies branch "v1.0.0"`, + } { + trigger := &v1alpha3.ArgoCDTrigger{ObjectMeta: metav1.ObjectMeta{Name: "platform"}} + trigger.Status.Source = &v1alpha3.ArgoCDTriggerSource{RepoURL: testRepoURL + ".git", TargetRevision: revision} + got := BindingMismatch(trigger, testBinding) + if want == "" { + assert.Empty(t, got, revision) + continue + } + assert.Contains(t, got, want, revision) + } +} diff --git a/internal/applier/dispatcher.go b/internal/applier/dispatcher.go index cac5f641f..23466e642 100644 --- a/internal/applier/dispatcher.go +++ b/internal/applier/dispatcher.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/otel/metric" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" @@ -25,7 +26,6 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/authz" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/telemetry" - itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) // Event reasons recorded on a GitTarget. @@ -33,8 +33,14 @@ const ( // EventReasonApplyRequested is a request that reached the applier. EventReasonApplyRequested = "ApplyRequested" // EventReasonApplyRequestFailed is the first failed attempt of a run of them. The branch worker - // keeps the obligation and asks again on a quiet branch; the Event is not repeated for its retries. + // keeps the reason pending and asks again on a quiet branch; the Event is not repeated for its retries. EventReasonApplyRequestFailed = "ApplyRequestFailed" + // EventReasonApplyFailed is a requested apply the applier ran and did not complete, the first of + // a run of failed attempts. The branch worker asks again on a quiet branch. + EventReasonApplyFailed = "ApplyFailed" + // EventReasonApplyUnresolved is a requested apply that did not complete within + // UnresolvedAfter. It is still followed, and its apply window stays open until it completes. + EventReasonApplyUnresolved = "ApplyUnresolved" ) // apply_requests_total label values. @@ -54,64 +60,94 @@ const shortCommitLength = 8 // errNotRunning fails an attempt made before the dispatcher started or after it stopped. var errNotRunning = errors.New("the apply request dispatcher is not running") -// SourceClients resolves the dynamic client of the cluster an applier lives in, by ClusterProvider -// name (GitTarget.ApplierCluster()). In production it is (*watch.Manager).ApplierDynamicClient. -type SourceClients func(ctx context.Context, clusterID string) (dynamic.Interface, error) +// Clients resolves the dynamic client of the cluster an applier lives in, by ClusterProvider +// name (a trigger's spec.clusterProviderRef). In production it is +// (*watch.Manager).ApplierDynamicClient. +type Clients func(ctx context.Context, clusterID string) (dynamic.Interface, error) -// DefaultAttemptTimeout bounds one attempt: a source cluster that does not answer fails it, and the -// branch worker decides when to try again. +// DefaultAttemptTimeout bounds one attempt up to its request: an applier's cluster that does not +// answer fails it, and the branch worker decides when to try again. const DefaultAttemptTimeout = 30 * time.Second -// DefaultApplyWindowTimeout bounds how long a delivered attempt stays out waiting for the applier to -// finish the apply. An apply that waits on health checks can take minutes; one the applier never -// finishes (the controller is down, the fetch failed) must not hold the applier forever, so the -// window then closes by time. It also bounds how long an Argo CD attempt waits for an operation -// already running on the Application before it sets its own. -const DefaultApplyWindowTimeout = 5 * time.Minute +// DefaultOperationWait bounds how long an Argo CD attempt waits for an operation already running on +// the Application before it sets its own. +const DefaultOperationWait = 5 * time.Minute -// DefaultHandledPollInterval is how often a delivered attempt reads whether the applier finished. -const DefaultHandledPollInterval = 2 * time.Second +// DefaultUnresolvedAfter is how long a requested apply may take before it is Unresolved. An apply +// that waits on health checks can take minutes. Past it, the operation is still followed, and its +// apply window stays open: the apply can still run, and closing the window would not stop it. +const DefaultUnresolvedAfter = 5 * time.Minute + +// DefaultPollInterval is how often a requested apply's progress is read, until it is Unresolved. +const DefaultPollInterval = 2 * time.Second + +// DefaultUnresolvedPollMax caps the interval an Unresolved operation is read at, which doubles from +// the poll interval. +const DefaultUnresolvedPollMax = time.Minute + +// pollTimeout bounds one read of a requested apply's progress. +const pollTimeout = 10 * time.Second // Dispatcher carries out the apply requests the branch workers decide. It implements // git.ApplyRequester, and runs as a manager.Runnable beside the branch workers (it asks for leader // election as they do, which keeps the two together should it ever be switched on). // // It keeps no queue and makes no decision of its own. Each request is one attempt for one applier, -// carried out on its own goroutine and reported back to the worker that decided it; the worker owns -// the obligation until an attempt delivers it, and decides every retry on a quiet branch. So nothing -// goes out that a worker did not check on a quiet branch, and nothing here can turn a delayed -// decision into an apply nobody checked. +// carried out on its own goroutine and reported back to the worker that decided it: the request, +// then the operation it started as it moves, until it completes or the worker abandons it. The +// worker keeps the reason pending until an operation answers it, and decides every retry on a quiet +// branch. So nothing goes out that a worker did not check on a quiet branch, and nothing here can +// turn a delayed decision into an apply nobody checked. // -// What it does check is what can change between the worker's decision and the patch. It reads each +// What it does check is what can change between the worker's decision and the write. It reads each // GitTarget again and asks only for those that are still the incarnation the worker decided for, -// still name this applier on this branch, are still admitted by their ClusterProvider, and, for a -// Refusal, still consent. When no target still holds the request's reason it asks nothing at all, -// rather than ask for something weaker: a weaker request is the worker's to decide. Then the -// applier's client (Flux or Argo CD, by the applier's kind) checks the source binding before it -// writes. +// still name a trigger that holds this applier on this branch, are still admitted by their +// ClusterProviders, and, for a Refusal, still consent. When no target still holds the request's +// reason it asks nothing at all, rather than ask for something weaker: a weaker request is the +// worker's to decide. Then the applier's client (Flux or Argo CD, by the applier's kind) checks the +// source binding before it writes. type Dispatcher struct { - reader client.Reader - sourceClients SourceClients - recorder record.EventRecorder - log logr.Logger + reader client.Reader + applierClients Clients + recorder record.EventRecorder + log logr.Logger // AttemptTimeout bounds one attempt up to its request. An Argo CD attempt may wait out an - // operation already running for up to ApplyWindowTimeout more. Set before Start. + // operation already running for up to OperationWait more. Set before Start, as are the rest. AttemptTimeout time.Duration - // ApplyWindowTimeout bounds the wait, after a request, for the applier to finish the apply it - // asked for; zero does not wait. Set before Start. - ApplyWindowTimeout time.Duration - // HandledPollInterval is how often that wait reads the applier's status. Set before Start. - HandledPollInterval time.Duration - // Now is the clock request tokens and the ledger read. + // OperationWait bounds an Argo CD attempt's wait for an operation already running. + OperationWait time.Duration + // UnresolvedAfter is how long a requested apply may take before it is reported Unresolved. + UnresolvedAfter time.Duration + // PollInterval is how often a requested apply's progress is read, and UnresolvedPollMax caps the + // interval an Unresolved one is read at. + PollInterval time.Duration + UnresolvedPollMax time.Duration + // Now is the clock request tokens are taken from. Now func() time.Time + // APIReader reads a FluxTrigger uncached just before its Kustomization is resumed, so a suspend + // its reconciler is making is seen (Ask.Held). NewDispatcher sets it to the reader it was given. + APIReader client.Reader runMu sync.RWMutex run context.Context attempts sync.WaitGroup + // stopped is set once Start returned. waiting are the recovered attempts asked for before Start, + // which it starts: a recovery runs once per process, so one failed for that reason would leave + // its operation followed by nobody. + stopped bool + waiting []git.ApplyRequest + + // following counts the attempts carrying out or following an operation, by trigger and token + // (Following). + followMu sync.Mutex + following map[followKey]int +} - ledgerMu sync.RWMutex - ledger map[string]ledgerEntry +// followKey is one operation of one trigger, by its request token. +type followKey struct { + trigger git.TriggerRef + token string } var ( @@ -121,90 +157,174 @@ var ( ) // NewDispatcher returns a Dispatcher that reads GitTargets through reader (the control cluster), -// reaches appliers through sourceClients, and records Events through recorder (nil records none). +// reaches appliers through applierClients, and records Events through recorder (nil records none). func NewDispatcher( reader client.Reader, - sourceClients SourceClients, + applierClients Clients, recorder record.EventRecorder, log logr.Logger, ) *Dispatcher { return &Dispatcher{ - reader: reader, - sourceClients: sourceClients, - recorder: recorder, - log: log, - AttemptTimeout: DefaultAttemptTimeout, - ApplyWindowTimeout: DefaultApplyWindowTimeout, - HandledPollInterval: DefaultHandledPollInterval, - Now: time.Now, - ledger: map[string]ledgerEntry{}, + reader: reader, + applierClients: applierClients, + APIReader: reader, + recorder: recorder, + log: log, + AttemptTimeout: DefaultAttemptTimeout, + OperationWait: DefaultOperationWait, + UnresolvedAfter: DefaultUnresolvedAfter, + PollInterval: DefaultPollInterval, + UnresolvedPollMax: DefaultUnresolvedPollMax, + Now: time.Now, } } // NeedLeaderElection is true: the branch workers that decide requests run only on the leader. func (d *Dispatcher) NeedLeaderElection() bool { return true } -// Start runs until ctx ends, and then waits for the attempts still running, which end with it. +// Start runs until ctx ends, and then waits for the attempts still running, which end with it. It +// first starts the recovered attempts asked for before it. func (d *Dispatcher) Start(ctx context.Context) error { d.runMu.Lock() d.run = ctx + waiting := d.waiting + d.waiting = nil + d.attempts.Add(len(waiting)) d.runMu.Unlock() - d.log.Info("apply request dispatcher started") + d.log.Info("apply request dispatcher started", "recoveredWaiting", len(waiting)) + for _, req := range waiting { + go d.launch(ctx, req) + } <-ctx.Done() d.runMu.Lock() - d.run = nil + d.run, d.stopped = nil, true d.runMu.Unlock() d.attempts.Wait() return nil } // RequestApply starts one attempt and returns at once; it is called on a branch worker's loop. The -// attempt reports on req.Report when it ends. +// attempt reports on req.Report: once when it asked nothing, or as its operation moves until it +// completes. An operation the worker abandons, or one still out when the dispatcher stops, reports +// nothing more. Before Start, an attempt fails at once, except a recovered one, which waits for +// Start: the worker recovers once, and the operation it follows may have the Kustomization resumed. func (d *Dispatcher) RequestApply(req git.ApplyRequest) { - d.runMu.RLock() + d.runMu.Lock() ctx := d.run - if ctx != nil { + switch { + case ctx != nil: d.attempts.Add(1) + case req.Recovered != nil && !d.stopped: + d.waiting = append(d.waiting, req) + d.runMu.Unlock() + d.log.V(1).Info("following a recorded operation once the dispatcher starts", + "applier", req.Applier.String(), "branch", req.Branch.String()) + return } - d.runMu.RUnlock() + d.runMu.Unlock() if ctx == nil { - req.Report(git.ApplyResult{ID: req.ID, Outcome: git.ApplyFailed, Err: errNotRunning}) + req.Report(git.ApplyResult{ID: req.ID, Outcome: git.ApplyRequestFailed, Err: errNotRunning}) return } - go func() { - defer d.attempts.Done() - req.Report(d.attempt(ctx, req)) - }() + go d.launch(ctx, req) } -// ledgerEntry is the last request delivered for one GitTarget, and which incarnation of it the -// request was for. -type ledgerEntry struct { - uid types.UID - status v1alpha3.GitTargetApplierStatus +// launch runs one attempt counted in d.attempts. +func (d *Dispatcher) launch(ctx context.Context, req git.ApplyRequest) { + defer d.attempts.Done() + if req.Recovered != nil { + d.followRecovered(ctx, req) + return + } + d.carryOut(ctx, req) } -// LastRequest is the last apply request delivered for a GitTarget's applier, for status.applier. -// It is false until one was delivered for this uid: a GitTarget deleted and recreated under the -// same name does not inherit its predecessor's. A CatchUp that found the applier already current is -// not a request made, and leaves it unchanged. -func (d *Dispatcher) LastRequest( - ref itypes.ResourceReference, uid types.UID, -) (v1alpha3.GitTargetApplierStatus, bool) { - d.ledgerMu.RLock() - defer d.ledgerMu.RUnlock() - entry, ok := d.ledger[ref.Key()] - if !ok || entry.uid != uid { - return v1alpha3.GitTargetApplierStatus{}, false +// followRecovered follows an operation recorded on the trigger to its end, asking the applier for +// nothing: the request was made by an earlier process, or by somebody else. A cluster that cannot +// be reached is tried again until the worker abandons the attempt, because the record can only be +// completed by following it. +func (d *Dispatcher) followRecovered(ctx context.Context, req git.ApplyRequest) { + log := d.log.WithValues("applier", req.Applier.String(), "branch", req.Branch.String(), + "reason", req.Reason, "commit", req.Commit, "recovered", true) + defer d.follows(req.Applier.Trigger, req.Recovered.Token)() + interval := d.PollInterval + for { + applierClient, err := d.applierClients(ctx, req.Applier.Cluster) + if err == nil { + applier := d.client(req.Applier.Kind, applierClient) + op := &operation{applier: applier, ref: applierName(req.Applier), started: recoveredStarted(req), log: log} + if res, followed := d.follow(ctx, req, op); followed { + req.Report(res) + } + return + } + log.V(1).Info("cannot reach the applier's cluster to follow a recorded operation", "error", err.Error()) + interval = min(unresolvedBackoff*interval, max(d.UnresolvedPollMax, d.PollInterval)) + if !wait(ctx, req.Abandoned, interval) { + return + } + } +} + +// recoveredStarted is what a Request would have returned for the operation a recovered attempt +// follows, from its record. +func recoveredStarted(req git.ApplyRequest) Started { + outcome := OutcomeResumed + if req.Applier.Kind == v1alpha3.ApplierKindApplication { + outcome = OutcomeSyncRequested + } + return Started{ + Outcome: outcome, Token: req.Recovered.Token, SuspendToken: req.Recovered.SuspendToken, + Commit: req.Recovered.Commit, Recovered: true, RequestedAt: req.Recovered.RequestedAt.Time, + } +} + +// Following reports whether an attempt of this dispatcher is carrying out or following the +// operation with token on trigger: from before it records the operation until the operation +// completed, or, abandoned, the Kustomization was suspended again. The FluxTrigger's reconciler +// asks it before it leaves a Kustomization resumed for an operation its trigger records: the +// dispatcher runs in the same process, on the leader, so an operation it does not follow is +// followed by nobody. +func (d *Dispatcher) Following(trigger git.TriggerRef, token string) bool { + d.followMu.Lock() + defer d.followMu.Unlock() + return d.following[followKey{trigger: trigger, token: token}] > 0 +} + +// follows registers an attempt as following the operation with token on trigger, and returns the +// func that ends it. +func (d *Dispatcher) follows(trigger git.TriggerRef, token string) func() { + key := followKey{trigger: trigger, token: token} + d.followMu.Lock() + defer d.followMu.Unlock() + if d.following == nil { + d.following = map[followKey]int{} + } + d.following[key]++ + return func() { + d.followMu.Lock() + defer d.followMu.Unlock() + if d.following[key]--; d.following[key] <= 0 { + delete(d.following, key) + } } - return *entry.status.DeepCopy(), true } -// Forget drops what the ledger holds for a GitTarget that is gone. -func (d *Dispatcher) Forget(ref itypes.ResourceReference) { - d.ledgerMu.Lock() - defer d.ledgerMu.Unlock() - delete(d.ledger, ref.Key()) +// carryOut runs one attempt and follows the operation it started, reporting each step. It counts +// as Following the operation from before the attempt records it. +func (d *Dispatcher) carryOut(ctx context.Context, req git.ApplyRequest) { + if req.Token == "" { + req.Token = requestToken(d.Now()) + } + defer d.follows(req.Applier.Trigger, req.Token)() + res, op := d.attempt(ctx, req) + req.Report(res) + if op == nil { + return + } + if res, followed := d.follow(ctx, req, op); followed { + req.Report(res) + } } // eligibleTarget is a GitTarget an attempt found may still ask the applier, as read for it. @@ -213,14 +333,20 @@ type eligibleTarget struct { reason v1alpha3.ApplyReason } -// attempt carries out one request and returns its report. A delivered one is reported once the -// applier has finished the apply (awaitHandled), so the branch worker's attempt stays out for -// exactly the apply window: the seconds in which an edit can be reverted by it. -func (d *Dispatcher) attempt(ctx context.Context, req git.ApplyRequest) git.ApplyResult { - run := ctx +// operation is an apply an attempt asked for, for follow to see through. +type operation struct { + applier Client + ref types.NamespacedName + started Started + log logr.Logger +} + +// attempt carries out one request and returns its report, with the operation it started when it +// asked the applier for an apply (the report is then ApplyRequested), or nil when it asked nothing. +func (d *Dispatcher) attempt(ctx context.Context, req git.ApplyRequest) (git.ApplyResult, *operation) { timeout := d.AttemptTimeout if req.Applier.Kind == v1alpha3.ApplierKindApplication { - timeout += d.ApplyWindowTimeout + timeout += d.OperationWait } ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -232,93 +358,356 @@ func (d *Dispatcher) attempt(ctx context.Context, req git.ApplyRequest) git.Appl eligible, want, err := d.recheck(ctx, req) if err != nil { - return d.failure(ctx, req, nil, report(git.ApplyFailed, err), log) + return d.failure(ctx, req, nil, report(git.ApplyRequestFailed, err), log), nil } if !holdsReason(eligible, req.Reason) { d.count(ctx, req, outcomeDropped) log.Info("no GitTarget may still ask this applier for this; asking nothing") - return report(git.ApplyDropped, nil) + return report(git.ApplyDropped, nil), nil } - sourceClient, err := d.sourceClients(ctx, req.Applier.Cluster) + applierClient, err := d.applierClients(ctx, req.Applier.Cluster) if err != nil { - return d.failure(ctx, req, eligible, - report(git.ApplyFailed, fmt.Errorf("source cluster %q: %w", req.Applier.Cluster, err)), log) + err = fmt.Errorf("the applier's cluster %q: %w", req.Applier.Cluster, err) + return d.failure(ctx, req, eligible, report(git.ApplyRequestFailed, err), log), nil } - applier := d.client(req.Applier.Kind, sourceClient) - stamped, err := applier.Request(ctx, applierName(req.Applier), want, req.Commit, req.Reason, req.Current) + applier := d.client(req.Applier.Kind, applierClient) + started, err := applier.Request(ctx, applierName(req.Applier), Ask{ + Want: want, Commit: req.Commit, Reason: req.Reason, Token: req.Token, Current: req.Current, Record: req.Record, + Held: func(ctx context.Context) error { return d.triggerHolds(ctx, req.Applier.Trigger) }, + }) switch { case errors.Is(err, errStale): d.count(ctx, req, outcomeStale) log.Info("the decision no longer holds; asking nothing, the branch worker decides again") - return report(git.ApplyStale, nil) + return report(git.ApplyStale, nil), nil case isPermanent(err): - return d.failure(ctx, req, eligible, report(git.ApplyRejected, err), log) + return d.failure(ctx, req, eligible, report(git.ApplyRejected, err), log), nil case err != nil: - return d.failure(ctx, req, eligible, report(git.ApplyFailed, err), log) + return d.failure(ctx, req, eligible, report(git.ApplyRequestFailed, err), log), nil } d.count(ctx, req, outcomeSent) - if stamped.Outcome == OutcomeCurrent { + if started.Outcome == OutcomeCurrent { log.V(1).Info("the applier needs no request for this commit; asking nothing") - return report(git.ApplyCurrent, nil) + return report(git.ApplyCurrent, nil), nil + } + log.Info("apply requested", "outcome", started.Outcome) + if started.Unconfirmed != nil { + log.Info("the request may not have reached the applier; following it by its token", + "error", started.Unconfirmed.Error()) } - log.Info("apply requested", "outcome", stamped.Outcome) d.delivered(req, eligible) - d.awaitHandled(run, applier, applierName(req.Applier), stamped, log) - return report(git.ApplyDelivered, nil) + return report( + git.ApplyRequested, + nil, + ), &operation{ + applier: applier, + ref: applierName(req.Applier), + started: started, + log: log, + } } // client is the Client for an applier of kind, over its cluster's client and this dispatcher's // clock and timings. -func (d *Dispatcher) client(kind string, sourceClient dynamic.Interface) Client { +func (d *Dispatcher) client(kind string, applierClient dynamic.Interface) Client { if kind == v1alpha3.ApplierKindApplication { - argo := NewArgoCD(sourceClient) + argo := NewArgoCD(applierClient) argo.Now = d.Now - argo.OperationWait = d.ApplyWindowTimeout - argo.PollInterval = d.HandledPollInterval + argo.OperationWait = d.OperationWait + argo.PollInterval = d.PollInterval return argo } - flux := NewFlux(sourceClient) + flux := NewFlux(applierClient) flux.Now = d.Now + flux.PollInterval = min(flux.PollInterval, d.PollInterval) return flux } -// awaitHandled waits, up to ApplyWindowTimeout, for the applier to finish the apply stamped asked -// for. It decides nothing: the request was delivered either way, and a wait that ends by time or by -// shutdown only closes the window sooner than the applier did. -func (d *Dispatcher) awaitHandled( - ctx context.Context, applier Client, ref types.NamespacedName, stamped Stamped, log logr.Logger, -) { - if d.ApplyWindowTimeout <= 0 { +// follow reads the operation's progress until it completes, reporting each move on req.Report, and +// returns the report that ends the attempt. It decides nothing: it reports what the applier shows, +// and lets the client take the steps that are its own (suspending a Flux Kustomization again). +// +// An operation that has not completed within UnresolvedAfter is reported Unresolved, once, and is +// read on at a doubling interval: the apply can still run, and only its completion closes the +// window. The client is told it is overdue, so a Flux Kustomization is suspended at once. The +// applier object being gone completes it as Failed, and so does its trigger being deleted: the +// trigger's own finalizer suspends the Kustomization and hands it back, and a suspend from here +// would undo that. It returns false, and reports nothing more, when the worker abandons the +// attempt or the dispatcher stops; a Flux Kustomization this process resumed is suspended first. +func (d *Dispatcher) follow(ctx context.Context, req git.ApplyRequest, op *operation) (git.ApplyResult, bool) { + started := d.Now() + phase, suspendToken := git.ApplyRequested, "" + interval := d.PollInterval + for { + progress, err := d.progress(ctx, req, op) + switch { + case err != nil: + if ctx.Err() == nil { + op.log.V(1).Info("could not read the requested apply's progress", "error", err.Error()) + } + case progress.Outcome.Ends(): + return d.completed(ctx, req, op, progress, d.Now().Sub(started)), true + case progress.Outcome != phase && phase != git.ApplyUnresolved, progress.SuspendToken != suspendToken: + if phase != git.ApplyUnresolved { + phase = progress.Outcome + } + suspendToken = progress.SuspendToken + req.Report(git.ApplyResult{ID: req.ID, Outcome: phase, SuspendToken: suspendToken}) + } + if phase != git.ApplyUnresolved && d.Now().Sub(started) >= d.UnresolvedAfter { + phase = git.ApplyUnresolved + op.started.Overdue = true + d.unresolved(ctx, req, op) + req.Report(git.ApplyResult{ID: req.ID, Outcome: phase, SuspendToken: suspendToken}) + } + if phase == git.ApplyUnresolved { + interval = min(unresolvedBackoff*interval, max(d.UnresolvedPollMax, d.PollInterval)) + } + if !wait(ctx, req.Abandoned, interval) { + d.letGo(ctx, req, op) + return git.ApplyResult{}, false + } + } +} + +// letGo gives an operation up that follow stops following: the worker abandoned it, or the +// dispatcher stops. Either way a Flux Kustomization it has resumed is suspended first. +func (d *Dispatcher) letGo(ctx context.Context, req git.ApplyRequest, op *operation) { + abandoned := ctx.Err() == nil + if abandoned { + d.suspendAbandoned(ctx, req, op) + } else { + d.suspendOnStop(ctx, req, op) + } + op.log.V(1).Info("stopped following the requested apply", "abandoned", abandoned) +} + +// suspendAbandoned sees an abandoned Flux operation through to its suspend before the attempt lets it +// go: the worker forgot its applier (no GitTarget names it any more), but the Kustomization may +// still be resumed, and nothing else follows the operation. It is treated as overdue, so it is +// suspended at once, and read until that suspend's token is handled, within AttemptTimeout and +// whether or not the dispatcher stops meanwhile. It reports nothing: nobody waits for the report. +// A trigger being deleted completes it without a suspend, since the trigger hands its Kustomization +// back itself. +func (d *Dispatcher) suspendAbandoned(ctx context.Context, req git.ApplyRequest, op *operation) { + if op.started.Outcome != OutcomeResumed { return } - ctx, cancel := context.WithTimeout(ctx, d.ApplyWindowTimeout) + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), d.AttemptTimeout) defer cancel() - started := d.Now() - ticker := time.NewTicker(d.HandledPollInterval) - defer ticker.Stop() + op.started.Overdue = true for { - handled, err := applier.Handled(ctx, ref, stamped) - switch { - case handled: - log.V(1).Info("the applier finished the requested apply", "after", d.Now().Sub(started).String()) + progress, err := d.progress(ctx, req, op) + if err == nil && progress.Outcome.Ends() { + op.log.Info("the worker abandoned the operation; its Kustomization is suspended again", + "outcome", progress.Outcome) return - case err != nil && ctx.Err() == nil: - log.V(1).Info("could not read whether the applier finished the requested apply", "error", err.Error()) } - select { - case <-ctx.Done(): - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - log.Info("the applier did not report the requested apply finished in time; the apply window closes now", - "waited", d.Now().Sub(started).String(), "timeout", d.ApplyWindowTimeout.String()) - } + if !wait(ctx, nil, d.PollInterval) { + op.log.Info("the worker abandoned the operation, and its Kustomization was not seen suspended again "+ + "in time; the trigger's reconciler suspends it", "suspendToken", op.started.SuspendToken) return - case <-ticker.C: } } } +// stopSuspendTimeout bounds the one suspend a stopping dispatcher makes, well inside the pod's +// termination grace period. +const stopSuspendTimeout = 5 * time.Second + +// suspendOnStop suspends a Flux Kustomization this process resumed and has not suspended again, once, +// as the dispatcher stops mid-apply: otherwise it stays resumed until the next process recovers the +// operation, and Flux applies a new artifact, or a drift reconcile reverts live edits, on its own in +// between. It does not wait for the suspend to be handled; the next process recovers a suspend of +// its own that this one did not report, as it does after a crash. A trigger being deleted hands +// its Kustomization back itself, and a suspend from here would undo that. +func (d *Dispatcher) suspendOnStop(ctx context.Context, req git.ApplyRequest, op *operation) { + flux, isFlux := op.applier.(*Flux) + if !isFlux || op.started.Outcome != OutcomeResumed || op.started.SuspendToken != "" { + return + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), stopSuspendTimeout) + defer cancel() + if released, err := d.triggerReleased(ctx, req.Applier.Trigger); err != nil || released { + return + } + token, err := flux.Suspend(ctx, op.ref) + if err != nil { + op.log.Info("stopping mid-apply, and the Kustomization could not be suspended; the next process "+ + "suspends it when it recovers the operation", "error", err.Error()) + return + } + op.log.Info("stopping mid-apply: suspended the Kustomization", "suspendToken", token) +} + +// progress reads an operation's progress once, within pollTimeout. An applier that is gone, or a +// trigger that is being deleted, completes it as Failed. +func (d *Dispatcher) progress(ctx context.Context, req git.ApplyRequest, op *operation) (Progress, error) { + read, cancel := context.WithTimeout(ctx, pollTimeout) + defer cancel() + released, err := d.triggerReleased(read, req.Applier.Trigger) + if err != nil { + return Progress{}, err + } + if released { + return Progress{ + Outcome: git.ApplyFailed, + Message: "the trigger is being deleted, and hands its applier back", + }, nil + } + progress, err := op.applier.Progress(read, op.ref, &op.started) + if apierrors.IsNotFound(err) { + return Progress{Outcome: git.ApplyFailed, Message: "the applier is gone: " + err.Error()}, nil + } + return progress, err +} + +// TriggerConditionHeld is a FluxTrigger's Held condition: True (reason Suspended) once its +// Kustomization is suspended under GitOps Reverser's field manager and that suspend's token is +// handled, and True (reason Applying) while an operation of a branch worker has it resumed. The +// trigger's reconciler sets it. +const TriggerConditionHeld = "Held" + +// triggerHolds is Ask.Held: it reads the FluxTrigger uncached, and errs unless it is +// Held=True and not being deleted. Its reconciler is otherwise suspending the Kustomization, +// or about to (an adoption, an external resume it suspends again, a hand-back), and that suspend would land +// on the resume: Flux would then handle the resume's token on a suspended return, and apply +// nothing. It also errs when another trigger comes first in line for the Kustomization, read +// uncached as well: the recheck's cached read can still miss one. The attempt fails, and the worker +// asks again after its backoff. +func (d *Dispatcher) triggerHolds(ctx context.Context, ref git.TriggerRef) error { + if ref.Kind != v1alpha3.ApplyTriggerKindFlux { + return nil + } + trigger := &v1alpha3.FluxTrigger{} + if err := d.APIReader.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, trigger); err != nil { + return fmt.Errorf("read %s: %w", ref, err) + } + held := apimeta.FindStatusCondition(trigger.Status.Conditions, TriggerConditionHeld) + holder, err := HeldByAnother(ctx, d.APIReader, trigger) + switch { + case err != nil: + return fmt.Errorf("check which trigger holds the Kustomization of %s: %w", ref, err) + case holder != nil: + return fmt.Errorf("FluxTrigger %s/%s no longer holds its Kustomization: %s", ref.Namespace, ref.Name, + HeldByAnotherMessage(trigger, holder)) + case !trigger.DeletionTimestamp.IsZero(): + return fmt.Errorf("FluxTrigger %s/%s is being deleted, and hands its Kustomization back", + ref.Namespace, ref.Name) + case held == nil || held.Status != metav1.ConditionTrue: + message := "it reports no Held condition yet" + if held != nil { + message = fmt.Sprintf("it is Held=%s, %s: %s", held.Status, held.Reason, held.Message) + } + return fmt.Errorf("FluxTrigger %s/%s does not hold its Kustomization suspended: %s", + ref.Namespace, ref.Name, message) + } + return nil +} + +// triggerReleased reports whether a trigger is gone or being deleted. It reads through APIReader, +// uncached in production, because the step that follows may suspend the Kustomization: after a +// hand-back's release, a suspend from a cached read that still showed the trigger would hold a +// Kustomization no trigger holds any more. +func (d *Dispatcher) triggerReleased(ctx context.Context, ref git.TriggerRef) (bool, error) { + trigger := v1alpha3.NewApplyTrigger(ref.Kind) + if trigger == nil { + return false, nil + } + err := d.APIReader.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, trigger) + switch { + case apierrors.IsNotFound(err): + return true, nil + case err != nil: + return false, fmt.Errorf("read %s: %w", ref, err) + } + return !trigger.GetDeletionTimestamp().IsZero(), nil +} + +// unresolvedBackoff is the factor an Unresolved operation's poll interval grows by on each read. +const unresolvedBackoff = 2 + +// wait waits for interval, and reports false when ctx ends or abandoned is closed first. +func wait(ctx context.Context, abandoned <-chan struct{}, interval time.Duration) bool { + timer := time.NewTimer(interval) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-abandoned: + return false + case <-timer.C: + return true + } +} + +// completed is the report of an operation that completed. A Failed one records ApplyFailed on its +// GitTargets for the first failure of a run, as a failed request records ApplyRequestFailed. +func (d *Dispatcher) completed( + ctx context.Context, req git.ApplyRequest, op *operation, progress Progress, took time.Duration, +) git.ApplyResult { + result := git.ApplyResult{ + ID: req.ID, Outcome: progress.Outcome, SuspendToken: progress.SuspendToken, + AppliedRevision: progress.AppliedRevision, + } + if progress.Outcome != git.ApplyFailed { + if progress.Message != "" { + // The residual race: the source moved between the recheck and the apply. The apply + // window withholds a late write over a newer edit; nothing more is asked here. + op.log.Info("the applier applied another revision than the commit it was asked for", + "after", took.String(), "message", progress.Message) + result.Message = progress.Message + if telemetry.ApplyRevisionMismatchesTotal != nil { + telemetry.ApplyRevisionMismatchesTotal.Add(context.WithoutCancel(ctx), 1, + metric.WithAttributes(attribute.String("applier_kind", applierKindOf(req)))) + } + } else { + op.log.V(1).Info("the applier finished the requested apply", "after", took.String()) + } + return result + } + err := errors.New(progress.Message) + op.log.Info("the applier did not complete the requested apply; the branch worker decides when to ask again", + "after", took.String(), "message", progress.Message, "priorFailures", req.PriorFailures) + if req.PriorFailures == 0 { + d.event(ctx, req, corev1.EventTypeWarning, EventReasonApplyFailed, + "%s did not complete the apply of %s (%s), and will be asked again once the branch is quiet: %v", + describeApplier(req.Applier), DescribeCommit(req.Commit), req.Reason, err) + } + result.Err = err + return result +} + +// unresolved logs an operation that did not complete within UnresolvedAfter and records +// ApplyUnresolved on its GitTargets. +func (d *Dispatcher) unresolved(ctx context.Context, req git.ApplyRequest, op *operation) { + op.log.Info("the applier has not completed the requested apply in time; still following it, and its "+ + "apply window stays open until it completes", "after", d.UnresolvedAfter.String()) + d.event(ctx, req, corev1.EventTypeWarning, EventReasonApplyUnresolved, + "%s has not completed the apply of %s (%s) within %s; GitOps Reverser keeps following it, and asks "+ + "it nothing more until it completes", + describeApplier(req.Applier), DescribeCommit(req.Commit), req.Reason, d.UnresolvedAfter) +} + +// event records an Event on each of the request's GitTargets that still exists. +func (d *Dispatcher) event( + ctx context.Context, req git.ApplyRequest, eventType, reason, messageFmt string, args ...any, +) { + if d.recorder == nil { + return + } + for _, candidate := range req.Targets { + target := &v1alpha3.GitTarget{} + key := client.ObjectKey{Namespace: candidate.Ref.Namespace, Name: candidate.Ref.Name} + if err := d.reader.Get(ctx, key, target); err != nil || target.UID != candidate.UID { + continue + } + d.recorder.Eventf(target, eventType, reason, messageFmt, args...) + } +} + // applierName is the object an applier reference names. func applierName(ref git.ApplierRef) types.NamespacedName { return types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name} @@ -346,7 +735,7 @@ func (d *Dispatcher) failure( for _, e := range eligible { d.recorder.Eventf(e.target, corev1.EventTypeWarning, EventReasonApplyRequestFailed, "could not ask %s to apply %s (%s), and will ask again once the branch is quiet: %v", - describeApplier(req.Applier), describeCommit(req.Commit), e.reason, res.Err) + describeApplier(req.Applier), DescribeCommit(req.Commit), e.reason, res.Err) } } return res @@ -384,8 +773,7 @@ func (d *Dispatcher) recheck(ctx context.Context, req git.ApplyRequest) ([]eligi // // A ClusterProvider's client is shared by every GitTarget that uses the cluster, so a client being // available is no permission to use it: a GitTarget its ClusterProviders do not admit (its own, and -// the applier's when that is another) would otherwise drive the applier on the strength of another -// GitTarget's admission. +// its trigger's) would otherwise drive the applier on the strength of another GitTarget's admission. func (d *Dispatcher) recheckTarget( ctx context.Context, req git.ApplyRequest, candidate git.ApplyTarget, ) (*v1alpha3.GitTarget, error) { @@ -398,11 +786,18 @@ func (d *Dispatcher) recheckTarget( } return nil, fmt.Errorf("read GitTarget %s: %w", candidate.Ref, err) } - if target.UID != candidate.UID || !target.DeletionTimestamp.IsZero() || !namesApplier(target, req) { + if target.UID != candidate.UID || !target.DeletionTimestamp.IsZero() || !onBranch(target, req) { log.V(1).Info("GitTarget is no longer the one that named this applier on this branch; not asking for it") return nil, nil //nolint:nilnil // as above } - decision, err := authz.ApplierAdmitted(ctx, d.reader, target) + trigger, err := holdingTrigger(ctx, d.reader, target, req) + if err != nil || trigger == nil { + if err == nil { + log.V(1).Info("GitTarget no longer names a trigger that holds this applier; not asking for it") + } + return nil, err + } + decision, err := authz.ApplierAdmitted(ctx, d.reader, target, trigger) if err != nil { return nil, fmt.Errorf("check GitTarget %s against its ClusterProviders: %w", candidate.Ref, err) } @@ -418,38 +813,40 @@ func (d *Dispatcher) recheckTarget( return target, nil } -// namesApplier reports whether a GitTarget still names the request's applier on its branch. -func namesApplier(target *v1alpha3.GitTarget, req git.ApplyRequest) bool { - if target.Spec.Applier == nil || target.Namespace != req.Branch.RepoNamespace || - target.Spec.GitProviderRef.Name != req.Branch.RepoName || target.Spec.Branch != req.Branch.Branch { - return false +// holdingTrigger is the trigger a GitTarget names when it still holds the request's applier, or nil. +// A trigger in line behind another naming the same applier (Holder) holds none of it. +func holdingTrigger( + ctx context.Context, reader client.Reader, target *v1alpha3.GitTarget, req git.ApplyRequest, +) (v1alpha3.ApplyTrigger, error) { + trigger, err := git.ResolveApplyTrigger(ctx, reader, target) + if err != nil || trigger == nil { + return nil, err } - return git.ApplierOf(target) == req.Applier + if !trigger.GetDeletionTimestamp().IsZero() || git.ApplierOf(trigger) != req.Applier { + return nil, nil //nolint:nilnil // a trigger that holds another applier holds none of this one + } + holder, err := HeldByAnother(ctx, reader, trigger) + if err != nil || holder != nil { + return nil, err + } + return trigger, nil } -// delivered records a delivered request in the ledger and as an ApplyRequested Event on every -// GitTarget it was for, each with its own reason. -func (d *Dispatcher) delivered(req git.ApplyRequest, eligible []eligibleTarget) { - at := metav1.NewTime(d.Now()) - d.ledgerMu.Lock() - for _, e := range eligible { - d.ledger[itypes.NewResourceReference(e.target.Name, e.target.Namespace).Key()] = ledgerEntry{ - uid: e.target.UID, - status: v1alpha3.GitTargetApplierStatus{ - RequestedCommit: req.Commit, - RequestedAt: at.DeepCopy(), - Reason: e.reason, - }, - } - } - d.ledgerMu.Unlock() +// onBranch reports whether a GitTarget still names a trigger, on the request's branch. +func onBranch(target *v1alpha3.GitTarget, req git.ApplyRequest) bool { + return target.Spec.ApplyTriggerRef != nil && target.Namespace == req.Branch.RepoNamespace && + target.Spec.GitProviderRef.Name == req.Branch.RepoName && target.Spec.Branch == req.Branch.Branch +} +// delivered records a delivered request as an ApplyRequested Event on every GitTarget it was for, +// each with its own reason. The operation itself is recorded on the trigger, by the branch worker. +func (d *Dispatcher) delivered(req git.ApplyRequest, eligible []eligibleTarget) { if d.recorder == nil { return } for _, e := range eligible { d.recorder.Eventf(e.target, corev1.EventTypeNormal, EventReasonApplyRequested, - "asked %s to apply %s (%s)", describeApplier(req.Applier), describeCommit(req.Commit), e.reason) + "asked %s to apply %s (%s)", describeApplier(req.Applier), DescribeCommit(req.Commit), e.reason) } } @@ -458,8 +855,9 @@ func describeApplier(ref git.ApplierRef) string { return ref.Kind + " " + applierName(ref).String() } -// describeCommit names a commit in an Event: its short hash, or what an empty one means. -func describeCommit(commit string) string { +// DescribeCommit names a commit in an Event or a message: its short hash, or what an empty one +// means. +func DescribeCommit(commit string) string { if commit == "" { return "the branch, which the remote does not carry" } @@ -474,13 +872,128 @@ func (d *Dispatcher) count(ctx context.Context, req git.ApplyRequest, outcome st if telemetry.ApplyRequestsTotal == nil { return } - kind := applierKindKustomization - if req.Applier.Kind == v1alpha3.ApplierKindApplication { - kind = applierKindApplication - } telemetry.ApplyRequestsTotal.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes( - attribute.String("applier_kind", kind), + attribute.String("applier_kind", applierKindOf(req)), attribute.String("reason", string(req.Reason)), attribute.String("outcome", outcome), )) } + +// applierKindOf is the applier_kind label of a request's applier. +func applierKindOf(req git.ApplyRequest) string { + if req.Applier.Kind == v1alpha3.ApplierKindApplication { + return applierKindApplication + } + return applierKindKustomization +} + +// PrefetchSource asks the GitRepository a FluxTrigger's Kustomization applies to fetch now, ahead of +// the apply, and returns at once. It asks nothing when the trigger's status.source.prefetch is not +// true, when a FluxTrigger on the same source has an operation in flight (a resumed Kustomization +// applies whatever the artifact becomes), or when the artifact already holds the commit. It applies +// nothing, so it reports nothing. +func (d *Dispatcher) PrefetchSource(req git.PrefetchRequest) { + if req.Applier.Kind != v1alpha3.ApplierKindKustomization { + return + } + d.runMu.RLock() + ctx := d.run + if ctx != nil { + d.attempts.Add(1) + } + d.runMu.RUnlock() + if ctx == nil { + return + } + go func() { + defer d.attempts.Done() + ctx, cancel := context.WithTimeout(ctx, d.AttemptTimeout) + defer cancel() + log := d.log.WithValues("applier", req.Applier.String(), "commit", req.Commit) + if err := d.prefetch(ctx, req); err != nil && !errors.Is(err, errNotPrefetched) { + log.V(1).Info("did not prefetch the source", "reason", err.Error()) + return + } + log.V(1).Info("prefetched the source") + }() +} + +// errPrefetchHeld is a prefetch held back by the trigger's source. +var errPrefetchHeld = errors.New("the source may not be prefetched now") + +// prefetch checks the trigger and its source's other triggers, then the source's consumers as they +// are now (heldNow), and stamps the source, which every check was of: status.source's GitRepository, +// only while the Kustomization still applies it. +func (d *Dispatcher) prefetch(ctx context.Context, req git.PrefetchRequest) error { + trigger := &v1alpha3.FluxTrigger{} + key := client.ObjectKey{Namespace: req.Applier.Trigger.Namespace, Name: req.Applier.Trigger.Name} + if err := d.reader.Get(ctx, key, trigger); err != nil { + return err + } + if trigger.Status.Source == nil || !trigger.Status.Source.Prefetch || git.ApplierOf(trigger) != req.Applier { + return errPrefetchHeld + } + var triggers v1alpha3.FluxTriggerList + if err := d.reader.List(ctx, &triggers); err != nil { + return err + } + for i := range triggers.Items { + other := &triggers.Items[i] + if other.ClusterProvider() != trigger.ClusterProvider() || other.Status.Source == nil || + other.Status.Source.GitRepository != trigger.Status.Source.GitRepository { + continue + } + if operation := other.Status.Operation; operation != nil && !operation.Phase.Completed() { + return errPrefetchHeld + } + } + flux, err := d.heldNow(ctx, req, trigger, triggers.Items) + if err != nil { + return err + } + checked, _ := trigger.SharedSource() + return flux.Prefetch(ctx, applierName(req.Applier), checked, req.Commit) +} + +// heldNow checks again, in the applier's cluster, what the trigger's status.source.prefetch said when +// its reconciler last ran, up to a minute ago: every consumer of its GitRepository is a +// Kustomization a FluxTrigger on the same cluster names, and is suspended, held by GitOps Reverser +// with its suspend handled now. A consumer that appeared since, or one Flux applies on its own again, would apply +// the prefetched artifact at once, over live edits not pushed yet (errPrefetchHeld). It returns the +// Flux client of the applier's cluster. +func (d *Dispatcher) heldNow( + ctx context.Context, req git.PrefetchRequest, trigger *v1alpha3.FluxTrigger, triggers []v1alpha3.FluxTrigger, +) (*Flux, error) { + applierClient, err := d.applierClients(ctx, req.Applier.Cluster) + if err != nil { + return nil, err + } + flux := NewFlux(applierClient) + flux.Now = d.Now + named := map[types.NamespacedName]bool{} + for i := range triggers { + other := &triggers[i] + if other.DeletionTimestamp.IsZero() && other.ClusterProvider() == trigger.ClusterProvider() { + named[other.Applier()] = true + } + } + source := trigger.Status.Source.GitRepository + consumers, err := flux.Consumers(ctx, types.NamespacedName{Namespace: source.Namespace, Name: source.Name}) + if err != nil { + return nil, err + } + for _, consumer := range consumers { + key := types.NamespacedName{Namespace: consumer.Namespace, Name: consumer.Name} + if consumer.Kind != v1alpha3.ApplierKindKustomization || !named[key] { + return nil, fmt.Errorf("%w: %s %s is not held by a FluxTrigger", errPrefetchHeld, consumer.Kind, key) + } + suspension, err := flux.Suspension(ctx, key) + if err != nil { + return nil, fmt.Errorf("read %s %s: %w", kindKustomization, key, err) + } + if !suspension.Suspended || !suspension.Held || !suspension.Handled { + return nil, fmt.Errorf("%w: %s %s is not held suspended now", errPrefetchHeld, kindKustomization, key) + } + } + return flux, nil +} diff --git a/internal/applier/dispatcher_test.go b/internal/applier/dispatcher_test.go index 1a386f76b..59f8d0f9c 100644 --- a/internal/applier/dispatcher_test.go +++ b/internal/applier/dispatcher_test.go @@ -4,9 +4,11 @@ package applier import ( "context" + "encoding/json" "fmt" "strings" "sync" + "sync/atomic" "testing" "time" @@ -22,11 +24,13 @@ import ( "k8s.io/client-go/dynamic" dynamicfake "k8s.io/client-go/dynamic/fake" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + k8stesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/telemetry" itypes "github.com/ConfigButler/gitops-reverser/internal/types" @@ -39,7 +43,17 @@ const ( var testBranchKey = git.BranchKey{RepoNamespace: targetNS, RepoName: "platform", Branch: testBranch} -// gitTarget builds a GitTarget on testBranchKey; a nil kustomizationRef declares no applier. +// Test-only annotations on a GitTarget, which newDispatcherFixture reads to create the trigger the +// GitTarget names: the applier it holds ("namespace/name") and the cluster it is in. +const ( + testKustomizationAnnotation = "test.gitops-reverser/kustomization" + testApplicationAnnotation = "test.gitops-reverser/application" + testTriggerClusterAnnotation = "test.gitops-reverser/trigger-cluster" +) + +// gitTarget builds a GitTarget on testBranchKey, mirroring cluster (the default when empty). A +// kustomizationRef makes it name the FluxTrigger of that Kustomization's name, which holds it in the +// same cluster; nil names no trigger. func gitTarget(name string, kustomizationRef *fluxmeta.NamespacedObjectReference, cluster string) *v1alpha3.GitTarget { target := &v1alpha3.GitTarget{ ObjectMeta: metav1.ObjectMeta{Namespace: targetNS, Name: name, UID: uid(name)}, @@ -49,15 +63,69 @@ func gitTarget(name string, kustomizationRef *fluxmeta.NamespacedObjectReference Path: name, }, } - if kustomizationRef != nil { - target.Spec.Applier = &v1alpha3.GitTargetApplier{KustomizationRef: kustomizationRef} - } if cluster != "" { target.Spec.ClusterProviderRef = &fluxmeta.LocalObjectReference{Name: cluster} } + if kustomizationRef != nil { + namesKustomization(target, kustomizationRef.Namespace, kustomizationRef.Name) + target.Annotations[testTriggerClusterAnnotation] = target.SourceCluster() + } return target } +// namesKustomization makes a GitTarget name the FluxTrigger called name, which holds the +// Kustomization namespace/name (the trigger's own namespace when namespace is empty). +func namesKustomization(target *v1alpha3.GitTarget, namespace, name string) { + if target.Annotations == nil { + target.Annotations = map[string]string{} + } + target.Spec.ApplyTriggerRef = &v1alpha3.ApplyTriggerReference{Kind: v1alpha3.ApplyTriggerKindFlux, Name: name} + target.Annotations[testKustomizationAnnotation] = namespace + "/" + name +} + +// triggersFor returns the trigger each GitTarget names, as its test annotations describe it. +func triggersFor(objects []client.Object) []client.Object { + var triggers []client.Object + seen := map[string]bool{} + for _, obj := range objects { + target, ok := obj.(*v1alpha3.GitTarget) + if !ok || target.Spec.ApplyTriggerRef == nil { + continue + } + key := target.Spec.ApplyTriggerRef.Kind + "/" + target.Spec.ApplyTriggerRef.Name + if seen[key] { + continue + } + seen[key] = true + cluster := target.Annotations[testTriggerClusterAnnotation] + if cluster == "" { + cluster = v1alpha3.DefaultClusterProviderName + } + meta := metav1.ObjectMeta{Namespace: target.Namespace, Name: target.Spec.ApplyTriggerRef.Name} + provider := &fluxmeta.LocalObjectReference{Name: cluster} + if app, ok := target.Annotations[testApplicationAnnotation]; ok { + namespace, name, _ := strings.Cut(app, "/") + triggers = append(triggers, &v1alpha3.ArgoCDTrigger{ObjectMeta: meta, Spec: v1alpha3.ArgoCDTriggerSpec{ + ClusterProviderRef: provider, + ApplicationRef: fluxmeta.NamespacedObjectReference{Namespace: namespace, Name: name}, + }}) + continue + } + if ks, ok := target.Annotations[testKustomizationAnnotation]; ok { + namespace, name, _ := strings.Cut(ks, "/") + // Its reconciler holds the Kustomization (Ask.Held), unless a test says otherwise. + triggers = append(triggers, &v1alpha3.FluxTrigger{ObjectMeta: meta, Spec: v1alpha3.FluxTriggerSpec{ + ClusterProviderRef: provider, + KustomizationRef: fluxmeta.NamespacedObjectReference{Namespace: namespace, Name: name}, + }, Status: v1alpha3.FluxTriggerStatus{Conditions: []metav1.Condition{{ + Type: TriggerConditionHeld, Status: metav1.ConditionTrue, Reason: "Suspended", + LastTransitionTime: metav1.Now(), + }}}}) + } + } + return triggers +} + // reapplying makes a GitTarget consent to a Refusal request. func reapplying(target *v1alpha3.GitTarget) *v1alpha3.GitTarget { target.Spec.OnRefusal = v1alpha3.RefusalActionReapply @@ -68,7 +136,11 @@ func reapplying(target *v1alpha3.GitTarget) *v1alpha3.GitTarget { func clusterProvider(name string, admits ...string) *v1alpha3.ClusterProvider { return &v1alpha3.ClusterProvider{ ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: v1alpha3.ClusterProviderSpec{AccessFrom: &v1alpha3.NamespaceMatcher{Names: admits}}, + Spec: v1alpha3.ClusterProviderSpec{ + AccessFrom: &v1alpha3.NamespaceMatcher{Names: admits}, + // The test appliers live in flux-system and argocd, not in the triggers' namespace. + AllowCrossNamespace: &v1alpha3.ClusterProviderCrossNamespace{Appliers: true}, + }, } } @@ -95,9 +167,13 @@ func ref(name string) itypes.ResourceReference { return itypes.NewResourceRefere // uid is the uid gitTarget gives the GitTarget of that name. func uid(name string) types.UID { return types.UID("uid-" + name) } -// kustomizationApplier is a Kustomization applier in a cluster. +// kustomizationApplier is a Kustomization applier in a cluster, held by the FluxTrigger in targetNS +// of the Kustomization's name. func kustomizationApplier(cluster, namespace, name string) git.ApplierRef { - return git.ApplierRef{Cluster: cluster, Kind: v1alpha3.ApplierKindKustomization, Namespace: namespace, Name: name} + return git.ApplierRef{ + Trigger: git.TriggerRef{Kind: v1alpha3.ApplyTriggerKindFlux, Namespace: targetNS, Name: name}, + Cluster: cluster, Kind: v1alpha3.ApplierKindKustomization, Namespace: namespace, Name: name, + } } // checkout is the applier most tests ask: flux-system/checkout in the default cluster. @@ -136,6 +212,7 @@ type dispatcherFixture struct { dispatcher *Dispatcher control client.Client source *sourceStub + sim *fluxsim.Sim recorder *record.FakeRecorder reader *sdkmetric.ManualReader } @@ -149,19 +226,69 @@ func newDispatcherFixture(t *testing.T, targets []client.Object, sourceObjs ...r require.NoError(t, clientgoscheme.AddToScheme(scheme)) require.NoError(t, v1alpha3.AddToScheme(scheme)) controlClient := fake.NewClientBuilder().WithScheme(scheme). - WithObjects(append(controlPlane(), targets...)...).Build() + WithObjects(append(append(controlPlane(), targets...), triggersFor(targets)...)...).Build() source := &sourceStub{client: newSourceClient(sourceObjs...)} + sim := fluxsim.New(t, source.client) + for _, obj := range sourceObjs { + if u, ok := obj.(*unstructured.Unstructured); ok && u.GetKind() == kindGitRepository { + sim.Fetches(types.NamespacedName{Namespace: u.GetNamespace(), Name: u.GetName()}, "main@sha1:"+testCommit) + } + } recorder := record.NewFakeRecorder(64) d := NewDispatcher(controlClient, source.get, recorder, logr.Discard()) - d.Now = func() time.Time { return stampTime } - d.ApplyWindowTimeout = 0 // the tests that wait for Flux set their own - return &dispatcherFixture{dispatcher: d, control: controlClient, source: source, recorder: recorder, reader: reader} + d.Now = fluxsim.TickingClock(stampTime) + d.PollInterval, d.UnresolvedPollMax = time.Millisecond, 5*time.Millisecond + return &dispatcherFixture{ + dispatcher: d, control: controlClient, source: source, sim: sim, recorder: recorder, reader: reader, + } } -// attempt carries out one request synchronously, as the dispatcher's goroutine does. +// attempt carries out one request synchronously, as the dispatcher's goroutine does, up to the +// request: it does not follow the operation a delivered request started. func (f *dispatcherFixture) attempt(req git.ApplyRequest) git.ApplyResult { - return f.dispatcher.attempt(context.Background(), req) + if req.Token == "" { + req.Token = requestToken(f.dispatcher.Now()) + } + res, _ := f.dispatcher.attempt(context.Background(), req) + return res +} + +// carryOut runs the request and follows its operation on a goroutine of its own, as RequestApply +// does, and returns its reports; the channel is closed once it reports nothing more. +func (f *dispatcherFixture) carryOut(ctx context.Context, req git.ApplyRequest) <-chan git.ApplyResult { + reports := make(chan git.ApplyResult, 64) + req.Report = func(res git.ApplyResult) { reports <- res } + go func() { + defer close(reports) + f.dispatcher.carryOut(ctx, req) + }() + return reports +} + +// next is the next report, failing the test when none comes. +func next(t *testing.T, reports <-chan git.ApplyResult) git.ApplyResult { + t.Helper() + select { + case res := <-reports: + return res + case <-time.After(5 * time.Second): + t.Fatal("no report") + return git.ApplyResult{} + } +} + +// quiet asserts that no report comes for a while. +func quiet(t *testing.T, reports <-chan git.ApplyResult, why string) { + t.Helper() + select { + case res, open := <-reports: + if open { + t.Fatalf("%s: reported %s", why, res.Outcome) + } + t.Fatalf("%s: stopped reporting", why) + case <-time.After(50 * time.Millisecond): + } } func (f *dispatcherFixture) count(reason v1alpha3.ApplyReason, outcome string) int64 { @@ -188,25 +315,42 @@ func (f *dispatcherFixture) events() []string { } } +// patches lists the writes to the source cluster, as "resource namespace", with "(suspend=...)" +// for a server-side apply of a Kustomization's spec.suspend. func (f *dispatcherFixture) patches() []string { var out []string for _, action := range f.source.client.Actions() { - if action.GetVerb() == "patch" { - out = append(out, action.GetResource().Resource+" "+action.GetNamespace()) + if action.GetVerb() != "patch" { + continue + } + write := action.GetResource().Resource + " " + action.GetNamespace() + if patch := action.(k8stesting.PatchActionImpl); patch.PatchType == types.ApplyPatchType { + var body map[string]any + _ = json.Unmarshal(patch.Patch, &body) + suspend, _, _ := unstructured.NestedBool(body, "spec", "suspend") + write += fmt.Sprintf(" (suspend=%t)", suspend) } + out = append(out, write) } return out } -// behindSource is a Kustomization flux-system/checkout whose GitRepository on testBranch holds an -// older commit than testCommit, so any request that is sent stamps the GitRepository. +// behindSource is a suspended Kustomization flux-system/checkout whose GitRepository on testBranch +// holds an older commit than testCommit, so a request fetches before it resumes. func behindSource() []runtime.Object { + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + _ = unstructured.SetNestedField(ks.Object, true, "spec", "suspend") return []runtime.Object{ - kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), + ks, gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+olderCommit), } } +// fetchedAndResumed is the writes of a request that fetched and resumed flux-system/checkout. +var fetchedAndResumed = []string{ + "gitrepositories " + fluxSystemNS, "kustomizations " + fluxSystemNS + " (suspend=false)", +} + // TestDispatcher_AsksOnceForEveryTarget: one attempt asks its applier once for all its targets, // records each target's own reason, and looks the applier up in its source cluster. func TestDispatcher_AsksOnceForEveryTarget(t *testing.T) { @@ -225,18 +369,9 @@ func TestDispatcher_AsksOnceForEveryTarget(t *testing.T) { res := f.attempt(req) - assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyDelivered}, res) - assert.Equal(t, []string{"gitrepositories " + fluxSystemNS}, f.patches(), "one stamp for the applier") + assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyRequested}, res) + assert.Equal(t, fetchedAndResumed, f.patches(), "one fetch and one resume for the applier") assert.Equal(t, []string{v1alpha3.DefaultClusterProviderName}, f.source.clusters) - for name, want := range map[string]v1alpha3.ApplyReason{ - "a": v1alpha3.ApplyReasonRefusal, "b": v1alpha3.ApplyReasonCatchUp, - } { - got, ok := f.dispatcher.LastRequest(ref(name), uid(name)) - require.True(t, ok, name) - assert.Equal(t, testCommit, got.RequestedCommit, name) - assert.Equal(t, want, got.Reason, name) - assert.True(t, got.RequestedAt.Equal(&metav1.Time{Time: stampTime}), name) - } assert.ElementsMatch(t, []string{ "Normal ApplyRequested asked Kustomization flux-system/checkout to apply 4f2c1ab9 (Refusal)", "Normal ApplyRequested asked Kustomization flux-system/checkout to apply 4f2c1ab9 (CatchUp)", @@ -249,20 +384,27 @@ func TestDispatcher_AsksOnceForEveryTarget(t *testing.T) { // namespace and so names one in its own. func TestDispatcher_LooksTheApplierUpInItsCluster(t *testing.T) { own := kustomizationApplier("prod-eu-1", targetNS, "own") + // Suspended, as the Kustomization its trigger holds is. A resumed one is reconciled by the + // simulated kustomize-controller as soon as the fetch lands a new artifact, and that status write + // can move its resourceVersion between the resume's read and its apply: the resume is then + // refused with a Conflict and made again, which is correct, but shows up here as two resumes. + ks := kustomizationObj(targetNS, "own", gitRepoSourceRef("own-source")) + _ = unstructured.SetNestedField(ks.Object, true, "spec", "suspend") f := newDispatcherFixture(t, []client.Object{gitTarget("c", &fluxmeta.NamespacedObjectReference{Name: "own"}, "prod-eu-1")}, - kustomizationObj(targetNS, "own", gitRepoSourceRef("own-source")), + ks, gitRepositoryObj(targetNS, "own-source", testRepoURL, branchRef(testBranch), ""), ) - assert.Equal(t, git.ApplyDelivered, f.attempt(request(own, v1alpha3.ApplyReasonForeignCommit, "c")).Outcome) + assert.Equal(t, git.ApplyRequested, f.attempt(request(own, v1alpha3.ApplyReasonForeignCommit, "c")).Outcome) assert.Equal(t, []string{"prod-eu-1"}, f.source.clusters) - assert.Equal(t, []string{"gitrepositories " + targetNS}, f.patches()) + assert.Equal(t, []string{"gitrepositories " + targetNS, "kustomizations " + targetNS + " (suspend=false)"}, + f.patches()) } -// appliedFrom puts a GitTarget's applier in the cluster another ClusterProvider names. +// appliedFrom puts the applier of a GitTarget's trigger in the cluster another ClusterProvider names. func appliedFrom(target *v1alpha3.GitTarget, cluster string) *v1alpha3.GitTarget { - target.Spec.Applier.ClusterProviderRef = &fluxmeta.LocalObjectReference{Name: cluster} + target.Annotations[testTriggerClusterAnnotation] = cluster return target } @@ -279,7 +421,7 @@ func TestDispatcher_AsksAnApplierInItsOwnCluster(t *testing.T) { behindSource()..., ) - assert.Equal(t, git.ApplyDelivered, f.attempt(request(hub, v1alpha3.ApplyReasonForeignCommit, "a")).Outcome) + assert.Equal(t, git.ApplyRequested, f.attempt(request(hub, v1alpha3.ApplyReasonForeignCommit, "a")).Outcome) assert.Equal(t, []string{"hub"}, f.source.clusters) // The same Kustomization asked as if it lived in the source cluster is not this target's applier. @@ -314,7 +456,7 @@ func TestDispatcher_AFailureIsReportedAndNeverRetried(t *testing.T) { f.source.failing = true res := f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) - assert.Equal(t, git.ApplyFailed, res.Outcome) + assert.Equal(t, git.ApplyRequestFailed, res.Outcome) require.Error(t, res.Err) assert.Len(t, f.source.clusters, 1, "one attempt, no retry") events := f.events() @@ -326,15 +468,13 @@ func TestDispatcher_AFailureIsReportedAndNeverRetried(t *testing.T) { retry := request(checkout, v1alpha3.ApplyReasonForeignCommit, "a") retry.PriorFailures = 1 - assert.Equal(t, git.ApplyFailed, f.attempt(retry).Outcome) + assert.Equal(t, git.ApplyRequestFailed, f.attempt(retry).Outcome) assert.Empty(t, f.events(), "a failure already reported is not reported again") assert.Equal(t, int64(2), f.count(v1alpha3.ApplyReasonForeignCommit, outcomeFailed)) - _, ok := f.dispatcher.LastRequest(ref("a"), uid("a")) - assert.False(t, ok) } // TestDispatcher_ASourceMismatchIsRejectedAndNeverStamped is the first review's finding 2: -// ApplierConfigured said SourceMismatch, and a foreign commit stamped the other branch's source. +// the applier was reported SourceMismatch, and a foreign commit stamped the other branch's source. func TestDispatcher_ASourceMismatchIsRejectedAndNeverStamped(t *testing.T) { f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, @@ -371,7 +511,7 @@ func TestDispatcher_ATargetItsClusterProviderDeniesAsksNothing(t *testing.T) { // TestDispatcher_WithdrawnConsentAsksNothing is the executor's half of the second review's first // finding. A Refusal whose target withdrew consent used to become a CatchUp sent at once, which // fetched a behind artifact without the quiet period. Now it asks nothing, even with the artifact -// behind, and the worker decides what is still owed. +// behind, and the worker decides what is still pending. func TestDispatcher_WithdrawnConsentAsksNothing(t *testing.T) { suspended := reapplying(gitTarget("a", checkoutRef(), "")) suspended.Spec.Suspend = true @@ -396,12 +536,15 @@ func TestDispatcher_WithdrawnConsentAsksNothing(t *testing.T) { // third finding is the first case: a recreated GitTarget executed its predecessor's request. func TestDispatcher_ATargetThatChangedAsksNothing(t *testing.T) { for name, change := range map[string]func(*v1alpha3.GitTarget){ - "it was deleted and recreated": func(g *v1alpha3.GitTarget) { g.UID = "uid-recreated" }, - "it moved to another branch": func(g *v1alpha3.GitTarget) { g.Spec.Branch = "release" }, - "it names another Kustomization": func(g *v1alpha3.GitTarget) { g.Spec.Applier.KustomizationRef.Name = "other" }, - "it names no applier any more": func(g *v1alpha3.GitTarget) { g.Spec.Applier = nil }, - "it applies through another cluster": func(g *v1alpha3.GitTarget) { - g.Spec.ClusterProviderRef = &fluxmeta.LocalObjectReference{Name: "prod-eu-1"} + "it was deleted and recreated": func(g *v1alpha3.GitTarget) { g.UID = "uid-recreated" }, + "it moved to another branch": func(g *v1alpha3.GitTarget) { g.Spec.Branch = "release" }, + "it names another trigger": func(g *v1alpha3.GitTarget) { namesKustomization(g, fluxSystemNS, "other") }, + "it names no trigger any more": func(g *v1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = nil }, + "its trigger is gone": func(g *v1alpha3.GitTarget) { + g.Spec.ApplyTriggerRef.Kind = v1alpha3.ApplyTriggerKindArgoCD // no test annotation creates one + }, + "its trigger holds the applier in another cluster": func(g *v1alpha3.GitTarget) { + appliedFrom(g, "prod-eu-1") }, } { t.Run(name, func(t *testing.T) { @@ -413,14 +556,12 @@ func TestDispatcher_ATargetThatChangedAsksNothing(t *testing.T) { assert.Equal(t, git.ApplyDropped, res.Outcome) assert.Empty(t, f.patches()) - _, ok := f.dispatcher.LastRequest(ref("a"), target.UID) - assert.False(t, ok, "and records nothing for the incarnation it finds") }) } } // TestDispatcher_AStaleDecisionAsksNothing: the worker's decision is checked again just before the -// patch, and an attempt whose branch moved on or became busy reports Stale and stamps nothing. +// patch, and an attempt whose branch moved on or became busy reports Stale and resumes nothing. func TestDispatcher_AStaleDecisionAsksNothing(t *testing.T) { f := newDispatcherFixture(t, []client.Object{reapplying(gitTarget("a", checkoutRef(), ""))}, kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), @@ -435,13 +576,16 @@ func TestDispatcher_AStaleDecisionAsksNothing(t *testing.T) { assert.Equal(t, int64(1), f.count(v1alpha3.ApplyReasonRefusal, outcomeStale)) } -// TestDispatcher_CatchUpAlreadyCurrent: a CatchUp the applier already satisfies counts as sent and -// records nothing, since nothing was asked of it. +// TestDispatcher_CatchUpAlreadyCurrent: a CatchUp the applier already satisfies (suspended, with +// the commit in its artifact and applied) counts as sent and records nothing, since nothing was +// asked of it. func TestDispatcher_CatchUpAlreadyCurrent(t *testing.T) { + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + _ = unstructured.SetNestedField(ks.Object, true, "spec", "suspend") + _ = unstructured.SetNestedField(ks.Object, "main@sha1:"+testCommit, "status", "lastAppliedRevision") f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, - kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), - gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+testCommit), + ks, gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+testCommit), ) res := f.attempt(request(checkout, v1alpha3.ApplyReasonCatchUp, "a")) @@ -449,27 +593,6 @@ func TestDispatcher_CatchUpAlreadyCurrent(t *testing.T) { assert.Equal(t, git.ApplyCurrent, res.Outcome) assert.Empty(t, f.patches()) assert.Empty(t, f.events()) - _, ok := f.dispatcher.LastRequest(ref("a"), uid("a")) - assert.False(t, ok) -} - -// TestDispatcher_TheLedgerIsPerIncarnation: a GitTarget deleted and recreated under the same name -// does not inherit its predecessor's last request, and a forgotten one holds nothing. -func TestDispatcher_TheLedgerIsPerIncarnation(t *testing.T) { - f := newDispatcherFixture( - t, - []client.Object{gitTarget("a", checkoutRef(), "")}, - behindSource()...) - require.Equal(t, git.ApplyDelivered, - f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")).Outcome) - - _, ok := f.dispatcher.LastRequest(ref("a"), uid("a")) - assert.True(t, ok) - _, ok = f.dispatcher.LastRequest(ref("a"), "the-recreated-one") - assert.False(t, ok, "a recreated GitTarget does not inherit the last request") - f.dispatcher.Forget(ref("a")) - _, ok = f.dispatcher.LastRequest(ref("a"), uid("a")) - assert.False(t, ok) } // TestDispatcher_RequestApplyReportsFromItsOwnGoroutine: RequestApply returns at once and the @@ -485,7 +608,7 @@ func TestDispatcher_RequestApplyReportsFromItsOwnGoroutine(t *testing.T) { f.dispatcher.RequestApply(req) res := <-reports - assert.Equal(t, git.ApplyFailed, res.Outcome, "not running yet") + assert.Equal(t, git.ApplyRequestFailed, res.Outcome, "not running yet") require.ErrorIs(t, res.Err, errNotRunning) ctx, cancel := context.WithCancel(context.Background()) @@ -501,7 +624,7 @@ func TestDispatcher_RequestApplyReportsFromItsOwnGoroutine(t *testing.T) { }, 5*time.Second, time.Millisecond) f.dispatcher.RequestApply(req) - assert.Equal(t, git.ApplyDelivered, (<-reports).Outcome) + assert.Equal(t, git.ApplyRequested, (<-reports).Outcome) cancel() <-done } @@ -515,6 +638,9 @@ func TestStrongerApplyReason(t *testing.T) { {v1alpha3.ApplyReasonRefusal, v1alpha3.ApplyReasonForeignCommit, v1alpha3.ApplyReasonForeignCommit}, {v1alpha3.ApplyReasonForeignCommit, v1alpha3.ApplyReasonRefusal, v1alpha3.ApplyReasonForeignCommit}, {"", v1alpha3.ApplyReasonCatchUp, v1alpha3.ApplyReasonCatchUp}, + {v1alpha3.ApplyReasonRefusal, v1alpha3.ApplyReasonRepair, v1alpha3.ApplyReasonRepair}, + {v1alpha3.ApplyReasonRepair, v1alpha3.ApplyReasonCatchUp, v1alpha3.ApplyReasonRepair}, + {v1alpha3.ApplyReasonRepair, v1alpha3.ApplyReasonForeignCommit, v1alpha3.ApplyReasonForeignCommit}, } for _, tt := range tests { assert.Equal(t, tt.want, git.StrongerApplyReason(tt.older, tt.newer), "%s then %s", tt.older, tt.newer) @@ -522,59 +648,295 @@ func TestStrongerApplyReason(t *testing.T) { } func TestDescribeCommit(t *testing.T) { - assert.Equal(t, "4f2c1ab9", describeCommit(testCommit)) - assert.Equal(t, "abc", describeCommit("abc")) - assert.Equal(t, "the branch, which the remote does not carry", describeCommit("")) + assert.Equal(t, "4f2c1ab9", DescribeCommit(testCommit)) + assert.Equal(t, "abc", DescribeCommit("abc")) + assert.Equal(t, "the branch, which the remote does not carry", DescribeCommit("")) } -// TestDispatcher_ADeliveredAttemptStaysOutUntilFluxFinished: the worker's attempt is the apply -// window, so a delivered one is reported only once Flux has finished the apply, or by time. -func TestDispatcher_ADeliveredAttemptStaysOutUntilFluxFinished(t *testing.T) { +// TestDispatcher_ARequestedApplyIsFollowedUntilFluxFinished: the worker's attempt is the apply +// window, so a requested apply reports each step of the suspended lifecycle: Requested once the +// Kustomization is resumed, Running until the reconcile that read the resume's token ended, Settling +// once it is suspended again, and Succeeded only once that suspend's token is handled. +func TestDispatcher_ARequestedApplyIsFollowedUntilFluxFinished(t *testing.T) { f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) - f.dispatcher.ApplyWindowTimeout = time.Minute - f.dispatcher.HandledPollInterval = 5 * time.Millisecond + held := f.sim.HoldNextApply() + + reports := f.carryOut(context.Background(), request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyRequested}, next(t, reports)) + <-held + assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyRunning}, next(t, reports)) + quiet(t, reports, "while Flux is still applying") + + f.sim.Release() + settling := next(t, reports) + assert.Equal(t, git.ApplySettling, settling.Outcome) + require.NotEmpty(t, settling.SuspendToken) + done := next(t, reports) + assert.Equal(t, git.ApplyResult{ + ID: 1, + Outcome: git.ApplySucceeded, + SuspendToken: settling.SuspendToken, + AppliedRevision: "main@sha1:" + testCommit, + }, done) + _, open := <-reports + assert.False(t, open, "and that ends the attempt") + assert.Equal(t, []string{"flux-system/checkout main@sha1:" + testCommit}, f.sim.Applied()) + assert.True(t, suspended(f.sim.Kustomization(checkoutKey)), "the Kustomization is left suspended") +} - done := make(chan git.ApplyResult, 1) - go func() { done <- f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) }() +// TestDispatcher_AnApplyPastItsBudgetIsUnresolvedAndStillFollowed is the third review's fourth +// finding: when the wait ran out, the attempt was reported delivered, and the window closed while +// the apply could still run. Now the operation is reported Unresolved, once, the Kustomization is +// suspended at once (which stops what would follow, not the apply running), and the operation is +// followed on until that suspend's token is handled. +func TestDispatcher_AnApplyPastItsBudgetIsUnresolvedAndStillFollowed(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + f.dispatcher.UnresolvedAfter = 20 * time.Millisecond + f.dispatcher.Now = time.Now + held := f.sim.HoldNextApply() + + reports := f.carryOut(context.Background(), request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + <-held + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + assert.Equal(t, git.ApplyUnresolved, next(t, reports).Outcome) + resuspended := next(t, reports) + assert.Equal(t, git.ApplyUnresolved, resuspended.Outcome, "Unresolved holds") + require.NotEmpty(t, resuspended.SuspendToken, "suspended at once") + assert.True(t, suspended(f.sim.Kustomization(checkoutKey))) + quiet(t, reports, "an unresolved operation is not closed by time") + events := f.events() + require.Len(t, events, 2) + assert.True(t, strings.HasPrefix(events[1], "Warning ApplyUnresolved Kustomization flux-system/checkout "+ + "has not completed the apply of 4f2c1ab9 (ForeignCommit) within 20ms"), events[1]) + + f.sim.Release() + done := next(t, reports) + assert.Equal(t, git.ApplySucceeded, done.Outcome, "it resolves when the suspend's token is handled") + assert.Equal(t, resuspended.SuspendToken, done.SuspendToken) +} - require.Eventually(t, func() bool { return len(f.patches()) == 1 }, 5*time.Second, 5*time.Millisecond) - select { - case res := <-done: - t.Fatalf("reported %s before Flux finished the apply", res.Outcome) - case <-time.After(50 * time.Millisecond): +// TestDispatcher_AFailedApplyIsSuspendedAgain: a failed apply is suspended like any other, which is +// what stops Flux's own retryInterval retry; the worker decides the retry, on a quiet branch. +func TestDispatcher_AFailedApplyIsSuspendedAgain(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + f.sim.FailApplies("ConfigMap/apps/x dry-run failed") + + reports := f.carryOut(context.Background(), request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + res := next(t, reports) + for !res.Outcome.Ends() { + res = next(t, reports) } + assert.Equal(t, git.ApplyFailed, res.Outcome) + require.ErrorContains(t, res.Err, "ConfigMap/apps/x dry-run failed") + assert.NotEmpty(t, res.SuspendToken) + assert.True(t, suspended(f.sim.Kustomization(checkoutKey)), "no retryInterval retry can fire") +} - setStatus(t, f.source.client, kustomizationGVR(), types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, - "main@sha1:"+testCommit, "lastAttemptedRevision") - select { - case res := <-done: - assert.Equal(t, git.ApplyDelivered, res.Outcome) - case <-time.After(5 * time.Second): - t.Fatal("not reported after Flux finished the apply") +// TestDispatcher_ADeletedTriggerEndsItsOperation: a trigger being deleted hands its Kustomization +// back itself, so its operation completes as Failed, and nothing more is written to the +// Kustomization from here. +func TestDispatcher_ADeletedTriggerEndsItsOperation(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + held := f.sim.HoldNextApply() + reports := f.carryOut(context.Background(), request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + <-held + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + + trigger := &v1alpha3.FluxTrigger{} + require.NoError(t, f.control.Get(context.Background(), + client.ObjectKey{Namespace: targetNS, Name: "checkout"}, trigger)) + require.NoError(t, f.control.Delete(context.Background(), trigger)) + res := next(t, reports) + assert.Equal(t, git.ApplyFailed, res.Outcome) + require.ErrorContains(t, res.Err, "the trigger is being deleted") + assert.Equal(t, fetchedAndResumed, f.patches(), "never suspended from here") +} + +// TestDispatcher_AnAbandonedOperationIsSuspendedBeforeItIsLetGo: once the worker stops following an +// attempt (its applier is no longer named), the dispatcher reports nothing more, but it does not let +// a resumed Kustomization go unguarded (review 4's B): it suspends it at once, and stops only once +// that suspend's token is handled. +func TestDispatcher_AnAbandonedOperationIsSuspendedBeforeItIsLetGo(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + held := f.sim.HoldNextApply() + abandoned := make(chan struct{}) + req := request(checkout, v1alpha3.ApplyReasonForeignCommit, "a") + req.Token = "T1" + req.Abandoned = abandoned + + reports := f.carryOut(context.Background(), req) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + <-held + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + assert.True(t, f.dispatcher.Following(checkout.Trigger, "T1")) + close(abandoned) + eventually(t, func() bool { return suspended(f.sim.Kustomization(checkoutKey)) }, "suspended at once") + quiet(t, reports, "still waiting for the suspend to be handled, behind the running apply") + assert.True(t, f.dispatcher.Following(checkout.Trigger, "T1"), "and it is still followed") + + f.sim.Release() + _, open := <-reports + assert.False(t, open, "nothing more is reported") + assert.True(t, tokenHandled(f.sim.Kustomization(checkoutKey), f.sim.Kustomization(checkoutKey). + GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]), "let go once the suspend is handled") + assert.False(t, f.dispatcher.Following(checkout.Trigger, "T1")) +} + +// TestDispatcher_AStopMidApplySuspendsTheKustomization is review 7's F1: an apply is in flight, and +// the operator is stopped (`helm upgrade`, a node drain). Nothing suspended the Kustomization, so it +// stayed resumed until the next process recovered the operation, and Flux could apply on its own in +// between. Now the stop suspends it once, without waiting for that to be handled: the next process +// recovers a suspend of its own it did not report. +func TestDispatcher_AStopMidApplySuspendsTheKustomization(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + held := f.sim.HoldNextApply() + req := request(checkout, v1alpha3.ApplyReasonForeignCommit, "a") + req.Token = "T1" + ctx, stop := context.WithCancel(context.Background()) + + reports := f.carryOut(ctx, req) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + <-held + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + require.False(t, suspended(f.sim.Kustomization(checkoutKey)), "resumed for the apply") + stop() + _, open := <-reports + assert.False(t, open, "nothing more is reported") + + assert.True(t, suspended(f.sim.Kustomization(checkoutKey)), "suspended as the process stops") + assert.True(t, heldByUs(f.sim.Kustomization(checkoutKey)), "under GitOps Reverser's field manager") + f.sim.Release() +} + +// TestDispatcher_AnUnheldTriggerIsNotResumed is review 4's F: the trigger's reconciler suspends the +// Kustomization on its own reads (an adoption, an external resume), and a resume between its read +// and its suspend would be suspended under the apply. So a request resumes only a Kustomization its +// trigger reports Held=True, read uncached; any other is a failed attempt the worker retries. +func TestDispatcher_AnUnheldTriggerIsNotResumed(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + trigger := &v1alpha3.FluxTrigger{} + require.NoError(t, f.control.Get(context.Background(), client.ObjectKey{Namespace: targetNS, Name: "checkout"}, + trigger)) + trigger.Status.Conditions[0].Status, trigger.Status.Conditions[0].Reason = metav1.ConditionFalse, "Suspending" + require.NoError(t, f.control.Update(context.Background(), trigger)) + + res := f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + + assert.Equal(t, git.ApplyRequestFailed, res.Outcome) + require.ErrorContains(t, res.Err, "does not hold its Kustomization suspended: it is Held=False, Suspending") + for _, write := range f.patches() { + assert.NotContains(t, write, "suspend=false", "nothing is resumed") } } -// TestDispatcher_AnApplyFluxNeverFinishesClosesByTime: Flux down, or a fetch that never lands, -// must not hold the applier's attempt out forever. -func TestDispatcher_AnApplyFluxNeverFinishesClosesByTime(t *testing.T) { +// TestDispatcher_ATriggerAnotherCameBeforeIsNotResumed is review 5's P2, the dispatcher's half: the +// recheck reads the triggers cached, and can miss one that came first in line since (created in the +// same second, after it by name, but with an apply of its own in flight). The hold check before the resume reads them +// uncached, and fails the attempt. +func TestDispatcher_ATriggerAnotherCameBeforeIsNotResumed(t *testing.T) { f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) - f.dispatcher.ApplyWindowTimeout = 30 * time.Millisecond - f.dispatcher.HandledPollInterval = 5 * time.Millisecond + own := &v1alpha3.FluxTrigger{} + require.NoError(t, f.control.Get(context.Background(), client.ObjectKey{Namespace: targetNS, Name: "checkout"}, + own)) + other := own.DeepCopy() + other.Namespace, other.ResourceVersion = "team-z", "" + other.Spec.KustomizationRef.Namespace = own.Applier().Namespace + other.Status.Operation = &v1alpha3.ApplyOperation{Phase: v1alpha3.ApplyOperationRunning, Token: "T0"} + other.Finalizers = []string{v1alpha3.FluxTriggerFinalizer} // the holder whose apply this is + scheme := runtime.NewScheme() + require.NoError(t, v1alpha3.AddToScheme(scheme)) + f.dispatcher.APIReader = fake.NewClientBuilder().WithScheme(scheme).WithObjects(own.DeepCopy(), other).Build() + + res := f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + + assert.Equal(t, git.ApplyRequestFailed, res.Outcome) + require.ErrorContains(t, res.Err, "no longer holds its Kustomization") + require.ErrorContains(t, res.Err, "held by FluxTrigger team-z/checkout, whose apply is in flight") + for _, write := range f.patches() { + assert.NotContains(t, write, "suspend=false", "nothing is resumed") + } +} + +// TestDispatcher_ARecoveredAttemptBeforeStartWaitsForIt is review 4's B at startup: a worker +// recovers its trigger's operation once, and one that failed because the dispatcher had not started +// yet left the Kustomization resumed with nobody following it. Now it is followed once Start runs. +func TestDispatcher_ARecoveredAttemptBeforeStartWaitsForIt(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, resumedSource(t, "T1")...) + f.sim.Requeue(checkoutKey) + reports := make(chan git.ApplyResult, 64) + req := recoveredRequest(&v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationRunning, Reason: v1alpha3.ApplyReasonRefusal, Commit: testCommit, Token: "T1", + }) + req.Report = func(res git.ApplyResult) { reports <- res } + + f.dispatcher.RequestApply(req) + quiet(t, reports, "nothing is reported before Start") + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + _ = f.dispatcher.Start(ctx) + }() + res := next(t, reports) + for !res.Outcome.Ends() { + res = next(t, reports) + } + assert.Equal(t, git.ApplySucceeded, res.Outcome) + assert.True(t, suspended(f.sim.Kustomization(checkoutKey)), "suspended again") + cancel() + <-done +} - assert.Equal(t, git.ApplyDelivered, f.attempt(request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")).Outcome) +// TestDispatcher_AnAbandonedOperationIsNoLongerFollowed: once the worker stops following an +// attempt (its applier is no longer named), the dispatcher stops reading it and reports nothing. +func TestDispatcher_AnAbandonedOperationIsNoLongerFollowed(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{argoTarget()}, manualApp()) + abandoned := make(chan struct{}) + req := request(argoPlatform, v1alpha3.ApplyReasonForeignCommit, "a") + req.Abandoned = abandoned + + reports := f.carryOut(context.Background(), req) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + close(abandoned) + _, open := <-reports + assert.False(t, open, "nothing more is reported") } -// argoTarget is the GitTarget "a", whose applier is the Application argocd/platform. +// TestDispatcher_AnApplierThatIsGoneFailsItsOperation: a deleted applier object runs nothing more, +// so its operation completes as Failed and the worker decides again. +func TestDispatcher_AnApplierThatIsGoneFailsItsOperation(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + held := f.sim.HoldNextApply() + reports := f.carryOut(context.Background(), request(checkout, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + <-held + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + + require.NoError(t, f.source.client.Resource(kustomizationGVR()).Namespace(fluxSystemNS). + Delete(context.Background(), "checkout", metav1.DeleteOptions{})) + res := next(t, reports) + assert.Equal(t, git.ApplyFailed, res.Outcome) + require.ErrorContains(t, res.Err, "the applier is gone") +} + +// argoTarget is the GitTarget "a", which names the ArgoCDTrigger "platform", holding the +// Application argocd/platform. func argoTarget() *v1alpha3.GitTarget { target := gitTarget("a", nil, "") - target.Spec.Applier = &v1alpha3.GitTargetApplier{ - ApplicationRef: &fluxmeta.NamespacedObjectReference{Namespace: argoNS, Name: "platform"}, + target.Annotations = map[string]string{testApplicationAnnotation: argoNS + "/platform"} + target.Spec.ApplyTriggerRef = &v1alpha3.ApplyTriggerReference{ + Kind: v1alpha3.ApplyTriggerKindArgoCD, + Name: "platform", } return target } -// argoPlatform is the applier argoTarget names. +// argoPlatform is the applier argoTarget's trigger holds. var argoPlatform = git.ApplierRef{ + Trigger: git.TriggerRef{Kind: v1alpha3.ApplyTriggerKindArgoCD, Namespace: targetNS, Name: "platform"}, Cluster: v1alpha3.DefaultClusterProviderName, Kind: v1alpha3.ApplierKindApplication, Namespace: argoNS, Name: "platform", } @@ -594,9 +956,18 @@ func (f *dispatcherFixture) updates() []string { func TestDispatcher_AsksAnApplicationToSync(t *testing.T) { f := newDispatcherFixture(t, []client.Object{reapplying(argoTarget())}, manualApp()) - res := f.attempt(request(argoPlatform, v1alpha3.ApplyReasonRefusal, "a")) + req := request(argoPlatform, v1alpha3.ApplyReasonRefusal, "a") + req.Token = "T1" + recorded := false + req.Record = func(context.Context) error { + assert.Empty(t, f.updates(), "recorded before the sync is set") + recorded = true + return nil + } + res := f.attempt(req) - assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyDelivered}, res) + assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyRequested}, res) + assert.True(t, recorded) assert.Equal(t, []string{"applications " + argoNS}, f.updates()) assert.Empty(t, f.patches(), "no Flux object is stamped") app, err := f.source.client.Resource(applicationGVR()).Namespace(argoNS). @@ -604,6 +975,8 @@ func TestDispatcher_AsksAnApplicationToSync(t *testing.T) { require.NoError(t, err) revision, _, _ := unstructured.NestedString(app.Object, "operation", "sync", "revision") assert.Equal(t, testCommit, revision) + operation, _, _ := unstructured.NestedMap(app.Object, "operation") + assert.Equal(t, "T1", operationToken(operation), "the operation carries the request's token") assert.Equal(t, []string{ "Normal ApplyRequested asked Application argocd/platform to apply 4f2c1ab9 (Refusal)", }, f.events()) @@ -636,21 +1009,249 @@ func TestDispatcher_AnApplicationIsNotTheKustomizationOfItsName(t *testing.T) { } // TestDispatcher_AnApplicationAttemptWaitsWithinItsWindow: an Argo CD attempt may wait out an -// operation already running, so it is bounded by the apply window on top of the attempt timeout. +// operation already running, so it is bounded by the operation wait on top of the attempt timeout. func TestDispatcher_AnApplicationAttemptWaitsWithinItsWindow(t *testing.T) { running := manualApp() _ = unstructured.SetNestedMap(running.Object, theirOperation(), "operation") f := newDispatcherFixture(t, []client.Object{argoTarget()}, running) f.dispatcher.AttemptTimeout = 30 * time.Millisecond - f.dispatcher.ApplyWindowTimeout = 100 * time.Millisecond - f.dispatcher.HandledPollInterval = time.Millisecond + f.dispatcher.OperationWait = 100 * time.Millisecond f.dispatcher.Now = time.Now res := f.attempt(request(argoPlatform, v1alpha3.ApplyReasonForeignCommit, "a")) - assert.Equal(t, git.ApplyFailed, res.Outcome) + assert.Equal(t, git.ApplyRequestFailed, res.Outcome) require.ErrorContains(t, res.Err, "has an operation in progress") require.NotErrorIs(t, res.Err, context.DeadlineExceeded, "the wait ends by the operation wait, not by the shorter attempt timeout") assert.Empty(t, f.updates()) } + +// argoOperationState sets the Application's operation state to the operation it carries, at phase, +// and clears the operation, as Argo CD's controller does once it completes one. +func argoOperationState(t *testing.T, f *dispatcherFixture, phase string) { + t.Helper() + app := readApp(t, f.source.client) + operation, _, _ := unstructured.NestedMap(app.Object, "operation") + if operation == nil { + operation, _, _ = unstructured.NestedMap(app.Object, "status", "operationState", "operation") + } + _ = unstructured.SetNestedMap(app.Object, map[string]any{ + "operation": operation, "phase": phase, "message": "a hook failed", + }, "status", "operationState") + if phase != "Running" { + unstructured.RemoveNestedField(app.Object, "operation") + } + writeApp(t, f.source.client, app) +} + +// TestDispatcher_AnApplicationsSyncIsFollowedToItsOutcome: an Argo CD sync reports Running once the +// controller started it, and its outcome once it completed; a failed one records ApplyFailed for the +// first failure of a run, so the worker's retry of it is not reported twice. +func TestDispatcher_AnApplicationsSyncIsFollowedToItsOutcome(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{argoTarget()}, manualApp()) + + reports := f.carryOut(context.Background(), request(argoPlatform, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + argoOperationState(t, f, "Running") + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + argoOperationState(t, f, "Failed") + res := next(t, reports) + assert.Equal(t, git.ApplyFailed, res.Outcome) + require.ErrorContains(t, res.Err, "the sync ended Failed: a hook failed") + events := f.events() + require.Len(t, events, 2) + assert.True(t, strings.HasPrefix(events[1], "Warning ApplyFailed Application argocd/platform did not "+ + "complete the apply of 4f2c1ab9 (ForeignCommit)"), events[1]) + + retry := request(argoPlatform, v1alpha3.ApplyReasonForeignCommit, "a") + retry.PriorFailures = 1 + reports = f.carryOut(context.Background(), retry) + assert.Equal(t, git.ApplyRequested, next(t, reports).Outcome) + argoOperationState(t, f, "Error") + assert.Equal(t, git.ApplyFailed, next(t, reports).Outcome) + assert.Len(t, f.events(), 1, "only the ApplyRequested: a failure already reported is not reported again") +} + +// TestDispatcher_AnUnansweredApplicationUpdateIsFollowed is review 5's first finding, end to end: +// an update the API server made but whose answer timed out is no RequestFailed (which would have +// the worker withdraw its record and close the apply window while Argo CD syncs). It is reported +// Requested, and followed until the sync completes. One that was never made is Failed once a write +// moved the Application on from the version it was made from, so the worker asks again. +func TestDispatcher_AnUnansweredApplicationUpdateIsFollowed(t *testing.T) { + for name, landed := range map[string]bool{"made": true, "never made": false} { + t.Run(name, func(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{argoTarget()}, manualApp()) + var answered atomic.Bool + f.source.client.PrependReactor("update", "applications", + func(action k8stesting.Action) (bool, runtime.Object, error) { + if answered.Swap(true) { + return false, nil, nil + } + if landed { + update, _ := action.(k8stesting.UpdateActionImpl) + tracker := f.source.client.Tracker() + require.NoError(t, tracker.Update(applicationGVR(), update.GetObject(), argoNS)) + } + return true, nil, context.DeadlineExceeded + }) + + reports := f.carryOut(context.Background(), request(argoPlatform, v1alpha3.ApplyReasonForeignCommit, "a")) + assert.Equal(t, git.ApplyResult{ID: 1, Outcome: git.ApplyRequested}, next(t, reports)) + if !landed { + res := next(t, reports) + assert.Equal(t, git.ApplyFailed, res.Outcome) + require.ErrorContains(t, res.Err, "was never made") + assert.Contains(t, readApp(t, f.source.client).GetAnnotations(), argoRefreshAnnotation, + "settled by a write that moved the Application on") + return + } + argoOperationState(t, f, "Running") + assert.Equal(t, git.ApplyRunning, next(t, reports).Outcome) + argoOperationState(t, f, "Succeeded") + assert.Equal(t, git.ApplySucceeded, next(t, reports).Outcome) + }) + } +} + +// TestDispatcher_AnApplyOfAnotherRevisionIsCounted is the residual race: somebody fetched the +// source between the recheck and the reconcile, so Flux applied another revision. It is still an +// operation that succeeded, and it is logged and counted. +func TestDispatcher_AnApplyOfAnotherRevisionIsCounted(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}) + req := request(checkout, v1alpha3.ApplyReasonForeignCommit, "a") + op := &operation{log: logr.Discard()} + + res := f.dispatcher.completed(context.Background(), req, op, Progress{ + Outcome: git.ApplySucceeded, AppliedRevision: "main@sha1:" + olderCommit, + Message: "Flux applied main@sha1:" + olderCommit + ", not commit " + testCommit, + }, time.Second) + + assert.Equal(t, git.ApplySucceeded, res.Outcome) + assert.Contains(t, res.Message, "not commit") + mismatches, _ := telemetry.CollectInt64Sum(f.reader, "gitopsreverser_apply_revision_mismatches_total", + map[string]string{"applier_kind": applierKindKustomization}) + assert.Equal(t, int64(1), mismatches) +} + +// TestApplyVerdict_ReadsTheReconcileThatHandledTheToken pins the outcome table. +func TestApplyVerdict_ReadsTheReconcileThatHandledTheToken(t *testing.T) { + ks := func(status metav1.ConditionStatus, reason, applied string) *unstructured.Unstructured { + obj := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + fluxsim.SetReady(obj, status, reason, "message") + _ = unstructured.SetNestedField(obj.Object, applied, "status", "lastAppliedRevision") + return obj + } + commit := "main@sha1:" + testCommit + assert.Equal(t, Progress{Outcome: git.ApplySucceeded, AppliedRevision: commit}, + applyVerdict(ks(metav1.ConditionTrue, "ReconciliationSucceeded", commit), testCommit)) + other := applyVerdict(ks(metav1.ConditionTrue, "ReconciliationSucceeded", "main@sha1:"+olderCommit), testCommit) + assert.Equal(t, git.ApplySucceeded, other.Outcome) + assert.Contains(t, other.Message, "not commit "+testCommit) + dependency := applyVerdict(ks(metav1.ConditionFalse, fluxmeta.DependencyNotReadyReason, commit), testCommit) + assert.Equal(t, git.ApplyFailed, dependency.Outcome) + assert.Contains(t, dependency.Message, "a dependency is not ready") + failed := applyVerdict(ks(metav1.ConditionFalse, "ReconciliationFailed", commit), testCommit) + assert.Equal(t, git.ApplyFailed, failed.Outcome) + assert.Contains(t, failed.Message, "ReconciliationFailed") + none := applyVerdict(kustomizationObj(fluxSystemNS, "checkout", nil), testCommit) + assert.Equal(t, git.ApplyFailed, none.Outcome) +} + +// recoveredRequest follows the recorded operation instead of asking. +func recoveredRequest(recorded *v1alpha3.ApplyOperation) git.ApplyRequest { + req := request(checkout, recorded.Reason, "a") + req.Commit, req.Token, req.Recovered = recorded.Commit, recorded.Token, recorded + return req +} + +// followToEnd runs RequestApply and returns the report that ends the attempt. +func (f *dispatcherFixture) followToEnd(t *testing.T, req git.ApplyRequest) git.ApplyResult { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = f.dispatcher.Start(ctx) }() + require.Eventually(t, func() bool { + f.dispatcher.runMu.RLock() + defer f.dispatcher.runMu.RUnlock() + return f.dispatcher.run != nil + }, 5*time.Second, time.Millisecond) + reports := make(chan git.ApplyResult, 64) + req.Report = func(res git.ApplyResult) { reports <- res } + f.dispatcher.RequestApply(req) + res := next(t, reports) + for !res.Outcome.Ends() { + res = next(t, reports) + } + return res +} + +// resumedSource is flux-system/checkout resumed with token, as a process that stopped mid-apply +// leaves it, with its GitRepository at testCommit. +func resumedSource(t *testing.T, token string) []runtime.Object { + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + ks = fluxsim.OwnSuspend(t, ks, false, SuspendFieldManager, metav1.ManagedFieldsOperationApply) + ks.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: token}) + ks.SetGeneration(2) + return []runtime.Object{ + ks, gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+testCommit), + } +} + +// TestDispatcher_ARecoveredOperationIsFollowedAndSuspended is step 5: a process stopped while the +// Kustomization was resumed for its operation. The next one follows that operation by its recorded +// token, asks for nothing, and suspends the Kustomization again before the operation completes. +func TestDispatcher_ARecoveredOperationIsFollowedAndSuspended(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, resumedSource(t, "T1")...) + f.sim.Requeue(checkoutKey) // the reconcile the resume enqueued, still to run + + res := f.followToEnd(t, recoveredRequest(&v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationRunning, Reason: v1alpha3.ApplyReasonRefusal, Commit: testCommit, Token: "T1", + })) + + assert.Equal(t, git.ApplySucceeded, res.Outcome) + assert.NotEmpty(t, res.SuspendToken) + assert.True(t, suspended(f.sim.Kustomization(checkoutKey)), "suspended again") + for _, write := range f.patches() { + assert.NotContains(t, write, "suspend=false", "nothing is resumed for a recovered operation") + assert.NotContains(t, write, "gitrepositories", "and nothing is fetched") + } + assert.Empty(t, f.events(), "and it records no ApplyRequested") +} + +// TestDispatcher_ARecoveredOperationThatNeverResumedFails: the record is written before the resume, +// so a process can stop between the two. Its token is nowhere on the Kustomization; no outcome can +// be read, so it ends Failed (after a suspend) and the worker decides again. +func TestDispatcher_ARecoveredOperationThatNeverResumedFails(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, behindSource()...) + + res := f.followToEnd(t, recoveredRequest(&v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationRequested, Reason: v1alpha3.ApplyReasonCatchUp, Commit: testCommit, Token: "T1", + })) + + assert.Equal(t, git.ApplyFailed, res.Outcome) + require.ErrorContains(t, res.Err, "no sign of its resume") + assert.Empty(t, f.sim.Applied()) + assert.True(t, suspended(f.sim.Kustomization(checkoutKey))) +} + +// TestDispatcher_AnApplyStartedElsewhereIsFollowedByItsSuspend: the trigger's reconciler recorded a +// `flux resume` it suspended again, with the suspend token and no request token; following it ends +// once that suspend is handled. +func TestDispatcher_AnApplyStartedElsewhereIsFollowedByItsSuspend(t *testing.T) { + objs := resumedSource(t, "F1") + _ = unstructured.SetNestedField(objs[0].(*unstructured.Unstructured).Object, true, "spec", "suspend") + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, objs...) + f.sim.Requeue(checkoutKey) + + res := f.followToEnd(t, recoveredRequest(&v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationUnresolved, SuspendToken: "F1", + })) + + assert.Equal(t, git.ApplyFailed, res.Outcome, + "the Kustomization reports no Ready: nothing of that apply can be read") + assert.Equal(t, "F1", res.SuspendToken) + for _, write := range f.patches() { + assert.NotContains(t, write, "kustomizations", "the suspend was the reconciler's; nothing is written") + } +} diff --git a/internal/applier/flux.go b/internal/applier/flux.go index 6f6da7258..3589cc8af 100644 --- a/internal/applier/flux.go +++ b/internal/applier/flux.go @@ -5,11 +5,13 @@ package applier import ( "context" "encoding/json" + "errors" "fmt" "strings" "time" fluxmeta "github.com/fluxcd/pkg/apis/meta" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -17,6 +19,7 @@ import ( "k8s.io/client-go/dynamic" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/repourl" ) @@ -40,150 +43,583 @@ const ( kindGitRepository = "GitRepository" ) -// What a Flux request stamped. OutcomeCurrent stamps nothing: a CatchUp or ForeignCommit found the -// artifact already at the commit, so the Kustomization was already woken by it. -const ( - // OutcomeStampedSource stamped the GitRepository: its artifact did not hold the commit, and the - // new artifact it produces wakes the Kustomization by itself. - OutcomeStampedSource Outcome = "StampedSource" - // OutcomeStampedKustomization stamped the Kustomization: the artifact already held the commit, - // so a source stamp would produce no new artifact and wake nothing. - OutcomeStampedKustomization Outcome = "StampedKustomization" -) +// OutcomeResumed resumed the Kustomization for one apply. OutcomeCurrent asks nothing: a CatchUp or +// ForeignCommit found it suspended, with the commit both in its artifact and applied. +const OutcomeResumed Outcome = "Resumed" + +// SuspendFieldManager is the field manager GitOps Reverser holds a Kustomization's spec.suspend and +// its reconcile.fluxcd.io/requestedAt with. Every write of them is a server-side apply under it, so +// the Kustomization's managedFields say who holds it. +const SuspendFieldManager = "gitops-reverser" -// Flux asks a Flux Kustomization to apply, in one source cluster. +// ReasonSuspendOwnedElsewhere is Configured=False on a FluxTrigger whose Kustomization's +// spec.suspend is applied by another field manager: whatever applies the Kustomization's manifest +// (a parent Kustomization, Helm, flux-operator) sets it, and would resume it on its next apply. +const ReasonSuspendOwnedElsewhere = "SuspendOwnedElsewhere" + +// Flux holds a Flux Kustomization suspended, and resumes it for each apply GitOps Reverser asks of +// it, in the applier's cluster. See "The Flux applier, suspended" in +// docs/design/apply-trigger.md: every fact about kustomize-controller this relies on is +// listed there, read in v1.9.6. type Flux struct { client dynamic.Interface - // Now is the clock the requestedAt stamp is taken from. + // Now is the clock request tokens are taken from. Now func() time.Time // CanI is the access check Validate runs for patch. NewFlux sets it to a // SelfSubjectAccessReview through the same client, so it is answered for the identity that // will make the request. CanI AccessCheck + // PollInterval is how often a request reads the GitRepository while it fetches. + PollInterval time.Duration } -// NewFlux returns a Flux client over a source cluster's dynamic client. +// NewFlux returns a Flux client over the dynamic client of the applier's cluster. func NewFlux(client dynamic.Interface) *Flux { - return &Flux{client: client, Now: time.Now, CanI: SelfSubjectAccessCheck(client)} + return &Flux{client: client, Now: time.Now, CanI: SelfSubjectAccessCheck(client), PollInterval: fetchPollInterval} } -// Request asks the Kustomization ref to apply commit, the way `flux reconcile kustomization -// --with-source` does, minus the waiting: Request only asks, and Handled tells when Flux is done. +// fetchPollInterval is how often a request reads the GitRepository while it fetches: a fetch takes +// tens of milliseconds to a few seconds. +const fetchPollInterval = 100 * time.Millisecond + +// Request starts one apply of commit by the Kustomization ref: it resumes the Kustomization with +// the request's token, and Progress follows it from there. +// +// It first checks that the Kustomization applies want, and that no other field manager applies +// its spec.suspend, and refuses with a permanent error when either does not hold. Then it makes +// the GitRepository's artifact hold commit: when it does not, it stamps the GitRepository and waits +// for that fetch. An artifact that ends on another commit is errStale when the decision no longer +// holds, and a failure otherwise: the source cannot fetch the commit the worker checked. // -// It first checks that the Kustomization applies want, and refuses with a permanent error when it -// does not: stamping a GitRepository that follows another branch would deploy that branch and -// leave this one unapplied. The check reads the same objects the stamp is decided from, so no -// change to either can slip in between. +// A ForeignCommit or a CatchUp asks nothing (OutcomeCurrent) when the Kustomization is suspended +// and has applied commit: nothing applied since, and nothing will until it is resumed. A Refusal +// and a Repair always run the operation, because they ask for an apply of an unchanged commit. // -// When the GitRepository's artifact does not hold commit, it stamps the GitRepository: the fetch -// produces a new artifact and the new artifact wakes the Kustomization by itself, so a second stamp -// would only be a second apply. When the artifact already holds it, only a Refusal stamps the -// Kustomization, because only a Refusal asks for an apply of an unchanged commit (to revert drift). -// A ForeignCommit or CatchUp then needs nothing: the artifact that holds the commit already woke the -// Kustomization when it arrived. An empty commit (the remote does not carry the branch) stamps the -// GitRepository, so Flux finds out itself. +// Just before the resume it reads the Kustomization again, since the fetch can take a while: one +// that was replaced, applies another source now, or has its spec.suspend applied by another +// manager now is not resumed. It reads the artifact again and asks current: when either has moved, +// the decision no longer holds and nothing is resumed (errStale). Then record writes the operation +// on the trigger, so a restart finds it, current is asked again, since recording can wait, and +// only then is the Kustomization resumed: one server-side apply sets spec.suspend: false and +// reconcile.fluxcd.io/requestedAt to the token. That write is a new generation and a new +// requestedAt, so it enqueues a reconcile that reads both, which applies. It is made over the +// Kustomization as last read, and fails rather than land on one that changed since. // -// current is asked just before the stamp. When it reports false the decision no longer holds (the -// branch moved on, or became busy) and nothing is stamped: errStale. Stamping the Kustomization of -// an artifact holding an older commit would re-apply that commit over an edit published since. -func (f *Flux) Request( - ctx context.Context, - ref types.NamespacedName, - want Binding, - commit string, - reason v1alpha3.ApplyReason, - current func() bool, -) (Stamped, error) { +// An empty commit (the remote does not carry the branch) has nothing to apply: the GitRepository +// is stamped, so Flux reports the missing branch itself, and nothing is resumed. +func (f *Flux) Request(ctx context.Context, ref types.NamespacedName, ask Ask) (Started, error) { ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { - return Stamped{}, fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) + return Started{}, fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) } sourceRef, err := gitRepositoryRef(ks) if err != nil { - return Stamped{}, permanentError{err} + return Started{}, permanentError{err} } - source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). - Get(ctx, sourceRef.Name, metav1.GetOptions{}) + if owner := suspendOwner(ks); owner != "" { + return Started{}, permanentError{errors.New(suspendOwnedMessage(ref, owner))} + } + source, err := f.readSource(ctx, sourceRef, ref, ask.Want) if err != nil { - return Stamped{}, fmt.Errorf("get %s %s: %w", kindGitRepository, sourceRef, err) + return Started{}, err } - if msg := sourceMismatch(source, sourceRef, want.RepositoryURL, want.Branch); msg != "" { - return Stamped{}, permanentError{fmt.Errorf("%s %s: %s", kindKustomization, ref, msg)} + if ask.Commit == "" { + if _, err := f.stampSource(ctx, sourceRef); err != nil { + return Started{}, err + } + return Started{Outcome: OutcomeCurrent}, nil + } + if err := f.fetchCommit(ctx, sourceRef, source, ask); err != nil { + return Started{}, err } + if (ask.Reason == v1alpha3.ApplyReasonForeignCommit || ask.Reason == v1alpha3.ApplyReasonCatchUp) && + suspended(ks) && revisionHolds(statusString(ks, "lastAppliedRevision"), ask.Commit) { + return Started{Outcome: OutcomeCurrent}, nil + } + return f.resume(ctx, ref, ks, sourceRef, ask) +} - if current != nil && !current() { - return Stamped{}, errStale +// fetchCommit makes the GitRepository's artifact hold ask.Commit: when source's does not, it +// fetches, and an artifact that ends on another commit is errStale or a failure (Request). +func (f *Flux) fetchCommit( + ctx context.Context, sourceRef types.NamespacedName, source *unstructured.Unstructured, ask Ask, +) error { + if artifactHolds(source, ask.Commit) { + return nil + } + source, err := f.fetch(ctx, sourceRef) + if err != nil { + return err } - stamped := Stamped{ - Source: sourceRef, - PriorAttemptedRevision: statusString(ks, "lastAttemptedRevision"), - PriorArtifactRevision: statusString(source, "artifact", "revision"), + if artifactHolds(source, ask.Commit) { + return nil } - if commit != "" && artifactHolds(source, commit) { - if reason != v1alpha3.ApplyReasonRefusal { - stamped.Outcome = OutcomeCurrent - return stamped, nil - } - token, err := f.stamp(ctx, kustomizationGVR(), ref) + if !ask.holds() { + return errStale + } + return fmt.Errorf("%s %s fetched %s, not commit %s", kindGitRepository, sourceRef, + statusString(source, "artifact", "revision"), ask.Commit) +} + +// maxResumeConflicts bounds how often one request reads the Kustomization again after its resume +// found it changed since the last read. +const maxResumeConflicts = 5 + +// resume reads the Kustomization again and checks it is still the object Request checked (read), +// rechecks the artifact, the decision and the trigger's hold, records the operation, and resumes +// the Kustomization with the request's token (Request). +// +// The resume carries the uid and resourceVersion of that last read, which the API server checks +// as preconditions of a server-side apply, and keeps out of managedFields (measured in envtest +// against kube-apiserver 1.37): a resourceVersion that moved is a Conflict, another object's uid is +// Invalid, and a deleted object's is a Conflict rather than a create. So a change made while the +// operation was recorded is never resumed unchecked. A Conflict is read again, up to +// maxResumeConflicts times: kustomize-controller or a label write may have moved the +// resourceVersion and changed nothing the request checked. Any refusal made no resume, and is an +// error. +// +// Once the resume's write may have reached the API server, the operation is followed whatever the +// write answered: a write that failed without saying whether it was made (a timeout, a lost +// connection) returns an Unconfirmed operation rather than an error, so the follow suspends the +// Kustomization again. An error would leave it resumed with nothing guarding it. +func (f *Flux) resume( + ctx context.Context, ref types.NamespacedName, read *unstructured.Unstructured, sourceRef types.NamespacedName, + ask Ask, +) (Started, error) { + record := ask.recordOnce() + for conflicts := 0; ; conflicts++ { + ks, err := f.readAgain(ctx, ref, read, sourceRef) if err != nil { - return Stamped{}, fmt.Errorf("stamp %s %s: %w", kindKustomization, ref, err) + return Started{}, err } - stamped.Outcome, stamped.Token = OutcomeStampedKustomization, token - return stamped, nil + if err := f.mayResume(ctx, ref, sourceRef, ask, record); err != nil { + return Started{}, err + } + // The mark the resume is applied over: its preconditions fail on any other. + before := markOf(ks) + started := Started{Outcome: OutcomeResumed, Token: ask.Token, Commit: ask.Commit, resumedFrom: &before} + resumed, err := f.hold(ctx, ref, false, ask.Token, ks) + switch { + case err == nil: + if resumed != nil { + // The answer is the object as the resume left it: no reconcile can have run since. + mark := markOf(resumed) + started.resumedFrom = &mark + } + return started, nil + case apierrors.IsConflict(err) && conflicts < maxResumeConflicts: + continue + case mayHaveLanded(err): + started.Unconfirmed = fmt.Errorf("resume %s %s: %w", kindKustomization, ref, err) + return started, nil + } + return Started{}, fmt.Errorf("resume %s %s: %w", kindKustomization, ref, err) + } +} + +// readAgain reads the Kustomization again before its resume, and errs when it is no longer the +// object Request checked (read): replaced (another uid), or applying a GitRepository other than +// sourceRef. A request asks again, and checks the new one from the start. Another manager applying +// its spec.suspend now is the permanent error Request gives it. +func (f *Flux) readAgain( + ctx context.Context, ref types.NamespacedName, read *unstructured.Unstructured, sourceRef types.NamespacedName, +) (*unstructured.Unstructured, error) { + ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) + } + if ks.GetUID() != read.GetUID() { + return nil, fmt.Errorf("%s %s was replaced since it was read; asking again", kindKustomization, ref) + } + now, err := gitRepositoryRef(ks) + if err != nil { + return nil, permanentError{err} + } + if now != sourceRef { + return nil, fmt.Errorf("%s %s applies %s %s now, not %s; asking again", kindKustomization, ref, + kindGitRepository, now, sourceRef) + } + if owner := suspendOwner(ks); owner != "" { + return nil, permanentError{errors.New(suspendOwnedMessage(ref, owner))} + } + return ks, nil +} + +// mayResume rechecks the artifact, the decision and the trigger's hold, records the operation (the +// first time only), and asks the decision again once it is recorded. +func (f *Flux) mayResume( + ctx context.Context, ref, sourceRef types.NamespacedName, ask Ask, record func(context.Context) error, +) error { + source, err := f.readSource(ctx, sourceRef, ref, ask.Want) + if err != nil { + return err + } + if !artifactHolds(source, ask.Commit) || !ask.holds() { + return errStale + } + if ask.Held != nil { + if err := ask.Held(ctx); err != nil { + return fmt.Errorf("not resuming %s %s: %w", kindKustomization, ref, err) + } + } + if err := record(ctx); err != nil { + return fmt.Errorf("record the operation on its trigger before resuming %s %s: %w", + kindKustomization, ref, err) + } + // Recording can wait on the status writer and the API server, and the branch can move on + // meanwhile: the decision is asked again just before the resume. + if !ask.holds() { + return errStale + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("resume %s %s: %w", kindKustomization, ref, err) + } + return nil +} + +// readSource reads a Kustomization's GitRepository and checks it fetches want, refusing with a +// permanent error when it does not: resuming a Kustomization whose source follows another branch +// would deploy that branch. +func (f *Flux) readSource( + ctx context.Context, sourceRef, ref types.NamespacedName, want Binding, +) (*unstructured.Unstructured, error) { + source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). + Get(ctx, sourceRef.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get %s %s: %w", kindGitRepository, sourceRef, err) + } + if msg := sourceMismatch(source, sourceRef, want.RepositoryURL, want.Branch); msg != "" { + return nil, permanentError{fmt.Errorf("%s %s: %s", kindKustomization, ref, msg)} } - token, err := f.stamp(ctx, gitRepositoryGVR(), sourceRef) + return source, nil +} + +// fetch stamps the GitRepository and waits for it to handle the stamp, which it marks once the +// fetch ended, whatever it found. A suspended Kustomization ignores the new artifact. +func (f *Flux) fetch(ctx context.Context, sourceRef types.NamespacedName) (*unstructured.Unstructured, error) { + token, err := f.stampSource(ctx, sourceRef) if err != nil { - return Stamped{}, fmt.Errorf("stamp %s %s: %w", kindGitRepository, sourceRef, err) + return nil, err + } + for { + source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). + Get(ctx, sourceRef.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get %s %s: %w", kindGitRepository, sourceRef, err) + } + if statusString(source, "lastHandledReconcileAt") == token { + return source, nil + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("%s %s did not fetch in time: %w", kindGitRepository, sourceRef, ctx.Err()) + case <-time.After(f.PollInterval): + } } - stamped.Outcome, stamped.Token = OutcomeStampedSource, token - return stamped, nil } -// Handled reports whether Flux has finished the apply s asked for, which is when the apply window -// closes: until then an edit to an object the Kustomization applies can be reverted by it. +// Progress reads where a resumed Kustomization's apply stands, and takes the operation's next step +// when it is due, so a Kustomization is never left resumed: +// +// - Running until the reconcile that read the resume's token has ended. That reconcile marks the +// token handled on every return (finalizeStatus), whether it applied, failed, or waited on a +// dependency; and reconciles of one object never overlap. A token someone else stamped over +// ours stands in for it: the reconcile that handles it started after the resume. +// - Then the outcome is read (verdict), from that reconcile's Ready and lastAppliedRevision, and +// the Kustomization is suspended again with a new token: Settling. +// - Succeeded or Failed once that suspend is handled. A suspended reconcile handles it without +// applying, so by then every reconcile that started before the suspend has ended. // -// Flux records a request it handled in status.lastHandledReconcileAt, and does so when the -// reconcile ends (external-sources/flux/pkg/runtime/reconcile/result.go). A stamped Kustomization -// has therefore finished once that field carries the stamp. A stamped GitRepository wakes the -// Kustomization through its new artifact, so that apply has finished once the Kustomization -// attempted a revision other than the one it had attempted before; and if the fetch produced no new -// artifact, there is nothing for it to apply, and the GitRepository's own handled request says so. -func (f *Flux) Handled(ctx context.Context, ref types.NamespacedName, s Stamped) (bool, error) { - if s.Outcome != OutcomeStampedKustomization && s.Outcome != OutcomeStampedSource { - return true, nil +// An Overdue operation (Unresolved) is suspended at once, without waiting for its token: the +// suspend does not stop an apply already running, but it stops whatever would follow, and its token +// still decides when the operation completes. A Kustomization found resumed after that suspend (by +// `flux resume`) is suspended again. A Recovered or Unconfirmed operation whose token is nowhere on the +// Kustomization was never resumed, or its token was stamped over (a suspend of its own the process +// stopped before reporting, `flux reconcile`): it is suspended, and judged once that suspend is +// handled on what ran since it was asked for. Nothing else is asked: an OutcomeCurrent request is +// finished. +func (f *Flux) Progress(ctx context.Context, ref types.NamespacedName, s *Started) (Progress, error) { + if s.Outcome != OutcomeResumed { + return Progress{Outcome: git.ApplySucceeded}, nil } ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { - return false, fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) + return Progress{}, fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) + } + if s.SuspendToken == "" { + if (s.Recovered || s.Unconfirmed != nil) && !carriesToken(ks, s.Token) { + s.unseen = true + } + switch { + case tokenHandled(ks, s.Token): + read, err := f.verdict(ctx, ks, s) + if err != nil { + return Progress{}, err + } + s.verdict = &read + case s.unseen: + // Stamped over before a reconcile handled it, or never written: what ran meanwhile is + // judged once the suspend is handled. + case !s.Overdue: + return Progress{Outcome: git.ApplyRunning}, nil + } + return f.suspendAgain(ctx, ref, s) + } + return f.settled(ctx, ref, ks, s) +} + +// settled is Progress once the Kustomization was suspended again: suspended again when it was found +// resumed, Settling until that suspend is handled, and then the outcome. +func (f *Flux) settled( + ctx context.Context, ref types.NamespacedName, ks *unstructured.Unstructured, s *Started, +) (Progress, error) { + if !suspended(ks) { + return f.suspendAgain(ctx, ref, s) + } + if !tokenHandled(ks, s.SuspendToken) { + return Progress{Outcome: git.ApplySettling, SuspendToken: s.SuspendToken}, nil + } + if s.verdict == nil { + // The resume's own reconcile was not seen to end before the suspend: what the + // Kustomization reports now is its last apply, which may be that one. + read, ran := s.judge(ks) + if ran && (read.Outcome != git.ApplySucceeded || + (s.Commit != "" && !revisionHolds(read.AppliedRevision, s.Commit))) { + read = Progress{Outcome: git.ApplyFailed, AppliedRevision: read.AppliedRevision, + Message: "the Kustomization was suspended again before an apply of the commit was seen to end"} + } + s.verdict = &read + } + done := *s.verdict + done.SuspendToken = s.SuspendToken + return done, nil +} + +// suspendAgain suspends the Kustomization with a new token, and reports the operation Settling. +func (f *Flux) suspendAgain(ctx context.Context, ref types.NamespacedName, s *Started) (Progress, error) { + token, err := f.Suspend(ctx, ref) + if err != nil { + return Progress{}, err } - if s.Outcome == OutcomeStampedKustomization { - return statusString(ks, "lastHandledReconcileAt") == s.Token, nil + s.SuspendToken = token + return Progress{Outcome: git.ApplySettling, SuspendToken: token}, nil +} + +// verdict reads the outcome of the operation's apply once the reconcile that read its token ended +// (judge), and holds a Succeeded of another revision than the commit to the source. The commit was +// in the GitRepository's artifact when the Kustomization was resumed, but kustomize-controller +// reads the GitRepository from its informer cache, so the reconcile can still have read the +// artifact before it. Only an applied revision the artifact holds now is the source having moved +// on (the residual race): Succeeded, with the message. Any other is Failed, so the worker asks +// again: the commit would otherwise land on a Kustomization suspended again, and never be applied. +func (f *Flux) verdict(ctx context.Context, ks *unstructured.Unstructured, s *Started) (Progress, error) { + read, _ := s.judge(ks) + if read.Outcome != git.ApplySucceeded || s.Commit == "" || revisionHolds(read.AppliedRevision, s.Commit) { + return read, nil } - if attempted := statusString( - ks, - "lastAttemptedRevision", - ); attempted != "" && - attempted != s.PriorAttemptedRevision { - return true, nil + sourceRef, err := gitRepositoryRef(ks) + if err != nil { + return Progress{}, err } - source, err := f.client.Resource(gitRepositoryGVR()).Namespace(s.Source.Namespace). - Get(ctx, s.Source.Name, metav1.GetOptions{}) + source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). + Get(ctx, sourceRef.Name, metav1.GetOptions{}) if err != nil { - return false, fmt.Errorf("get %s %s: %w", kindGitRepository, s.Source, err) + return Progress{}, fmt.Errorf("get %s %s: %w", kindGitRepository, sourceRef, err) + } + if _, applied := splitRevision(read.AppliedRevision); artifactHolds(source, applied) { + return read, nil + } + return Progress{Outcome: git.ApplyFailed, AppliedRevision: read.AppliedRevision, Message: fmt.Sprintf( + "Flux applied %s, not commit %s, while %s %s holds %s: the reconcile read an artifact older than the "+ + "one the Kustomization was resumed for", read.AppliedRevision, s.Commit, kindGitRepository, sourceRef, + statusString(source, "artifact", "revision"))}, nil +} + +// judge reads the outcome of the operation's apply from the Kustomization (applyVerdict), and +// returns false with a Failed outcome when no reconcile that builds is seen to have run since the +// resume, where it would read a Succeeded or (for an operation whose token was never seen) any +// outcome. A token handled is no proof of that: a suspend that lands between the resume and the +// reconcile that reads it (`flux suspend`, the trigger's own suspend) makes that reconcile a +// suspended return, which marks the token handled and leaves the Ready of an earlier apply. +func (s *Started) judge(ks *unstructured.Unstructured) (Progress, bool) { + read := applyVerdict(ks, s.Commit) + if s.Token == "" { + // An external resume the trigger's reconciler recorded and suspended again: no resume of ours to + // have run since. It is judged as the reconciler judges it, on the Kustomization's last + // apply, so the two agree whichever completes the record. + return read, true + } + if s.ranResumed(ks) || (read.Outcome != git.ApplySucceeded && !s.unseen) { + return read, true + } + message := "no reconcile applied anything after the resume: the Kustomization was suspended again " + + "before Flux read it" + switch { + case s.unseen && s.Recovered: + message = "recovered after a restart, with no sign of its resume on the Kustomization" + case s.unseen: + message = fmt.Sprintf("no sign on the Kustomization of a resume that may not have been made (%v)", + s.Unconfirmed) + } + return Progress{Outcome: git.ApplyFailed, AppliedRevision: read.AppliedRevision, Message: message}, false +} + +// ranResumed reports whether a reconcile that builds ran since the operation's resume: the +// Kustomization's reconcile mark moved from the one the resume left, or, for an operation recorded +// by an earlier process, moved at or after the second it was asked for. +func (s *Started) ranResumed(ks *unstructured.Unstructured) bool { + if s.resumedFrom != nil { + return markOf(ks) != *s.resumedFrom + } + return markOf(ks).since(s.RequestedAt) +} + +// reconcileMark is what a reconcile of a Kustomization that gets past its suspend check moves, and +// a suspended return leaves as it was (kustomize-controller v1.9.6): Ready's lastTransitionTime, +// since such a reconcile marks Ready Unknown before it fetches (kustomization_controller.go:331), +// and the latest status.history snapshot, which a reconcile that builds counts (:492-563). The +// first has a resolution of a second; the second is there from kustomize-controller v1.7 on. +type reconcileMark struct { + readyTransition string + historyDigest string + historyCount string + historyAt string +} + +// markOf reads a Kustomization's reconcile mark. +func markOf(ks *unstructured.Unstructured) reconcileMark { + var mark reconcileMark + conditions, _, _ := unstructured.NestedSlice(ks.Object, "status", "conditions") + for _, item := range conditions { + if cond, _ := item.(map[string]any); cond["type"] == fluxmeta.ReadyCondition { + mark.readyTransition, _ = cond["lastTransitionTime"].(string) + } + } + history, _, _ := unstructured.NestedSlice(ks.Object, "status", "history") + if len(history) > 0 { + latest, _ := history[0].(map[string]any) + mark.historyDigest, _ = latest["digest"].(string) + mark.historyCount = fmt.Sprint(latest["totalReconciliations"]) + mark.historyAt, _ = latest["lastReconciled"].(string) + } + return mark +} + +// since reports whether the mark moved at or after t, to the second its times are written in. A +// zero t is any mark at all. +func (m reconcileMark) since(t time.Time) bool { + for _, at := range []string{m.readyTransition, m.historyAt} { + moved, err := time.Parse(time.RFC3339, at) + if err == nil && !moved.Before(t.Truncate(time.Second)) { + return true + } } - return statusString(source, "lastHandledReconcileAt") == s.Token && - statusString(source, "artifact", "revision") == s.PriorArtifactRevision, nil + return false +} + +// applyVerdict reads the outcome of a Kustomization's last reconcile, once the token of the apply +// it was asked for is handled (the outcome table in docs/design/apply-trigger.md). +func applyVerdict(ks *unstructured.Unstructured, commit string) Progress { + applied := statusString(ks, "lastAppliedRevision") + ready, found := readyCondition(ks) + switch { + case found && ready.Status == metav1.ConditionTrue: + read := Progress{Outcome: git.ApplySucceeded, AppliedRevision: applied} + if commit != "" && !revisionHolds(applied, commit) { + read.Message = fmt.Sprintf("Flux applied %s, not commit %s", applied, commit) + } + return read + case found && ready.Reason == fluxmeta.DependencyNotReadyReason: + return Progress{Outcome: git.ApplyFailed, AppliedRevision: applied, + Message: "a dependency is not ready: " + ready.Message} + case found: + return Progress{Outcome: git.ApplyFailed, AppliedRevision: applied, + Message: fmt.Sprintf("%s: %s", ready.Reason, ready.Message)} + default: + return Progress{Outcome: git.ApplyFailed, AppliedRevision: applied, + Message: "the Kustomization reports no Ready condition"} + } +} + +// readyCondition is a Kustomization's Ready condition. +func readyCondition(ks *unstructured.Unstructured) (metav1.Condition, bool) { + conditions, _, _ := unstructured.NestedSlice(ks.Object, "status", "conditions") + for _, item := range conditions { + cond, _ := item.(map[string]any) + if cond["type"] != fluxmeta.ReadyCondition { + continue + } + status, _ := cond["status"].(string) + reason, _ := cond["reason"].(string) + message, _ := cond["message"].(string) + return metav1.Condition{ + Type: fluxmeta.ReadyCondition, Status: metav1.ConditionStatus(status), Reason: reason, Message: message, + }, true + } + return metav1.Condition{}, false +} + +// tokenHandled reports whether the reconcile that read token has ended: the Kustomization reports +// it handled, or reports handled a token stamped over it, which a later reconcile read. +func tokenHandled(ks *unstructured.Unstructured, token string) bool { + handled := statusString(ks, "lastHandledReconcileAt") + stamped := ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation] + return handled == token || (stamped != token && stamped != "" && handled == stamped) +} + +// carriesToken reports whether token is the Kustomization's requestedAt, or the one it last handled. +func carriesToken(ks *unstructured.Unstructured, token string) bool { + return token != "" && (ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation] == token || + statusString(ks, "lastHandledReconcileAt") == token) +} + +// suspended reports a Kustomization's spec.suspend. +func suspended(ks *unstructured.Unstructured) bool { + value, _, _ := unstructured.NestedBool(ks.Object, "spec", "suspend") + return value } -// stamp sets reconcile.fluxcd.io/requestedAt to now, which is the whole of what `flux reconcile` -// does to ask: requestReconciliation in external-sources/flux/flux2/cmd/flux/reconcile.go -// (@1ed703a3) merge-patches that annotation, and only that, with time.Now().Format(time.RFC3339Nano). -// Flux reconciles when the value differs from the status.lastHandledReconcileAt it last answered, -// so any new value asks; no read-modify-write is needed and no conflict is possible. +// hold applies spec.suspend and reconcile.fluxcd.io/requestedAt, server-side and under +// SuspendFieldManager. It is forced: a `flux suspend` or `flux resume` took the field with a merge +// patch, and the field returns to GitOps Reverser. A manager that APPLIES the field is refused +// before this is reached (suspendOwner). It returns the Kustomization as the write left it. // -// It returns the value stamped, which Flux copies to status.lastHandledReconcileAt once handled. -func (f *Flux) stamp(ctx context.Context, gvr schema.GroupVersionResource, ref types.NamespacedName) (string, error) { +// over, when set, is the Kustomization as last read, and the write is made only over it: it +// carries its uid and resourceVersion as preconditions (resume). A suspend passes nil, since it +// must land whatever changed. +func (f *Flux) hold( + ctx context.Context, ref types.NamespacedName, suspend bool, token string, over *unstructured.Unstructured, +) (*unstructured.Unstructured, error) { + obj := kustomizationApply(ref) + if over != nil { + obj.SetUID(over.GetUID()) + obj.SetResourceVersion(over.GetResourceVersion()) + } + obj.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: token}) + if err := unstructured.SetNestedField(obj.Object, suspend, "spec", "suspend"); err != nil { + return nil, err + } + return f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace). + Apply(ctx, ref.Name, obj, metav1.ApplyOptions{FieldManager: SuspendFieldManager, Force: true}) +} + +// kustomizationApply is the empty apply configuration of a Kustomization. +func kustomizationApply(ref types.NamespacedName) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(kustomizationGVR().GroupVersion().String()) + obj.SetKind(kindKustomization) + obj.SetNamespace(ref.Namespace) + obj.SetName(ref.Name) + return obj +} + +// stampSource sets the GitRepository's reconcile.fluxcd.io/requestedAt to a new token, which is the +// whole of what `flux reconcile source git` does to ask: requestReconciliation in +// external-sources/flux/flux2/cmd/flux/reconcile.go (@1ed703a3) merge-patches that annotation, and +// only that, with time.Now().Format(time.RFC3339Nano). It returns the token, which source-controller +// copies to status.lastHandledReconcileAt once the fetch ended. +func (f *Flux) stampSource(ctx context.Context, sourceRef types.NamespacedName) (string, error) { token := requestToken(f.Now()) patch, err := json.Marshal(map[string]any{ "metadata": map[string]any{ @@ -193,9 +629,142 @@ func (f *Flux) stamp(ctx context.Context, gvr schema.GroupVersionResource, ref t if err != nil { return "", err } - _, err = f.client.Resource(gvr).Namespace(ref.Namespace). - Patch(ctx, ref.Name, types.MergePatchType, patch, metav1.PatchOptions{}) - return token, err + _, err = f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). + Patch(ctx, sourceRef.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return "", fmt.Errorf("stamp %s %s: %w", kindGitRepository, sourceRef, err) + } + return token, nil +} + +// suspendOwner is the field manager, other than GitOps Reverser, that applies (operation Apply) a +// Kustomization's spec.suspend, or "". Such a manager sets the field on every apply of its manifest, +// so it would resume the Kustomization whenever it applies. A manager that wrote the field with an +// update (`flux resume`, `kubectl edit`) wrote it once, and GitOps Reverser takes it back. +func suspendOwner(ks *unstructured.Unstructured) string { + return suspendManager(ks, metav1.ManagedFieldsOperationApply) +} + +// suspendManager is the field manager, other than GitOps Reverser, that owns a Kustomization's +// spec.suspend through a write of operation, or "". +func suspendManager(ks *unstructured.Unstructured, operation metav1.ManagedFieldsOperationType) string { + for _, entry := range ks.GetManagedFields() { + if entry.Manager == SuspendFieldManager || entry.Operation != operation || + entry.Subresource != "" || !ownsField(entry, "f:spec", "f:suspend") { + continue + } + return entry.Manager + } + return "" +} + +// heldByUs reports whether GitOps Reverser's field manager holds a Kustomization's spec.suspend. +func heldByUs(ks *unstructured.Unstructured) bool { + for _, entry := range ks.GetManagedFields() { + if entry.Manager == SuspendFieldManager && ownsField(entry, "f:spec", "f:suspend") { + return true + } + } + return false +} + +// ownsField reports whether a managedFields entry owns the field at path. +func ownsField(entry metav1.ManagedFieldsEntry, path ...string) bool { + if entry.FieldsV1 == nil { + return false + } + var fields map[string]any + if err := json.Unmarshal(entry.FieldsV1.GetRawBytes(), &fields); err != nil { + return false + } + _, found, _ := unstructured.NestedFieldNoCopy(fields, path...) + return found +} + +// suspendOwnedMessage says which manager applies a Kustomization's spec.suspend. +func suspendOwnedMessage(ref types.NamespacedName, owner string) string { + return fmt.Sprintf("%s %s has spec.suspend applied by field manager %q, which would resume it on its "+ + "next apply; remove spec.suspend from the manifest it applies", kindKustomization, ref, owner) +} + +// Suspension is how a Kustomization stands with GitOps Reverser, for its FluxTrigger's reconciler. +type Suspension struct { + // Suspended is spec.suspend. + Suspended bool + // Held is spec.suspend held by GitOps Reverser's field manager. + Held bool + // Handled is the last requestedAt handled: no reconcile that read an earlier one is running. + Handled bool + // Applied is what the Kustomization's last apply reported, read for an apply GitOps Reverser + // did not ask for. + Applied Progress +} + +// Suspension reads how the Kustomization ref stands. +func (f *Flux) Suspension(ctx context.Context, ref types.NamespacedName) (Suspension, error) { + ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) + if err != nil { + return Suspension{}, err + } + token := ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation] + return Suspension{ + Suspended: suspended(ks), + Held: heldByUs(ks), + Handled: statusString(ks, "lastHandledReconcileAt") == token, + Applied: applyVerdict(ks, ""), + }, nil +} + +// Suspend suspends the Kustomization ref with a new token, and returns it. Once the +// Kustomization reports it handled, nothing that started before the suspend is still running. +func (f *Flux) Suspend(ctx context.Context, ref types.NamespacedName) (string, error) { + token := requestToken(f.Now()) + if _, err := f.hold(ctx, ref, true, token, nil); err != nil { + return "", fmt.Errorf("suspend %s %s: %w", kindKustomization, ref, err) + } + return token, nil +} + +// Release hands the Kustomization ref back to Flux: an empty apply under SuspendFieldManager drops +// spec.suspend and the requestedAt GitOps Reverser held, so the Kustomization is resumed and Flux +// owns its apply moments again. It returns the field manager that still keeps the Kustomization +// suspended, or "" once it is resumed. +// +// An apply of the value a field holds shares the field with its owner, even forced, so a suspend +// that took over a `flux suspend` co-owns spec.suspend with that update, and the empty apply alone +// would leave the field set, suspended (measured in envtest: +// internal/controller/fluxtrigger_suspend_apiserver_test.go). Such an update entry is taken back +// first, with a forced spec.suspend: false, which a different value takes from every owner. A +// manager that applies spec.suspend keeps it: its manifest sets the field. +func (f *Flux) Release(ctx context.Context, ref types.NamespacedName) (string, error) { + kustomizations := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace) + ks, err := kustomizations.Get(ctx, ref.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("release %s %s: %w", kindKustomization, ref, err) + } + options := metav1.ApplyOptions{FieldManager: SuspendFieldManager, Force: true} + if suspended(ks) && suspendManager(ks, metav1.ManagedFieldsOperationUpdate) != "" { + resume := kustomizationApply(ref) + if err := unstructured.SetNestedField(resume.Object, false, "spec", "suspend"); err != nil { + return "", err + } + if _, err := kustomizations.Apply(ctx, ref.Name, resume, options); err != nil { + return "", fmt.Errorf("release %s %s: %w", kindKustomization, ref, err) + } + } + released, err := kustomizations.Apply(ctx, ref.Name, kustomizationApply(ref), options) + if err != nil { + return "", fmt.Errorf("release %s %s: %w", kindKustomization, ref, err) + } + if !suspended(released) { + return "", nil + } + for _, owner := range []string{suspendOwner(released), suspendManager(released, metav1.ManagedFieldsOperationUpdate)} { + if owner != "" { + return owner, nil + } + } + return "(unknown)", nil } // gitRepositoryRef reads a Kustomization's spec.sourceRef, which must name a GitRepository. Its @@ -220,7 +789,11 @@ func gitRepositoryRef(ks *unstructured.Unstructured) (types.NamespacedName, erro // artifactHolds reports whether the GitRepository's current artifact was built from commit. func artifactHolds(source *unstructured.Unstructured, commit string) bool { - revision, _, _ := unstructured.NestedString(source.Object, "status", "artifact", "revision") + return revisionHolds(statusString(source, "artifact", "revision"), commit) +} + +// revisionHolds reports whether a Flux revision names commit. +func revisionHolds(revision, commit string) bool { _, hash := splitRevision(revision) return hash != "" && strings.EqualFold(hash, commit) } @@ -256,21 +829,23 @@ func splitRevision(revision string) (string, string) { return named, revision } -// Validate answers ApplierConfigured for a Kustomization: that it exists in the source cluster and -// its kind is served there, that it applies a GitRepository fetching want (the GitProvider's -// spec.url, compared after normalization, and the GitTarget's branch), and that this identity may -// patch both objects, since a request patches one or the other. +// Validate answers Configured for a FluxTrigger's Kustomization: that it exists in its cluster and +// its kind is served there, that it applies a GitRepository that follows a branch, that no other +// field manager applies its spec.suspend, and that this identity may patch both objects, since a +// request stamps the one and suspends and resumes the other. Which repository and +// branch that is, it reports in the verdict's Source, for the GitTargets naming the trigger to +// check against what they write. // // It reads; it never watches. A cluster without Flux installed answers ApplierNotFound from one // failed GET and keeps no informer for kinds it does not serve. -func (f *Flux) Validate(ctx context.Context, ref types.NamespacedName, want Binding) Verdict { +func (f *Flux) Validate(ctx context.Context, ref types.NamespacedName) Verdict { ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) if err != nil { return readFailure("Flux", kindKustomization, kustomizationGVR(), ref, err) } sourceRef, err := gitRepositoryRef(ks) if err != nil { - return Verdict{Reason: ReasonSourceMismatch, Message: err.Error()} + return Verdict{Reason: ReasonUnsupportedSource, Message: err.Error()} } source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). Get(ctx, sourceRef.Name, metav1.GetOptions{}) @@ -279,8 +854,14 @@ func (f *Flux) Validate(ctx context.Context, ref types.NamespacedName, want Bind verdict.Message = fmt.Sprintf("%s (the sourceRef of %s %s)", verdict.Message, kindKustomization, ref) return verdict } - if msg := sourceMismatch(source, sourceRef, want.RepositoryURL, want.Branch); msg != "" { - return Verdict{Reason: ReasonSourceMismatch, Message: msg} + url, _, _ := unstructured.NestedString(source.Object, "spec", "url") + branch, _, _ := unstructured.NestedString(source.Object, "spec", "ref", "branch") + read := &Source{GitRepository: sourceRef, URL: url, Revision: branch} + if msg := sourceUnsupported(source, sourceRef); msg != "" { + return Verdict{Reason: ReasonUnsupportedSource, Message: msg, Source: read} + } + if owner := suspendOwner(ks); owner != "" { + return Verdict{Reason: ReasonSuspendOwnedElsewhere, Message: suspendOwnedMessage(ref, owner), Source: read} } for _, obj := range []struct { @@ -292,6 +873,7 @@ func (f *Flux) Validate(ctx context.Context, ref types.NamespacedName, want Bind {kindKustomization, kustomizationGVR(), ref}, } { if verdict := accessVerdict(ctx, f.CanI, obj.kind, obj.gvr, obj.ref, "patch"); verdict != nil { + verdict.Source = read return *verdict } } @@ -299,33 +881,43 @@ func (f *Flux) Validate(ctx context.Context, ref types.NamespacedName, want Bind return Verdict{ OK: true, Reason: ReasonSucceeded, - Message: fmt.Sprintf("%s %s applies %s %s, which fetches branch %s", - kindKustomization, ref, kindGitRepository, sourceRef, want.Branch), + Message: fmt.Sprintf("%s %s applies %s %s, which fetches branch %s of %s", + kindKustomization, ref, kindGitRepository, sourceRef, branch, url), + Source: read, } } +// sourceUnsupported returns why a GitRepository does not follow a branch, or "" when it does. A ref +// that pins anything stronger than a branch does not follow one even when it also names it: Flux +// resolves commit, then name, then semver, then tag, before branch, so the pin wins and the +// artifact never follows the branch GitOps Reverser writes. +func sourceUnsupported(source *unstructured.Unstructured, ref types.NamespacedName) string { + gitRef, found, _ := unstructured.NestedStringMap(source.Object, "spec", "ref") + for _, pin := range []string{"commit", "name", "semver", "tag"} { + if value := gitRef[pin]; value != "" { + return fmt.Sprintf("%s %s pins spec.ref.%s %q, so it never follows a branch", + kindGitRepository, ref, pin, value) + } + } + if !found || gitRef["branch"] == "" { + return fmt.Sprintf("%s %s names no spec.ref.branch, so it follows no branch GitOps Reverser writes", + kindGitRepository, ref) + } + return "" +} + // sourceMismatch returns why a GitRepository does not fetch branch of repositoryURL, or "" when it -// does. A ref that pins anything stronger than a branch is a mismatch even when it also names the -// branch: Flux resolves commit, then name, then semver, then tag, before branch, so the pin wins and -// the artifact never follows the branch GitOps Reverser writes. +// does. func sourceMismatch(source *unstructured.Unstructured, ref types.NamespacedName, repositoryURL, branch string) string { url, _, _ := unstructured.NestedString(source.Object, "spec", "url") if !repourl.Same(url, repositoryURL) { return fmt.Sprintf("%s %s fetches %s, not %s", kindGitRepository, ref, url, repositoryURL) } - gitRef, found, _ := unstructured.NestedStringMap(source.Object, "spec", "ref") - if !found || len(gitRef) == 0 { - return fmt.Sprintf("%s %s names no spec.ref.branch; GitOps Reverser writes branch %s", - kindGitRepository, ref, branch) - } - for _, pin := range []string{"commit", "name", "semver", "tag"} { - if value := gitRef[pin]; value != "" { - return fmt.Sprintf("%s %s pins spec.ref.%s %q, so it never follows branch %s", - kindGitRepository, ref, pin, value, branch) - } + if msg := sourceUnsupported(source, ref); msg != "" { + return msg } - if gitRef["branch"] != branch { - return fmt.Sprintf("%s %s fetches branch %q, not %s", kindGitRepository, ref, gitRef["branch"], branch) + if got, _, _ := unstructured.NestedString(source.Object, "spec", "ref", "branch"); got != branch { + return fmt.Sprintf("%s %s fetches branch %q, not %s", kindGitRepository, ref, got, branch) } return "" } diff --git a/internal/applier/flux_consumers.go b/internal/applier/flux_consumers.go new file mode 100644 index 000000000..fb395b3a2 --- /dev/null +++ b/internal/applier/flux_consumers.go @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "errors" + "fmt" + "sort" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +// A GitRepository's consumers, and prefetching it (docs/design/apply-trigger.md, "Shared sources and prefetch"). +// A suspended Kustomization ignores a new artifact, so a GitRepository whose every consumer is held +// suspended by a FluxTrigger can fetch on every push without applying anything. One consumer that is +// not held applies each new artifact the moment it arrives, so then the source is never prefetched. + +// A HelmRelease applies a GitRepository through a HelmChart that source-controller builds from it: +// one helm-controller creates from the release's spec.chart, or one the release names in +// spec.chartRef. Both kinds are listed at the newest version the cluster serves, out of the +// versions Flux has served them at, so an older Flux is read too. +const ( + kindHelmRelease = "HelmRelease" + kindHelmChart = "HelmChart" +) + +// helmReleaseGVR is Flux's HelmRelease at its current version. +func helmReleaseGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "helm.toolkit.fluxcd.io", Version: "v2", Resource: "helmreleases"} +} + +// helmChartGVR is source-controller's HelmChart at its current version. +func helmChartGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmcharts"} +} + +// olderHelmReleaseVersions and olderHelmChartVersions are the versions Flux served the kinds at +// before their current one, newest first. +func olderHelmReleaseVersions() []string { return []string{"v2beta2", "v2beta1"} } +func olderHelmChartVersions() []string { return []string{"v1beta2"} } + +// Consumers lists the objects in the cluster that apply the GitRepository source, sorted: +// Kustomizations whose spec.sourceRef names it, and HelmReleases whose chart is built from it, +// through spec.chart.spec.sourceRef or through a HelmChart spec.chartRef names. Any failure is an +// error, and then nothing is known: the caller must treat the source as shared. +// +// A kind the cluster does not serve has no objects. For HelmReleases that holds only when no +// HelmChart is built from the source either: a release at a version not listed here would still +// apply through one, so then the consumers are unknown and that is an error. +func (f *Flux) Consumers( + ctx context.Context, + source types.NamespacedName, +) ([]fluxmeta.NamespacedObjectKindReference, error) { + kustomizations, _, err := f.listServed(ctx, kustomizationGVR()) + if err != nil { + return nil, err + } + var consumers []fluxmeta.NamespacedObjectKindReference + for i := range kustomizations { + if consumes(&kustomizations[i], []string{"spec", "sourceRef"}, source) { + consumers = append(consumers, kindReference(kindKustomization, &kustomizations[i])) + } + } + releases, err := f.helmReleasesOf(ctx, source) + if err != nil { + return nil, err + } + consumers = append(consumers, releases...) + sort.Slice(consumers, func(i, j int) bool { + a, b := consumers[i], consumers[j] + return a.Kind+"/"+a.Namespace+"/"+a.Name < b.Kind+"/"+b.Namespace+"/"+b.Name + }) + return consumers, nil +} + +// helmReleasesOf is the HelmReleases whose chart is built from the GitRepository source. +func (f *Flux) helmReleasesOf( + ctx context.Context, + source types.NamespacedName, +) ([]fluxmeta.NamespacedObjectKindReference, error) { + releases, served, err := f.listServed(ctx, helmReleaseGVR(), olderHelmReleaseVersions()...) + if err != nil { + return nil, err + } + if !served { + charts, _, err := f.chartsBuiltFrom(ctx, source) + if err != nil { + return nil, err + } + if len(charts) > 0 { + return nil, fmt.Errorf("%d %s(s) are built from %s %s, but the cluster serves %ss at no version known here", + len(charts), kindHelmChart, kindGitRepository, source, kindHelmRelease) + } + return nil, nil + } + var consumers []fluxmeta.NamespacedObjectKindReference + var viaChart []*unstructured.Unstructured + for i := range releases { + release := &releases[i] + switch { + case consumes(release, []string{"spec", "chart", "spec", "sourceRef"}, source): + consumers = append(consumers, kindReference(kindHelmRelease, release)) + case namesHelmChart(release): + viaChart = append(viaChart, release) + } + } + if len(viaChart) == 0 { + return consumers, nil + } + charts, served, err := f.chartsBuiltFrom(ctx, source) + if err != nil { + return nil, err + } + if !served { + return nil, fmt.Errorf("%ss name a %s in spec.chartRef, but the cluster serves %ss at no version known here", + kindHelmRelease, kindHelmChart, kindHelmChart) + } + for _, release := range viaChart { + if charts[chartRefOf(release)] { + consumers = append(consumers, kindReference(kindHelmRelease, release)) + } + } + return consumers, nil +} + +// chartsBuiltFrom is the HelmCharts whose spec.sourceRef names the GitRepository source; a +// HelmChart's source is always in its own namespace. served is false when the cluster serves no +// HelmChart version, and then none is built at all. +func (f *Flux) chartsBuiltFrom( + ctx context.Context, + source types.NamespacedName, +) (map[types.NamespacedName]bool, bool, error) { + charts, served, err := f.listServed(ctx, helmChartGVR(), olderHelmChartVersions()...) + if err != nil || !served { + return nil, served, err + } + built := map[types.NamespacedName]bool{} + for i := range charts { + if consumes(&charts[i], []string{"spec", "sourceRef"}, source) { + built[types.NamespacedName{Namespace: charts[i].GetNamespace(), Name: charts[i].GetName()}] = true + } + } + return built, true, nil +} + +// listServed lists every object of gvr, or of the same resource at the first of the older versions +// the cluster serves; served is false when it serves none of them. +func (f *Flux) listServed( + ctx context.Context, + gvr schema.GroupVersionResource, + older ...string, +) ([]unstructured.Unstructured, bool, error) { + for _, version := range append([]string{gvr.Version}, older...) { + gvr.Version = version + list, err := f.client.Resource(gvr).List(ctx, metav1.ListOptions{}) + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + continue + } + if err != nil { + return nil, false, fmt.Errorf("list %s: %w", gvr.GroupResource(), err) + } + return list.Items, true, nil + } + return nil, false, nil +} + +// namesHelmChart reports whether a HelmRelease's spec.chartRef names a HelmChart. +func namesHelmChart(release *unstructured.Unstructured) bool { + kind, _, _ := unstructured.NestedString(release.Object, "spec", "chartRef", "kind") + return kind == kindHelmChart +} + +// chartRefOf is the HelmChart a HelmRelease's spec.chartRef names, its namespace defaulted to the +// release's own, as Flux defaults it. +func chartRefOf(release *unstructured.Unstructured) types.NamespacedName { + ref, _, _ := unstructured.NestedStringMap(release.Object, "spec", "chartRef") + namespace := ref["namespace"] + if namespace == "" { + namespace = release.GetNamespace() + } + return types.NamespacedName{Namespace: namespace, Name: ref["name"]} +} + +func kindReference(kind string, obj *unstructured.Unstructured) fluxmeta.NamespacedObjectKindReference { + return fluxmeta.NamespacedObjectKindReference{Kind: kind, Namespace: obj.GetNamespace(), Name: obj.GetName()} +} + +// consumes reports whether obj's source reference at path names the GitRepository source, its +// namespace defaulted to obj's own, as Flux defaults it. +func consumes(obj *unstructured.Unstructured, path []string, source types.NamespacedName) bool { + ref, found, _ := unstructured.NestedStringMap(obj.Object, path...) + if !found || ref["kind"] != kindGitRepository || ref["name"] != source.Name { + return false + } + namespace := ref["namespace"] + if namespace == "" { + namespace = obj.GetNamespace() + } + return namespace == source.Namespace +} + +// errNotPrefetched is a prefetch that was not made, because the artifact already holds the commit. +var errNotPrefetched = errors.New("the artifact already holds the commit") + +// Prefetch stamps checked, the GitRepository of the Kustomization ref whose consumers were checked, +// so it fetches now, unless its artifact already holds commit (errNotPrefetched). A Kustomization +// that applies another GitRepository now was repointed since its trigger published checked: that +// source's consumers were never checked, so nothing is stamped (errPrefetchHeld). It waits for +// nothing: the operation that applies the commit reads the artifact again before it resumes. +func (f *Flux) Prefetch(ctx context.Context, ref, checked types.NamespacedName, commit string) error { + ks, err := f.client.Resource(kustomizationGVR()).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get %s %s: %w", kindKustomization, ref, err) + } + sourceRef, err := gitRepositoryRef(ks) + if err != nil { + return err + } + if sourceRef != checked { + return fmt.Errorf("%w: %s %s applies %s %s now, not %s", errPrefetchHeld, kindKustomization, ref, + kindGitRepository, sourceRef, checked) + } + source, err := f.client.Resource(gitRepositoryGVR()).Namespace(sourceRef.Namespace). + Get(ctx, sourceRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get %s %s: %w", kindGitRepository, sourceRef, err) + } + if artifactHolds(source, commit) { + return errNotPrefetched + } + _, err = f.stampSource(ctx, sourceRef) + return err +} diff --git a/internal/applier/flux_consumers_test.go b/internal/applier/flux_consumers_test.go new file mode 100644 index 000000000..0495f8f35 --- /dev/null +++ b/internal/applier/flux_consumers_test.go @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "slices" + "testing" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// helmReleaseObj is a HelmRelease whose chart comes from the GitRepository "platform", in its namespace +// unless sourceNamespace is set. +func helmReleaseObj(namespace, name, sourceNamespace string) *unstructured.Unstructured { + sourceRef := map[string]any{"kind": kindGitRepository, "name": "platform"} + if sourceNamespace != "" { + sourceRef["namespace"] = sourceNamespace + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "helm.toolkit.fluxcd.io/v2", "kind": kindHelmRelease, + "metadata": map[string]any{"namespace": namespace, "name": name}, + "spec": map[string]any{"chart": map[string]any{"spec": map[string]any{ + "chart": "./charts/app", "sourceRef": sourceRef, + }}}, + }} +} + +// TestFluxConsumers_ListsEveryObjectApplyingTheSource: Kustomizations and HelmReleases, their source +// namespace defaulted to their own, and nothing that applies another source. +func TestFluxConsumers_ListsEveryObjectApplyingTheSource(t *testing.T) { + client := newSourceClient( + kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), + kustomizationObj("apps", "elsewhere", gitRepoSourceRef("platform")), + kustomizationObj("apps", "across", map[string]any{ + "kind": kindGitRepository, "name": "platform", "namespace": fluxSystemNS, + }), + kustomizationObj(fluxSystemNS, "other", gitRepoSourceRef("another")), + helmReleaseObj("apps", "podinfo", fluxSystemNS), + helmReleaseObj(fluxSystemNS, "local", ""), + ) + consumers, err := pinnedFlux(client).Consumers(context.Background(), platformKey) + require.NoError(t, err) + assert.Equal(t, []fluxmeta.NamespacedObjectKindReference{ + {Kind: kindHelmRelease, Namespace: "apps", Name: "podinfo"}, + {Kind: kindHelmRelease, Namespace: fluxSystemNS, Name: "local"}, + {Kind: kindKustomization, Namespace: "apps", Name: "across"}, + {Kind: kindKustomization, Namespace: fluxSystemNS, Name: "checkout"}, + }, consumers) +} + +// helmChartObj is a HelmChart built from the GitRepository sourceName in its own namespace. +func helmChartObj(namespace, name, sourceName string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "source.toolkit.fluxcd.io/v1", "kind": kindHelmChart, + "metadata": map[string]any{"namespace": namespace, "name": name}, + "spec": map[string]any{ + "chart": "./charts/app", "sourceRef": map[string]any{"kind": kindGitRepository, "name": sourceName}, + }, + }} +} + +// chartRefRelease is a HelmRelease whose spec.chartRef names an object of kind, in its own +// namespace unless refNamespace is set. +func chartRefRelease(namespace, name, kind, refName, refNamespace string) *unstructured.Unstructured { + chartRef := map[string]any{"kind": kind, "name": refName} + if refNamespace != "" { + chartRef["namespace"] = refNamespace + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "helm.toolkit.fluxcd.io/v2", "kind": kindHelmRelease, + "metadata": map[string]any{"namespace": namespace, "name": name}, + "spec": map[string]any{"chartRef": chartRef}, + }} +} + +// helmClient is a source cluster that serves HelmReleases and HelmCharts at every version Flux +// served them at, except those listed in unserved, which answer a list with NotFound as an API +// server does for a version it does not serve. +func helmClient(unserved []schema.GroupVersionResource, objs ...runtime.Object) *dynamicfake.FakeDynamicClient { + listKinds := map[schema.GroupVersionResource]string{kustomizationGVR(): "KustomizationList"} + for _, gvr := range everyVersion(helmReleaseGVR(), olderHelmReleaseVersions()) { + listKinds[gvr] = "HelmReleaseList" + } + for _, gvr := range everyVersion(helmChartGVR(), olderHelmChartVersions()) { + listKinds[gvr] = "HelmChartList" + } + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), listKinds, objs...) + client.PrependReactor("list", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + if slices.Contains(unserved, action.GetResource()) { + return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "") + } + return false, nil, nil + }) + return client +} + +// everyVersion is gvr at its own version and at each of older. +func everyVersion(gvr schema.GroupVersionResource, older []string) []schema.GroupVersionResource { + gvrs := []schema.GroupVersionResource{gvr} + for _, version := range older { + gvr.Version = version + gvrs = append(gvrs, gvr) + } + return gvrs +} + +// TestFluxConsumers_AHelmReleaseThroughAHelmChart is the fourth review's finding J: a HelmRelease +// whose spec.chartRef names a HelmChart built from the source applies it too. +func TestFluxConsumers_AHelmReleaseThroughAHelmChart(t *testing.T) { + client := helmClient(nil, + helmChartObj(fluxSystemNS, "podinfo", "platform"), + helmChartObj(fluxSystemNS, "elsewhere", "another"), + chartRefRelease("apps", "podinfo", kindHelmChart, "podinfo", fluxSystemNS), + chartRefRelease(fluxSystemNS, "local", kindHelmChart, "podinfo", ""), + chartRefRelease("apps", "other-source", kindHelmChart, "elsewhere", fluxSystemNS), + chartRefRelease("apps", "missing-chart", kindHelmChart, "podinfo", ""), + chartRefRelease("apps", "oci", "OCIRepository", "podinfo", fluxSystemNS), + ) + consumers, err := pinnedFlux(client).Consumers(context.Background(), platformKey) + require.NoError(t, err) + assert.Equal(t, []fluxmeta.NamespacedObjectKindReference{ + {Kind: kindHelmRelease, Namespace: "apps", Name: "podinfo"}, + {Kind: kindHelmRelease, Namespace: fluxSystemNS, Name: "local"}, + }, consumers) +} + +// TestFluxConsumers_AClusterServingOnlyAnOlderHelmRelease: a Flux that serves HelmReleases only at +// v2beta2 has them listed there, not read as having none. +func TestFluxConsumers_AClusterServingOnlyAnOlderHelmRelease(t *testing.T) { + release := helmReleaseObj("apps", "podinfo", fluxSystemNS) + release.SetAPIVersion("helm.toolkit.fluxcd.io/v2beta2") + client := helmClient([]schema.GroupVersionResource{helmReleaseGVR()}, release) + + consumers, err := pinnedFlux(client).Consumers(context.Background(), platformKey) + require.NoError(t, err) + assert.Equal(t, []fluxmeta.NamespacedObjectKindReference{ + {Kind: kindHelmRelease, Namespace: "apps", Name: "podinfo"}, + }, consumers) +} + +// TestFluxConsumers_NoHelmReleaseAPIIsNoneOnlyWithoutAChart: a cluster that serves HelmReleases at +// no version has none, unless a HelmChart is built from the source: then a release at some other +// version may apply it, and the consumers are unknown. +func TestFluxConsumers_NoHelmReleaseAPIIsNoneOnlyWithoutAChart(t *testing.T) { + unserved := everyVersion(helmReleaseGVR(), olderHelmReleaseVersions()) + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + + consumers, err := pinnedFlux(helmClient(unserved, ks)).Consumers(context.Background(), platformKey) + require.NoError(t, err, "no Helm at all: none") + assert.Equal(t, []fluxmeta.NamespacedObjectKindReference{ + {Kind: kindKustomization, Namespace: fluxSystemNS, Name: "checkout"}, + }, consumers) + + _, err = pinnedFlux(helmClient(unserved, ks, helmChartObj(fluxSystemNS, "podinfo", "platform"))). + Consumers(context.Background(), platformKey) + require.Error(t, err, "a chart is built from the source, by a release that cannot be listed") +} + +// TestFluxProbe_ASharedSourceAppliesOnlyTheConsumerAsked is the third review's third finding: A and +// B share a GitRepository, and A's request fetched it, which woke B too, outside any window. With +// both suspended, A's fetch and apply leave B alone: an edit to B's objects keeps its value. +func TestFluxProbe_ASharedSourceAppliesOnlyTheConsumerAsked(t *testing.T) { + a := fluxsim.OwnSuspend(t, kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), true, + SuspendFieldManager, metav1.ManagedFieldsOperationApply) + b := fluxsim.OwnSuspend(t, kustomizationObj(fluxSystemNS, "infra", gitRepoSourceRef("platform")), true, + SuspendFieldManager, metav1.ManagedFieldsOperationApply) + sim := fluxsim.New(t, newSourceClient(a, b, + gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+olderCommit))) + sim.Fetches(platformKey, "main@sha1:"+testCommit) + flux := simFlux(sim) + + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + follow(t, flux, &started) + + assert.Equal(t, []string{"flux-system/checkout main@sha1:" + testCommit}, sim.Applied(), + "B ignored the artifact A's request fetched") + assert.True(t, suspended(sim.Kustomization(checkoutKey))) +} + +// prefetchFixture is a dispatcher whose FluxTrigger "checkout" publishes source flux-system/platform +// with prefetch, over a cluster whose GitRepository is behind and whose Kustomization it holds. +func prefetchFixture(t *testing.T, prefetch bool, extra ...client.Object) *dispatcherFixture { + t.Helper() + return prefetchFixtureOver(t, prefetch, heldSource(t), extra...) +} + +// prefetchFixtureOver is prefetchFixture over the source cluster objects source. +func prefetchFixtureOver(t *testing.T, prefetch bool, source []runtime.Object, + extra ...client.Object) *dispatcherFixture { + t.Helper() + f := newDispatcherFixture(t, []client.Object{gitTarget("a", checkoutRef(), "")}, source...) + setTriggerSource(t, f, "checkout", prefetch, nil) + for _, obj := range extra { + require.NoError(t, f.control.Create(context.Background(), obj)) + } + return f +} + +// setTriggerSource writes a FluxTrigger's published source, and its operation. +func setTriggerSource(t *testing.T, f *dispatcherFixture, name string, prefetch bool, + operation *v1alpha3.ApplyOperation) { + t.Helper() + trigger := &v1alpha3.FluxTrigger{} + require.NoError(t, f.control.Get(context.Background(), client.ObjectKey{Namespace: targetNS, Name: name}, trigger)) + trigger.Status.Source = &v1alpha3.FluxTriggerSource{ + GitRepository: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "platform"}, + URL: testRepoURL, Branch: testBranch, Prefetch: prefetch, + } + trigger.Status.Operation = operation + require.NoError(t, f.control.Update(context.Background(), trigger)) +} + +func prefetchRequest(commit string) git.PrefetchRequest { + return git.PrefetchRequest{Applier: checkout, Commit: commit} +} + +// TestDispatcher_PrefetchFetchesTheSourceOnlyWhenItMay: the source is stamped when every consumer is +// held and nothing on it is in flight, and left alone otherwise. +func TestDispatcher_PrefetchFetchesTheSourceOnlyWhenItMay(t *testing.T) { + f := prefetchFixture(t, true) + require.NoError(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit))) + assert.Equal(t, []string{"gitrepositories " + fluxSystemNS}, f.patches(), "fetched; nothing applied") + + held := prefetchFixture(t, false) + require.ErrorIs(t, held.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), errPrefetchHeld, + "a source another consumer applies on its own is never prefetched") + assert.Empty(t, held.patches()) + + busy := prefetchFixture(t, true, &v1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: "team-b", Name: "infra"}, + Spec: v1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &fluxmeta.LocalObjectReference{Name: v1alpha3.DefaultClusterProviderName}, + KustomizationRef: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "infra"}, + }, + Status: v1alpha3.FluxTriggerStatus{ + Source: &v1alpha3.FluxTriggerSource{ + GitRepository: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "platform"}, + }, + Operation: &v1alpha3.ApplyOperation{Phase: v1alpha3.ApplyOperationRunning, Token: "T1"}, + }, + }) + require.ErrorIs(t, busy.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), errPrefetchHeld, + "not while a consumer of the source is resumed") + assert.Empty(t, busy.patches()) + + current := prefetchFixture(t, true) + require.ErrorIs(t, current.dispatcher.prefetch(context.Background(), prefetchRequest(olderCommit)), + errNotPrefetched, "the artifact already holds it") + assert.Empty(t, current.patches()) +} + +// heldSource is behindSource with flux-system/checkout held by GitOps Reverser, and extra. +func heldSource(t *testing.T, extra ...runtime.Object) []runtime.Object { + t.Helper() + source := behindSource() + source[0] = fluxsim.OwnSuspend(t, source[0].(*unstructured.Unstructured), true, SuspendFieldManager, + metav1.ManagedFieldsOperationApply) + return append(source, extra...) +} + +// TestDispatcher_PrefetchChecksTheConsumersAsTheyAreNow is the fourth review's finding C: the +// trigger's status.source.prefetch is up to a minute old, so the consumers are read again before +// the source is stamped. A consumer that appeared since, or one Flux applies on its own, would apply +// the prefetched artifact over live edits not pushed yet. +func TestDispatcher_PrefetchChecksTheConsumersAsTheyAreNow(t *testing.T) { + infraTrigger := func() *v1alpha3.FluxTrigger { + return &v1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: targetNS, Name: "infra"}, + Spec: v1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &fluxmeta.LocalObjectReference{Name: v1alpha3.DefaultClusterProviderName}, + KustomizationRef: fluxmeta.NamespacedObjectReference{Namespace: fluxSystemNS, Name: "infra"}, + }, + } + } + infra := func(held bool) *unstructured.Unstructured { + ks := kustomizationObj(fluxSystemNS, "infra", gitRepoSourceRef("platform")) + if held { + return fluxsim.OwnSuspend(t, ks, true, SuspendFieldManager, metav1.ManagedFieldsOperationApply) + } + return ks + } + resumed := heldSource(t) + require.NoError(t, unstructured.SetNestedField(resumed[0].(*unstructured.Unstructured).Object, false, + "spec", "suspend")) + + for name, f := range map[string]*dispatcherFixture{ + "a Kustomization no trigger names": prefetchFixtureOver(t, true, heldSource(t, infra(true))), + "a Kustomization Flux applies on its own": prefetchFixtureOver(t, true, heldSource(t, infra(false)), + infraTrigger()), + "a HelmRelease": prefetchFixtureOver(t, true, + heldSource(t, helmReleaseObj("apps", "podinfo", fluxSystemNS))), + "its own Kustomization, resumed": prefetchFixtureOver(t, true, resumed), + } { + t.Run(name, func(t *testing.T) { + require.ErrorIs(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), + errPrefetchHeld) + assert.Empty(t, f.patches(), "nothing is fetched") + }) + } + + f := prefetchFixtureOver(t, true, heldSource(t, infra(true)), infraTrigger()) + require.NoError(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit))) + assert.Equal(t, []string{"gitrepositories " + fluxSystemNS}, f.patches(), "every consumer is held") +} + +// TestDispatcher_PrefetchStampsOnlyTheSourceItChecked: the trigger's status.source is up to a minute +// old, and the Kustomization was repointed since, to a second GitRepository over the same repository +// and branch, which a Kustomization Flux applies on its own consumes too. The checks pass for the old +// source, which nothing consumes now, so stamping the Kustomization's new one would have that consumer +// apply the commit at once, over live edits not pushed yet. +func TestDispatcher_PrefetchStampsOnlyTheSourceItChecked(t *testing.T) { + source := heldSource(t, + gitRepositoryObj(fluxSystemNS, "platform-next", testRepoURL, branchRef(testBranch), "main@sha1:"+olderCommit), + kustomizationObj(fluxSystemNS, "billing", gitRepoSourceRef("platform-next"))) + require.NoError(t, unstructured.SetNestedStringMap(source[0].(*unstructured.Unstructured).Object, + map[string]string{"kind": kindGitRepository, "name": "platform-next"}, "spec", "sourceRef")) + f := prefetchFixtureOver(t, true, source) + + require.ErrorIs(t, f.dispatcher.prefetch(context.Background(), prefetchRequest(testCommit)), errPrefetchHeld) + assert.Empty(t, f.patches(), "neither source is fetched") +} + +var _ runtime.Object = (*unstructured.Unstructured)(nil) diff --git a/internal/applier/flux_guide_test.go b/internal/applier/flux_guide_test.go index 8f6501776..420cd64ec 100644 --- a/internal/applier/flux_guide_test.go +++ b/internal/applier/flux_guide_test.go @@ -21,28 +21,36 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilyaml "k8s.io/apimachinery/pkg/util/yaml" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" + "github.com/ConfigButler/gitops-reverser/internal/authz" ) // fluxGuide is the Flux user guide. Its "A complete example" section is the configuration this // file holds to the code: a reader copies it, so it must be accepted as written. -const fluxGuide = "../../docs/flux.md" +const fluxGuide = "../../docs/bi-directional-flux.md" // TestFluxGuide_TheCompleteExampleIsAccepted reads the guide's complete example and checks it // against the code that validates and drives a Flux applier: the GitOps Reverser objects decode -// into the API types with no unknown field, ApplierConfigured comes out True for the Flux objects -// as written, and a request stamps the GitRepository. It also holds the example to the guide's own +// into the API types with no unknown field, Configured comes out True for the Flux objects as +// written, the trigger's source binds the GitTarget, and a request fetches and resumes the +// Kustomization. It also holds the example to the guide's own // advice, so the page cannot recommend one thing and show another. func TestFluxGuide_TheCompleteExampleIsAccepted(t *testing.T) { objects := guideExampleObjects(t, fluxGuide) var provider v1alpha3.GitProvider var target v1alpha3.GitTarget + var trigger v1alpha3.FluxTrigger var secret corev1.Secret decodeStrict(t, objects, "GitProvider", &provider) decodeStrict(t, objects, "GitTarget", &target) + decodeStrict(t, objects, v1alpha3.ApplyTriggerKindFlux, &trigger) decodeStrict(t, objects, "Secret", &secret) + exampleReachesItsApplier(t, objects, &trigger) repository := one(t, objects, "GitRepository") kustomization := one(t, objects, "Kustomization") assert.NotContains(t, objects, "Receiver", "the guide tells the reader to remove the Flux Receiver") @@ -54,44 +62,45 @@ func TestFluxGuide_TheCompleteExampleIsAccepted(t *testing.T) { assert.Equal(t, secret.Name, provider.Spec.Receiver.SecretRef.Name) assert.Equal(t, provider.Namespace, secret.Namespace) assert.NotEmpty(t, secret.StringData[v1alpha3.ReceiverTokenKey], "the receiver reads the token key") - require.NotNil(t, target.Spec.Applier) + require.NotNil(t, target.Spec.ApplyTriggerRef) + assert.Equal(t, v1alpha3.ApplyTriggerReference{Kind: v1alpha3.ApplyTriggerKindFlux, Name: trigger.Name}, + *target.Spec.ApplyTriggerRef) + assert.Equal(t, target.Namespace, trigger.Namespace, "a GitTarget names a trigger in its own namespace") assert.Equal(t, v1alpha3.RefusalActionReapply, target.Spec.OnRefusal) - assert.LessOrEqual(t, target.Spec.Applier.EffectiveQuietPeriod(), time.Hour) + assert.LessOrEqual(t, trigger.EffectiveQuietPeriod(), time.Hour) // The Flux objects follow the guide's advice. - for _, obj := range []*unstructured.Unstructured{repository, kustomization} { - interval, _, _ := unstructured.NestedString(obj.Object, "spec", "interval") - d, err := time.ParseDuration(interval) - require.NoError(t, err, "%s spec.interval", obj.GetKind()) - assert.GreaterOrEqual( - t, - d, - time.Hour, - "%s: a short interval fetches, and so applies, on its own", - obj.GetKind(), - ) - } - suspended, _, _ := unstructured.NestedBool(kustomization.Object, "spec", "suspend") - assert.False(t, suspended, "a suspended Kustomization ignores the requests") - - // The applier, as written, is accepted and driven. - ref := types.NamespacedName{ - Namespace: target.Spec.Applier.KustomizationRef.Namespace, Name: target.Spec.Applier.KustomizationRef.Name, - } - if ref.Namespace == "" { - ref.Namespace = target.Namespace - } + interval, _, _ := unstructured.NestedString(repository.Object, "spec", "interval") + d, err := time.ParseDuration(interval) + require.NoError(t, err, "GitRepository spec.interval") + assert.GreaterOrEqual(t, d, time.Hour, + "a source that fetches often can move the artifact between the recheck and the resume") + _, setsSuspend, _ := unstructured.NestedFieldNoCopy(kustomization.Object, "spec", "suspend") + assert.False(t, setsSuspend, "the manifest omits spec.suspend, which GitOps Reverser holds") + + // The applier, as written, is accepted, binds the GitTarget, and is driven. + ref := trigger.Applier() + assert.Equal(t, types.NamespacedName{Namespace: kustomization.GetNamespace(), Name: kustomization.GetName()}, ref) want := Binding{RepositoryURL: provider.Spec.URL, Branch: target.Spec.Branch} client := newSourceClient(kustomization, repository) flux := pinnedFlux(client) flux.CanI = allowAll - verdict := flux.Validate(context.Background(), ref, want) - assert.True(t, verdict.OK, "ApplierConfigured for the guide's example: %s: %s", verdict.Reason, verdict.Message) - - stamped, err := flux.Request(context.Background(), ref, want, testCommit, v1alpha3.ApplyReasonForeignCommit, nil) + verdict := flux.Validate(context.Background(), ref) + assert.True(t, verdict.OK, "Configured for the guide's example: %s: %s", verdict.Reason, verdict.Message) + require.NotNil(t, verdict.Source) + trigger.Status.Source = &v1alpha3.FluxTriggerSource{URL: verdict.Source.URL, Branch: verdict.Source.Revision} + assert.Empty(t, BindingMismatch(&trigger, want), "the trigger's Kustomization applies what the GitTarget writes") + + sim := fluxsim.New(t, client) + sim.Fetches(types.NamespacedName{Namespace: repository.GetNamespace(), Name: repository.GetName()}, + target.Spec.Branch+"@sha1:"+testCommit) + flux.Now = fluxsim.TickingClock(stampTime) + started, err := flux.Request(context.Background(), ref, Ask{ + Want: want, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, Token: "T1", + }) require.NoError(t, err) - assert.Equal(t, OutcomeStampedSource, stamped.Outcome, "a source with no artifact yet is asked to fetch") + assert.Equal(t, OutcomeResumed, started.Outcome, "a source with no artifact yet fetches, then applies") assert.NotEmpty(t, requestedAt(t, client, gitRepositoryGVR(), repository.GetNamespace(), repository.GetName())) } @@ -149,3 +158,22 @@ func decodeStrict(t *testing.T, objects map[string][]*unstructured.Unstructured, decoder.DisallowUnknownFields() require.NoError(t, decoder.Decode(into), "the guide's %s must decode with no unknown field", kind) } + +// exampleReachesItsApplier checks that the guide's example ClusterProvider admits the trigger's +// namespace and lets it reach its applier, through the admission the trigger reconciler runs. +func exampleReachesItsApplier( + t *testing.T, objects map[string][]*unstructured.Unstructured, trigger v1alpha3.ApplyTrigger, +) { + t.Helper() + var provider v1alpha3.ClusterProvider + decodeStrict(t, objects, "ClusterProvider", &provider) + assert.Equal(t, trigger.ClusterProvider(), provider.Name, "the example declares the trigger's ClusterProvider") + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, v1alpha3.AddToScheme(scheme)) + control := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&provider).Build() + decision, err := authz.ApplyTriggerAdmitted(context.Background(), control, trigger) + require.NoError(t, err) + assert.True(t, decision.Allowed, "the example's ClusterProvider lets the trigger reach its applier: %s: %s", + decision.Reason, decision.Message) +} diff --git a/internal/applier/flux_lifecycle_test.go b/internal/applier/flux_lifecycle_test.go new file mode 100644 index 000000000..671f31448 --- /dev/null +++ b/internal/applier/flux_lifecycle_test.go @@ -0,0 +1,936 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +// The suspended Flux lifecycle, against fluxSim: a resume with a token, the outcome read once the +// reconcile that read it ended, the suspend after the apply, and the hand-back. The first tests are the third +// review's probes and the measurements of docs/design/apply-trigger.md, each of which +// failed against the request-based client this replaced. + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + k8stesting "k8s.io/client-go/testing" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +var ( + checkoutKey = types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"} + platformKey = types.NamespacedName{Namespace: fluxSystemNS, Name: "platform"} +) + +// simCluster is a source cluster with the Kustomization flux-system/checkout applying the +// GitRepository flux-system/platform on testBranch, whose artifact and last apply are at revision, +// and whose fetch finds testCommit. +func simCluster(t *testing.T, revision string, suspend bool) *fluxsim.Sim { + t.Helper() + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + ks.SetGeneration(1) + ks.SetUID("checkout-1") + if suspend { + ks = fluxsim.OwnSuspend(t, ks, true, SuspendFieldManager, metav1.ManagedFieldsOperationApply) + } + require.NoError(t, unstructured.SetNestedField(ks.Object, revision, "status", "lastAppliedRevision")) + fluxsim.SetReady(ks, metav1.ConditionTrue, fluxmeta.ReconciliationSucceededReason, "Applied revision: "+revision) + sim := fluxsim.New(t, newSourceClient(ks, + gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), revision))) + sim.Fetches(platformKey, "main@sha1:"+testCommit) + return sim +} + +// simFlux is a Flux client of the simulated cluster, with a clock that never repeats a token. +func simFlux(sim *fluxsim.Sim) *Flux { + flux := NewFlux(sim.Client) + flux.Now = fluxsim.TickingClock(stampTime) + flux.PollInterval = time.Millisecond + return flux +} + +// ask is a request for testCommit, for reason, with token T1. +func ask(reason v1alpha3.ApplyReason) Ask { + return Ask{Want: testBinding, Commit: testCommit, Reason: reason, Token: "T1"} +} + +// follow reads the operation's progress until it ends, and returns every step it read, the last +// being the end. +func follow(t *testing.T, flux *Flux, s *Started) []Progress { + t.Helper() + var steps []Progress + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + progress, err := flux.Progress(context.Background(), checkoutKey, s) + require.NoError(t, err) + if len(steps) == 0 || steps[len(steps)-1] != progress { + steps = append(steps, progress) + } + if progress.Outcome.Ends() { + return steps + } + time.Sleep(time.Millisecond) + } + t.Fatalf("the operation did not end; it read %v", steps) + return nil +} + +func outcomes(steps []Progress) []git.ApplyOutcome { + out := make([]git.ApplyOutcome, 0, len(steps)) + for _, step := range steps { + out = append(out, step.Outcome) + } + return out +} + +// TestFluxProbe_AnApplyIsNotFinishedWhenItWasAttempted is the third review's first finding: +// kustomize-controller writes lastAttemptedRevision before it builds, so reading it as "finished" +// closed the apply window while the apply still ran. Now the operation runs until the reconcile +// that read the resume's token ended, and completes once the suspend after it is handled. +func TestFluxProbe_AnApplyIsNotFinishedWhenItWasAttempted(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + held := sim.HoldNextApply() + flux := simFlux(sim) + + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonForeignCommit)) + require.NoError(t, err) + require.Equal(t, OutcomeResumed, started.Outcome) + <-held + assert.Equal(t, "main@sha1:"+testCommit, statusString(sim.Kustomization(checkoutKey), "lastAttemptedRevision")) + progress, err := flux.Progress(context.Background(), checkoutKey, &started) + require.NoError(t, err) + assert.Equal(t, Progress{Outcome: git.ApplyRunning}, progress, "the apply is still running") + + sim.Release() + steps := follow(t, flux, &started) + require.GreaterOrEqual(t, len(steps), 2) + tail := steps[len(steps)-2:] + assert.Equal(t, []git.ApplyOutcome{git.ApplySettling, git.ApplySucceeded}, outcomes(tail)) + assert.Equal(t, "main@sha1:"+testCommit, tail[1].AppliedRevision) + assert.Equal(t, started.SuspendToken, tail[1].SuspendToken) + assert.Equal(t, []string{"flux-system/checkout main@sha1:" + testCommit}, sim.Applied()) + assert.True(t, suspended(sim.Kustomization(checkoutKey)), "suspended again") +} + +// TestFluxProbe_ATokenHandledWhileSuspendedIsNoApply is measurement point 3: a suspended +// Kustomization marks a requestedAt handled without applying anything, so a Refusal that only +// stamped it completed with nothing reverted. Now a Refusal resumes it (point 4: a resume alone +// applies), and is reverted by that apply. +func TestFluxProbe_ATokenHandledWhileSuspendedIsNoApply(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + flux := simFlux(sim) + + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + steps := follow(t, flux, &started) + assert.Equal(t, git.ApplySucceeded, steps[len(steps)-1].Outcome) + assert.Equal(t, []string{"flux-system/checkout main@sha1:" + testCommit}, sim.Applied(), + "the refusal is reverted by an apply of the unchanged commit") +} + +// TestFluxProbe_AnArtifactNotAppliedIsNotCurrent is the first finding's second half: an artifact +// that holds the commit says nothing about whether the Kustomization applied it. +func TestFluxProbe_AnArtifactNotAppliedIsNotCurrent(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + sim.WriteStatus(context.Background(), kustomizationGVR(), checkoutKey, func(obj *unstructured.Unstructured) { + _ = unstructured.SetNestedField(obj.Object, "main@sha1:"+olderCommit, "status", "lastAppliedRevision") + }) + started, err := simFlux(sim).Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonCatchUp)) + require.NoError(t, err) + assert.Equal(t, OutcomeResumed, started.Outcome, "the Kustomization never applied the commit") +} + +// TestFluxRequest_Outcomes pins what a request does, by where the artifact and the last apply are, +// and by reason. Every request that asks for an apply resumes the Kustomization with its token, and +// fetches first when the artifact does not hold the commit. +func TestFluxRequest_Outcomes(t *testing.T) { + tests := []struct { + name string + artifact string + applied string + suspended bool + commit string + reason v1alpha3.ApplyReason + want Outcome + wantFetch bool + }{ + { + name: "a foreign commit not fetched yet fetches, then resumes", artifact: "main@sha1:" + olderCommit, + suspended: true, commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, + want: OutcomeResumed, wantFetch: true, + }, + { + name: "a catch-up behind fetches, then resumes", artifact: "main@sha1:" + olderCommit, + suspended: true, commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, + want: OutcomeResumed, wantFetch: true, + }, + { + name: "no artifact yet fetches, then resumes", suspended: true, + commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, + want: OutcomeResumed, wantFetch: true, + }, + { + name: "a branch absent on the remote fetches, and resumes nothing", artifact: "main@sha1:" + testCommit, + suspended: true, commit: "", reason: v1alpha3.ApplyReasonForeignCommit, + want: OutcomeCurrent, wantFetch: true, + }, + { + name: "a refusal at the applied commit resumes", artifact: "main@sha1:" + testCommit, + applied: "main@sha1:" + testCommit, suspended: true, + commit: testCommit, reason: v1alpha3.ApplyReasonRefusal, want: OutcomeResumed, + }, + { + name: "a repair at the applied commit resumes", artifact: "main@sha1:" + testCommit, + applied: "main@sha1:" + testCommit, suspended: true, + commit: testCommit, reason: v1alpha3.ApplyReasonRepair, want: OutcomeResumed, + }, + { + name: "a foreign commit applied while suspended asks nothing", artifact: "main/" + testCommit, + applied: "main@sha1:" + testCommit, suspended: true, + commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, want: OutcomeCurrent, + }, + { + name: "a catch-up applied while suspended asks nothing", artifact: "sha1:" + testCommit, + applied: "main@sha1:" + testCommit, suspended: true, + commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, want: OutcomeCurrent, + }, + { + // Not suspended yet (the trigger has not adopted it): it may apply on its own, so the + // operation runs, and its suspend after the apply holds it again. + name: "a catch-up applied but not suspended resumes", artifact: "main@sha1:" + testCommit, + applied: "main@sha1:" + testCommit, + commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, want: OutcomeResumed, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sim := simCluster(t, tt.artifact, tt.suspended) + sim.WriteStatus( + context.Background(), + kustomizationGVR(), + checkoutKey, + func(obj *unstructured.Unstructured) { + _ = unstructured.SetNestedField(obj.Object, tt.applied, "status", "lastAppliedRevision") + }, + ) + a := ask(tt.reason) + a.Commit = tt.commit + got, err := simFlux(sim).Request(context.Background(), checkoutKey, a) + require.NoError(t, err) + assert.Equal(t, tt.want, got.Outcome) + + assert.Equal(t, tt.wantFetch, + requestedAt(t, sim.Client, gitRepositoryGVR(), fluxSystemNS, "platform") != "") + ks := sim.Kustomization(checkoutKey) + if tt.want == OutcomeResumed { + assert.Equal(t, "T1", ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]) + assert.Equal(t, []any{OutcomeResumed, "T1", testCommit, nil}, + []any{got.Outcome, got.Token, got.Commit, got.Unconfirmed}) + return + } + assert.Empty(t, ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]) + assert.Equal(t, tt.suspended, suspended(ks), "nothing resumed") + }) + } +} + +// TestFluxRequest_TheOperationIsRecordedBeforeTheResume: a restart reads the trigger's +// status.operation to find every apply that may have started, so the record is written before the +// resume, and a record that cannot be written resumes nothing. +func TestFluxRequest_TheOperationIsRecordedBeforeTheResume(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + a := ask(v1alpha3.ApplyReasonRefusal) + recorded := false + a.Record = func(context.Context) error { + assert.True(t, suspended(sim.Kustomization(checkoutKey)), "not resumed before it is recorded") + recorded = true + return nil + } + _, err := simFlux(sim).Request(context.Background(), checkoutKey, a) + require.NoError(t, err) + assert.True(t, recorded) + + sim = simCluster(t, "main@sha1:"+testCommit, true) + a.Record = func(context.Context) error { return errors.New("the trigger is being deleted") } + _, err = simFlux(sim).Request(context.Background(), checkoutKey, a) + require.ErrorContains(t, err, "record the operation") + assert.False(t, isPermanent(err)) + assert.True(t, suspended(sim.Kustomization(checkoutKey)), "nothing resumed") +} + +// TestFluxRequest_AStaleDecisionResumesNothing is the customer review's first blocker, still: a +// decision is checked again just before the resume, and one that no longer holds resumes nothing. +func TestFluxRequest_AStaleDecisionResumesNothing(t *testing.T) { + for _, artifact := range []string{"main@sha1:" + testCommit, "main@sha1:" + olderCommit} { + sim := simCluster(t, artifact, true) + a := ask(v1alpha3.ApplyReasonRefusal) + a.Current = func() bool { return false } + _, err := simFlux(sim).Request(context.Background(), checkoutKey, a) + require.ErrorIs(t, err, errStale, artifact) + assert.True(t, suspended(sim.Kustomization(checkoutKey)), "a stale decision resumes nothing") + } +} + +// TestFluxRequest_AnArtifactThatMovesBeforeTheResumeIsStale is measurement point 6: a resume applies +// whatever the artifact holds then, so the artifact is read again just before it, and one that +// moved off the commit resumes nothing. +func TestFluxRequest_AnArtifactThatMovesBeforeTheResumeIsStale(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + reads := 0 + sim.Client.PrependReactor("get", "gitrepositories", func(k8stesting.Action) (bool, runtime.Object, error) { + reads++ + if reads < 2 { + return false, nil, nil + } + return true, gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), + "main@sha1:"+olderCommit), nil + }) + _, err := simFlux(sim).Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.ErrorIs(t, err, errStale) + assert.True(t, suspended(sim.Kustomization(checkoutKey))) +} + +// resumedWithT1 reports whether the Kustomization was resumed with the token ask gives, T1. +func resumedWithT1(ks *unstructured.Unstructured) bool { + return !suspended(ks) || ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation] == "T1" +} + +// TestFluxRequest_AKustomizationThatChangesDuringTheFetchIsNotResumed is review 5's fourth +// finding: a request read the Kustomization once, waited for the fetch, and resumed whatever the +// Kustomization was by then. One that applies another source now, one that was replaced, and one +// whose spec.suspend another manager applies now are all read again before the resume, and none is +// resumed. +func TestFluxRequest_AKustomizationThatChangesDuringTheFetchIsNotResumed(t *testing.T) { + for name, tt := range map[string]struct { + change func(t *testing.T, sim *fluxsim.Sim) + wantErr string + wantPerm bool + }{ + "it applies another GitRepository now": { + change: func(_ *testing.T, sim *fluxsim.Sim) { + sim.Edit(checkoutKey, func(ks *unstructured.Unstructured) { + _ = unstructured.SetNestedMap(ks.Object, gitRepoSourceRef("other"), "spec", "sourceRef") + }) + }, + wantErr: "applies GitRepository flux-system/other now", + }, + "it was replaced": { + change: func(t *testing.T, sim *fluxsim.Sim) { + // Through the tracker: a reactor runs while the client is locked. + tracker := sim.Client.Tracker() + held, err := tracker.Get(kustomizationGVR(), fluxSystemNS, checkoutKey.Name) + require.NoError(t, err) + replacement, _ := held.(*unstructured.Unstructured) + replacement = replacement.DeepCopy() + require.NoError(t, tracker.Delete(kustomizationGVR(), fluxSystemNS, checkoutKey.Name)) + replacement.SetUID("checkout-2") + replacement.SetResourceVersion("replaced") + require.NoError(t, tracker.Create(kustomizationGVR(), replacement, fluxSystemNS)) + }, + wantErr: "was replaced", + }, + "another manager applies its spec.suspend now": { + change: func(t *testing.T, sim *fluxsim.Sim) { + sim.Edit(checkoutKey, func(ks *unstructured.Unstructured) { + fluxsim.OwnSuspend(t, ks, true, "kustomize-controller", metav1.ManagedFieldsOperationApply) + }) + }, + wantErr: `applied by field manager "kustomize-controller"`, wantPerm: true, + }, + } { + t.Run(name, func(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + require.NoError(t, sim.Client.Tracker().Add( + gitRepositoryObj(fluxSystemNS, "other", testRepoURL, branchRef(testBranch), "main@sha1:"+testCommit))) + changed := false + stamped := func(k8stesting.Action) (bool, runtime.Object, error) { + if !changed { + // The GitRepository is stamped: the request waits for the fetch now. + changed = true + tt.change(t, sim) + } + return false, nil, nil + } + sim.Client.PrependReactor("patch", "gitrepositories", stamped) + recorded := false + a := ask(v1alpha3.ApplyReasonForeignCommit) + a.Record = func(context.Context) error { + recorded = true + return nil + } + + _, err := simFlux(sim).Request(context.Background(), checkoutKey, a) + + require.ErrorContains(t, err, tt.wantErr) + assert.Equal(t, tt.wantPerm, isPermanent(err)) + require.NotErrorIs(t, err, errStale) + assert.True(t, changed) + assert.False(t, recorded, "found before anything is recorded") + assert.False(t, resumedWithT1(sim.Kustomization(checkoutKey)), "nothing resumed") + }) + } +} + +// TestFluxRequest_TheResumeIsGuardedByTheKustomizationAsRead: what changes between the last read +// and the resume (here, while the operation is recorded) makes the resume's preconditions fail, +// so it never lands on an object other than the one checked. A change that leaves the binding as +// it was is read again and resumed; one that does not is not resumed. +func TestFluxRequest_TheResumeIsGuardedByTheKustomizationAsRead(t *testing.T) { + for name, tt := range map[string]struct { + change func(ks *unstructured.Unstructured) + wantErr string + }{ + "a label, which changes nothing it was checked for": { + change: func(ks *unstructured.Unstructured) { ks.SetLabels(map[string]string{"team": "platform"}) }, + }, + "another GitRepository": { + change: func(ks *unstructured.Unstructured) { + _ = unstructured.SetNestedMap(ks.Object, gitRepoSourceRef("other"), "spec", "sourceRef") + }, + wantErr: "applies GitRepository flux-system/other now", + }, + } { + t.Run(name, func(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + var resumes []string + counted := func(action k8stesting.Action) (bool, runtime.Object, error) { + patch, _ := action.(k8stesting.PatchActionImpl) + if patch.PatchType == types.ApplyPatchType && strings.Contains(string(patch.Patch), `"suspend":false`) { + resumes = append(resumes, string(patch.Patch)) + } + return false, nil, nil + } + sim.Client.PrependReactor("patch", "kustomizations", counted) + a := ask(v1alpha3.ApplyReasonRefusal) + a.Record = func(context.Context) error { + sim.Edit(checkoutKey, tt.change) + return nil + } + + started, err := simFlux(sim).Request(context.Background(), checkoutKey, a) + + require.NotEmpty(t, resumes) + assert.Contains(t, resumes[0], `"resourceVersion"`, "the resume carries the resourceVersion read") + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.False(t, resumedWithT1(sim.Kustomization(checkoutKey)), "nothing resumed") + return + } + require.NoError(t, err) + require.NoError(t, started.Unconfirmed) + assert.Len(t, resumes, 2, "the first resume conflicted, and the second was made over a fresh read") + assert.True(t, resumedWithT1(sim.Kustomization(checkoutKey))) + }) + } +} + +// TestFluxRequest_ADecisionThatGoesStaleWhileRecordedResumesNothing is review 5's sixth finding: +// recording the operation can wait on the status writer and the API server, and the branch can +// move on meanwhile. The decision is asked again after the record, just before the resume. +func TestFluxRequest_ADecisionThatGoesStaleWhileRecordedResumesNothing(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + holds := true + a := ask(v1alpha3.ApplyReasonRefusal) + a.Current = func() bool { return holds } + a.Record = func(context.Context) error { + holds = false + return nil + } + + _, err := simFlux(sim).Request(context.Background(), checkoutKey, a) + + require.ErrorIs(t, err, errStale) + assert.False(t, resumedWithT1(sim.Kustomization(checkoutKey)), "nothing resumed") +} + +// TestFluxRequest_AFetchThatEndsElsewhere: a fetch that does not bring the commit is the decision +// going stale when the branch moved on, and a failure (retried with backoff) when it did not. +func TestFluxRequest_AFetchThatEndsElsewhere(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + sim.Fetches(platformKey, "main@sha1:"+olderCommit) + _, err := simFlux(sim).Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonForeignCommit)) + require.ErrorContains(t, err, "fetched main@sha1:"+olderCommit+", not commit "+testCommit) + assert.False(t, isPermanent(err)) + require.NotErrorIs(t, err, errStale) + + a := ask(v1alpha3.ApplyReasonForeignCommit) + a.Current = func() bool { return false } + _, err = simFlux(sim).Request(context.Background(), checkoutKey, a) + require.ErrorIs(t, err, errStale) + assert.True(t, suspended(sim.Kustomization(checkoutKey))) +} + +// TestFluxRequest_TakesSuspendBackFromAnUpdate: `flux resume` and `flux suspend` write spec.suspend +// with a merge patch, once. That is no manifest owning the field, so a request takes it back. +func TestFluxRequest_TakesSuspendBackFromAnUpdate(t *testing.T) { + ks := fluxsim.OwnSuspend(t, kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), true, + "flux", metav1.ManagedFieldsOperationUpdate) + sim := fluxsim.New(t, newSourceClient(ks, + gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+testCommit))) + flux := simFlux(sim) + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + follow(t, flux, &started) + assert.True(t, heldByUs(sim.Kustomization(checkoutKey))) + assert.Empty(t, suspendOwner(sim.Kustomization(checkoutKey))) +} + +// TestFluxProgress_AFailedApplyIsSuspendedAgain: the reconcile that read the token failed, which +// is Failed with Flux's message; the Kustomization is suspended again, so Flux's own retryInterval +// requeue applies nothing (a requeue into a suspended object does nothing). The worker owns the +// retry. +func TestFluxProgress_AFailedApplyIsSuspendedAgain(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + sim.FailApplies("ConfigMap/apps/x dry-run failed") + flux := simFlux(sim) + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonForeignCommit)) + require.NoError(t, err) + + steps := follow(t, flux, &started) + done := steps[len(steps)-1] + assert.Equal(t, git.ApplyFailed, done.Outcome) + assert.Contains(t, done.Message, "ConfigMap/apps/x dry-run failed") + assert.True(t, suspended(sim.Kustomization(checkoutKey))) + + sim.FailApplies("") + before := sim.Reconciles(checkoutKey) + sim.Requeue(checkoutKey) + eventually(t, func() bool { return sim.Reconciles(checkoutKey) > before }, "the retryInterval requeue runs") + assert.Empty(t, sim.Applied(), "and applies nothing") +} + +// TestFluxProgress_ADependencyNotReadyFails: a dependency that is not ready is a failure of its own +// reason, retried by the worker like any other. +func TestFluxProgress_ADependencyNotReadyFails(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + sim.BlockDependencies(true) + flux := simFlux(sim) + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + done := follow(t, flux, &started) + assert.Equal(t, git.ApplyFailed, done[len(done)-1].Outcome) + assert.Contains(t, done[len(done)-1].Message, "a dependency is not ready") +} + +// TestFluxProgress_ASuspendDoesNotStopARunningApply is measurement point 2, and the Unresolved half +// of finding 4: an overdue operation is suspended at once, but that suspend does not stop the apply +// already running, so the operation completes only once the suspend's token is handled, which is +// after that apply ended. +func TestFluxProgress_ASuspendDoesNotStopARunningApply(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + held := sim.HoldNextApply() + flux := simFlux(sim) + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonForeignCommit)) + require.NoError(t, err) + <-held + + started.Overdue = true + progress, err := flux.Progress(context.Background(), checkoutKey, &started) + require.NoError(t, err) + assert.Equal(t, git.ApplySettling, progress.Outcome, "suspended at once") + require.NotEmpty(t, progress.SuspendToken) + assert.True(t, suspended(sim.Kustomization(checkoutKey))) + progress, err = flux.Progress(context.Background(), checkoutKey, &started) + require.NoError(t, err) + assert.Equal(t, git.ApplySettling, progress.Outcome, "the apply still runs, so the suspend is not handled") + + sim.Release() + steps := follow(t, flux, &started) + assert.Equal(t, git.ApplySucceeded, steps[len(steps)-1].Outcome) + assert.Equal(t, []string{"flux-system/checkout main@sha1:" + testCommit}, sim.Applied(), + "the suspend did not stop the apply") +} + +// TestFluxProgress_ATokenStampedOverOursStandsInForIt: `flux reconcile ks` between the resume and +// its reconcile replaces the token; the reconcile that handles the newer one started after the +// resume, so it ends the operation as ours would have. +func TestFluxProgress_ATokenStampedOverOursStandsInForIt(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + held := sim.HoldNextApply() + flux := simFlux(sim) + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + <-held + patch := []byte(`{"metadata":{"annotations":{"reconcile.fluxcd.io/requestedAt":"by-hand"}}}`) + _, err = sim.Client.Resource(kustomizationGVR()).Namespace(fluxSystemNS). + Patch(context.Background(), "checkout", types.MergePatchType, patch, metav1.PatchOptions{FieldManager: "flux"}) + require.NoError(t, err) + + sim.Release() + steps := follow(t, flux, &started) + assert.Equal(t, git.ApplySucceeded, steps[len(steps)-1].Outcome) + assert.True(t, suspended(sim.Kustomization(checkoutKey))) +} + +// TestFluxProgress_AResumeAfterTheSuspendIsSuspendedAgain: a Kustomization resumed by somebody else while +// the operation settles is suspended again before the operation completes. +func TestFluxProgress_AResumeAfterTheSuspendIsSuspendedAgain(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + flux := simFlux(sim) + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + var progress Progress + for progress.Outcome != git.ApplySettling { + progress, err = flux.Progress(context.Background(), checkoutKey, &started) + require.NoError(t, err) + } + first := started.SuspendToken + patch := []byte(`{"spec":{"suspend":false}}`) + _, err = sim.Client.Resource(kustomizationGVR()).Namespace(fluxSystemNS). + Patch(context.Background(), "checkout", types.MergePatchType, patch, metav1.PatchOptions{FieldManager: "flux"}) + require.NoError(t, err) + + steps := follow(t, flux, &started) + assert.Equal(t, git.ApplySucceeded, steps[len(steps)-1].Outcome) + assert.NotEqual(t, first, started.SuspendToken, "suspended again") + assert.True(t, suspended(sim.Kustomization(checkoutKey))) +} + +// TestFluxProgress_ASuspendBeforeTheReconcileIsNoApply is review 4's E: a suspend that lands +// between the resume and the reconcile that reads it makes that reconcile a suspended return, which +// marks the token handled and leaves the Ready of the last apply. A Refusal of the applied commit +// read that Ready as a clean Succeeded, with nothing reverted. Now no reconcile that built since +// the resume is no apply: Failed, and the worker asks again. +func TestFluxProgress_ASuspendBeforeTheReconcileIsNoApply(t *testing.T) { + for name, suspend := range map[string]func(t *testing.T, sim *fluxsim.Sim, flux *Flux){ + "flux suspend": func(t *testing.T, sim *fluxsim.Sim, _ *Flux) { + _, err := sim.Client.Resource(kustomizationGVR()).Namespace(fluxSystemNS).Patch(context.Background(), + "checkout", types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), + metav1.PatchOptions{FieldManager: "flux"}) + require.NoError(t, err) + }, + "the trigger's own suspend": func(t *testing.T, _ *fluxsim.Sim, flux *Flux) { + _, err := flux.Suspend(context.Background(), checkoutKey) + require.NoError(t, err) + }, + } { + t.Run(name, func(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + flux := simFlux(sim) + resume := sim.PauseKustomizations() + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + suspend(t, sim, flux) + resume() + + steps := follow(t, flux, &started) + done := steps[len(steps)-1] + assert.Equal(t, git.ApplyFailed, done.Outcome, "nothing was applied") + assert.Contains(t, done.Message, "no reconcile applied anything after the resume") + assert.Empty(t, sim.Applied()) + }) + } +} + +// TestFluxProgress_AnApplyOfAnOlderArtifactIsAskedAgain is review 4's G: kustomize-controller reads +// the GitRepository from its informer cache, so the reconcile the resume started can apply the +// artifact before the one the resume was checked against. That was Succeeded ("another revision"), +// and the commit then sat in a Kustomization suspended again, never applied. Now only an applied +// revision the source holds now is the source having moved on; an older one is Failed, and the next +// request applies the commit. +func TestFluxProgress_AnApplyOfAnOlderArtifactIsAskedAgain(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + sim.LagSourceCache(platformKey, "main@sha1:"+olderCommit) + flux := simFlux(sim) + + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonForeignCommit)) + require.NoError(t, err) + steps := follow(t, flux, &started) + done := steps[len(steps)-1] + assert.Equal(t, git.ApplyFailed, done.Outcome) + assert.Contains(t, done.Message, "an artifact older than the one the Kustomization was resumed for") + assert.Equal(t, "main@sha1:"+olderCommit, done.AppliedRevision) + + sim.LagSourceCache(platformKey, "") + again := ask(v1alpha3.ApplyReasonForeignCommit) + again.Token = "T2" + started, err = flux.Request(context.Background(), checkoutKey, again) + require.NoError(t, err) + require.Equal(t, OutcomeResumed, started.Outcome, "the commit is not applied, so it is not current") + steps = follow(t, flux, &started) + assert.Equal(t, git.ApplySucceeded, steps[len(steps)-1].Outcome) + assert.Equal(t, "main@sha1:"+testCommit, steps[len(steps)-1].AppliedRevision) +} + +// TestFluxProgress_AnApplyOfANewerArtifactSucceeds is the residual race the design accepts: the +// source moved on between the recheck and the reconcile, which applied what the source holds now. +func TestFluxProgress_AnApplyOfANewerArtifactSucceeds(t *testing.T) { + const newer = "main@sha1:9999999999999999999999999999999999999999" + sim := simCluster(t, "main@sha1:"+testCommit, true) + flux := simFlux(sim) + resume := sim.PauseKustomizations() + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + require.NoError(t, err) + sim.WriteStatus(context.Background(), gitRepositoryGVR(), platformKey, func(obj *unstructured.Unstructured) { + _ = unstructured.SetNestedField(obj.Object, newer, "status", "artifact", "revision") + }) + resume() + + steps := follow(t, flux, &started) + done := steps[len(steps)-1] + assert.Equal(t, git.ApplySucceeded, done.Outcome) + assert.Equal(t, newer, done.AppliedRevision) + assert.Contains(t, done.Message, "not commit "+testCommit) +} + +// TestFluxProgress_ARecoveredTokenStampedOverIsJudgedOnWhatRan is review 4's I: a process that +// resumed with T1, saw the apply end and suspended it again, then stopped before reporting, +// leaves T1 nowhere on the Kustomization. That ended Failed, though the apply ran. Now a recovered +// token handled by a later one is judged on what ran since the operation was asked for. +func TestFluxProgress_ARecoveredTokenStampedOverIsJudgedOnWhatRan(t *testing.T) { + sim := simCluster(t, "main@sha1:"+olderCommit, true) + flux := simFlux(sim) + asked := time.Now() + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonForeignCommit)) + require.NoError(t, err) + var progress Progress + for progress.Outcome != git.ApplySettling { + progress, err = flux.Progress(context.Background(), checkoutKey, &started) + require.NoError(t, err) + } + eventually(t, func() bool { return tokenHandled(sim.Kustomization(checkoutKey), started.SuspendToken) }, + "the suspend is handled; then the process stops") + + recovered := Started{Outcome: OutcomeResumed, Token: "T1", Commit: testCommit, Recovered: true, RequestedAt: asked} + steps := follow(t, flux, &recovered) + done := steps[len(steps)-1] + assert.Equal(t, git.ApplySucceeded, done.Outcome, done.Message) + assert.Equal(t, "main@sha1:"+testCommit, done.AppliedRevision) + assert.Len(t, sim.Applied(), 1) +} + +// TestFluxRequest_AnUnansweredResumeIsFollowed is review 4's B, the undelivered half: a resume +// whose write timed out may still have been made, and a RequestFailed left the Kustomization +// resumed with nothing following it. Now it is an Unconfirmed operation, followed like a recovered +// one: a resume that was made applies and is suspended again; one that was not is Failed, and +// nothing ran. A write the API server refused is still an error. +func TestFluxRequest_AnUnansweredResumeIsFollowed(t *testing.T) { + for name, tt := range map[string]struct { + landed bool + answer error + want git.ApplyOutcome + applied int + }{ + "made, then timed out": {landed: true, answer: context.DeadlineExceeded, want: git.ApplySucceeded, applied: 1}, + "not made, and timed out": {answer: context.DeadlineExceeded, want: git.ApplyFailed}, + "made, then a 504 answers": {landed: true, answer: apierrors.NewTimeoutError("slow", 1), want: git.ApplySucceeded, applied: 1}, + "refused": {answer: apierrors.NewForbidden(kustomizationGVR().GroupResource(), "checkout", errors.New("no"))}, + } { + t.Run(name, func(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, true) + flux := simFlux(sim) + answered := false + resumeTheResume := func(obj *unstructured.Unstructured) { + _ = unstructured.SetNestedField(obj.Object, false, "spec", "suspend") + obj.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: "T1"}) + obj.SetGeneration(obj.GetGeneration() + 1) + } + sim.Client.PrependReactor( + "patch", + "kustomizations", + func(action k8stesting.Action) (bool, runtime.Object, error) { + patch, _ := action.(k8stesting.PatchActionImpl) + resume := patch.PatchType == types.ApplyPatchType && + strings.Contains(string(patch.Patch), `"suspend":false`) + if answered || !resume { + return false, nil, nil + } + answered = true + if tt.landed { + sim.WriteStatus(context.Background(), kustomizationGVR(), checkoutKey, resumeTheResume) + } + return true, nil, tt.answer + }, + ) + + started, err := flux.Request(context.Background(), checkoutKey, ask(v1alpha3.ApplyReasonRefusal)) + if tt.want == "" { + require.ErrorContains(t, err, "resume") + return + } + require.NoError(t, err) + require.Error(t, started.Unconfirmed) + steps := follow(t, flux, &started) + done := steps[len(steps)-1] + assert.Equal(t, tt.want, done.Outcome, done.Message) + assert.Len(t, sim.Applied(), tt.applied) + assert.True(t, suspended(sim.Kustomization(checkoutKey)), "suspended again either way") + }) + } +} + +// TestReconcileMark_MovesOnAReconcileThatBuilt pins the evidence a verdict reads: Ready's +// lastTransitionTime, and the latest status.history snapshot (kustomize-controller v1.7 and later), +// either of which moving is a reconcile that got past the suspend check. +func TestReconcileMark_MovesOnAReconcileThatBuilt(t *testing.T) { + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + fluxsim.SetReady(ks, metav1.ConditionTrue, fluxmeta.ReconciliationSucceededReason, "applied") + before := markOf(ks) + + moved := ks.DeepCopy() + conditions, _, _ := unstructured.NestedSlice(moved.Object, "status", "conditions") + conditions[0].(map[string]any)["lastTransitionTime"] = "2026-10-18T10:00:05Z" + require.NoError(t, unstructured.SetNestedSlice(moved.Object, conditions, "status", "conditions")) + assert.NotEqual(t, before, markOf(moved), "Ready moved") + assert.True(t, markOf(moved).since(time.Date(2026, 10, 18, 10, 0, 5, 900, time.UTC)), "within the second") + assert.False(t, markOf(moved).since(time.Date(2026, 10, 18, 10, 0, 6, 0, time.UTC))) + + counted := ks.DeepCopy() + require.NoError(t, unstructured.SetNestedSlice(counted.Object, []any{map[string]any{ + "digest": "sha256:abc", "totalReconciliations": int64(2), "lastReconciled": "2026-10-18T10:00:05Z", + }}, "status", "history")) + again := counted.DeepCopy() + history, _, _ := unstructured.NestedSlice(again.Object, "status", "history") + history[0].(map[string]any)["totalReconciliations"] = int64(3) + require.NoError(t, unstructured.SetNestedSlice(again.Object, history, "status", "history")) + assert.NotEqual(t, markOf(counted), markOf(again), "one more reconcile of the same digest, in the same second") + assert.False(t, before.since(time.Time{}), "no mark at all is no evidence") +} + +// TestFlux_SuspendAndRelease: the trigger reconciler's half. Suspend suspends, Suspension reports +// held and handled once the token is handled, and Release hands spec.suspend back: the Kustomization +// is resumed, and holds nothing of GitOps Reverser's. +func TestFlux_SuspendAndRelease(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, false) + flux := simFlux(sim) + + before, err := flux.Suspension(context.Background(), checkoutKey) + require.NoError(t, err) + assert.False(t, before.Suspended) + assert.False(t, before.Held) + + token, err := flux.Suspend(context.Background(), checkoutKey) + require.NoError(t, err) + eventually(t, func() bool { + held, err := flux.Suspension(context.Background(), checkoutKey) + return err == nil && held.Suspended && held.Held && held.Handled && + sim.Kustomization(checkoutKey).GetAnnotations()[fluxmeta.ReconcileRequestAnnotation] == token + }, "held and handled") + + keeper, err := flux.Release(context.Background(), checkoutKey) + require.NoError(t, err) + assert.Empty(t, keeper) + ks := sim.Kustomization(checkoutKey) + assert.False(t, suspended(ks)) + assert.False(t, heldByUs(ks)) + assert.Empty(t, ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]) +} + +// TestFlux_ReleaseResumesASuspendItSharedWithFlux is the fourth review's co-ownership gap: a suspend +// that took over a `flux suspend` shares spec.suspend with that update, so the empty apply alone +// left the Kustomization suspended while the Released Event said it resumed. Release takes the +// update's share first. A manager that applies spec.suspend keeps it, and Release names it. +func TestFlux_ReleaseResumesASuspendItSharedWithFlux(t *testing.T) { + sim := simCluster(t, "main@sha1:"+testCommit, false) + flux := simFlux(sim) + patch := []byte(`{"spec":{"suspend":true}}`) + _, err := sim.Client.Resource(kustomizationGVR()).Namespace(fluxSystemNS). + Patch(context.Background(), "checkout", types.MergePatchType, patch, metav1.PatchOptions{FieldManager: "flux"}) + require.NoError(t, err) + _, err = flux.Suspend(context.Background(), checkoutKey) + require.NoError(t, err) + require.ElementsMatch(t, []string{"flux", SuspendFieldManager}, suspendManagers(sim.Kustomization(checkoutKey)), + "shared, as the API server shares it") + + keeper, err := flux.Release(context.Background(), checkoutKey) + require.NoError(t, err) + assert.Empty(t, keeper) + ks := sim.Kustomization(checkoutKey) + assert.False(t, suspended(ks), "resumed, as the Released Event says") + assert.Empty(t, suspendManagers(ks)) + + applied := fluxsim.OwnSuspend(t, kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), true, + "kustomize-controller", metav1.ManagedFieldsOperationApply) + sim = fluxsim.New(t, newSourceClient(applied)) + flux = simFlux(sim) + _, err = flux.Suspend(context.Background(), checkoutKey) + require.NoError(t, err) + keeper, err = flux.Release(context.Background(), checkoutKey) + require.NoError(t, err) + assert.Equal(t, "kustomize-controller", keeper, "its manifest sets spec.suspend") + assert.True(t, suspended(sim.Kustomization(checkoutKey))) +} + +// suspendManagers lists the field managers owning a Kustomization's spec.suspend. +func suspendManagers(ks *unstructured.Unstructured) []string { + var managers []string + for _, entry := range ks.GetManagedFields() { + if ownsField(entry, "f:spec", "f:suspend") { + managers = append(managers, entry.Manager) + } + } + return managers +} + +// TestFluxValidate_SuspendOwnedElsewhere: a manifest that sets spec.suspend, applied by another +// manager, would resume the Kustomization on its next apply: Configured=False, naming the manager. +// A merge patch by `flux resume` is no such owner. +func TestFluxValidate_SuspendOwnedElsewhere(t *testing.T) { + for name, tt := range map[string]struct { + operation metav1.ManagedFieldsOperationType + want string + }{ + "applied by a parent Kustomization": {metav1.ManagedFieldsOperationApply, ReasonSuspendOwnedElsewhere}, + "written once by flux resume": {metav1.ManagedFieldsOperationUpdate, ReasonSucceeded}, + } { + t.Run(name, func(t *testing.T) { + ks := fluxsim.OwnSuspend(t, kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), false, + git.FluxFieldManager, tt.operation) + flux := pinnedFlux(newSourceClient(ks, + gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), ""))) + flux.CanI = allowAll + verdict := flux.Validate(context.Background(), checkoutKey) + assert.Equal(t, tt.want, verdict.Reason, verdict.Message) + if tt.want == ReasonSuspendOwnedElsewhere { + assert.Contains(t, verdict.Message, `field manager "kustomize-controller"`) + assert.NotNil(t, verdict.Source) + } + }) + } +} + +// eventually waits for cond, failing the test after a few seconds. +func eventually(t *testing.T, cond func() bool, why string) { + t.Helper() + require.Eventually(t, cond, 5*time.Second, time.Millisecond, why) +} + +// TestFluxProgress_AnExternalResumeIsJudgedAsItsReconcilerJudgesIt is review 6: the trigger's +// reconciler records an external resume when it notices it, which can be after Flux applied it, and +// completes it on the Kustomization's last apply. A worker that recovered the record judged it on a +// reconcile run since the record's requestedAt, found none, and its Failed won over the +// reconciler's Succeeded. Now both read the last apply. +func TestFluxProgress_AnExternalResumeIsJudgedAsItsReconcilerJudgesIt(t *testing.T) { + noticed := stampTime.Truncate(time.Second) + appliedAt := noticed.Add(-40 * time.Second).UTC().Format(time.RFC3339) + ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) + ks = fluxsim.OwnSuspend(t, ks, true, SuspendFieldManager, metav1.ManagedFieldsOperationApply) + ks.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: "F1"}) + _ = unstructured.SetNestedField(ks.Object, "F1", "status", "lastHandledReconcileAt") + _ = unstructured.SetNestedField(ks.Object, "main@sha1:"+testCommit, "status", "lastAppliedRevision") + _ = unstructured.SetNestedSlice(ks.Object, []any{map[string]any{ + "type": "Ready", "status": "True", "reason": fluxmeta.ReconciliationSucceededReason, "message": "Applied", + "lastTransitionTime": appliedAt, + }}, "status", "conditions") + client := newSourceClient(ks, gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), + "main@sha1:"+testCommit)) + + recovered := &Started{Outcome: OutcomeResumed, SuspendToken: "F1", Recovered: true, RequestedAt: noticed} + got, err := NewFlux(client).Progress(context.Background(), checkoutKey, recovered) + + require.NoError(t, err) + assert.Equal(t, applyVerdict(ks, "").Outcome, got.Outcome, "the reconciler's verdict") + assert.Equal(t, git.ApplySucceeded, got.Outcome) +} diff --git a/internal/applier/flux_test.go b/internal/applier/flux_test.go index d0cbd7181..16105b93a 100644 --- a/internal/applier/flux_test.go +++ b/internal/applier/flux_test.go @@ -23,6 +23,8 @@ import ( k8stesting "k8s.io/client-go/testing" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" + "github.com/ConfigButler/gitops-reverser/internal/git" ) const ( @@ -43,6 +45,7 @@ func newSourceClient(objs ...runtime.Object) *dynamicfake.FakeDynamicClient { map[schema.GroupVersionResource]string{ kustomizationGVR(): "KustomizationList", gitRepositoryGVR(): "GitRepositoryList", + helmReleaseGVR(): "HelmReleaseList", applicationGVR(): "ApplicationList", }, objs...) } @@ -126,133 +129,6 @@ func TestSplitRevision_Formats(t *testing.T) { } } -// TestFluxRequest_Outcomes pins which object a request stamps, by whether the artifact already -// holds the commit and by reason. -func TestFluxRequest_Outcomes(t *testing.T) { - tests := []struct { - name string - revision string - commit string - reason v1alpha3.ApplyReason - want Outcome - wantSourceStamp bool - wantKustomStamps bool - }{ - { - name: "foreign commit not yet fetched stamps the source", revision: "main@sha1:" + olderCommit, - commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, - want: OutcomeStampedSource, wantSourceStamp: true, - }, - { - name: "catch-up behind stamps the source", revision: "main@sha1:" + olderCommit, - commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, - want: OutcomeStampedSource, wantSourceStamp: true, - }, - { - name: "no artifact yet stamps the source", revision: "", - commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, - want: OutcomeStampedSource, wantSourceStamp: true, - }, - { - name: "branch absent on the remote stamps the source", revision: "main@sha1:" + testCommit, - commit: "", reason: v1alpha3.ApplyReasonForeignCommit, - want: OutcomeStampedSource, wantSourceStamp: true, - }, - { - name: "refusal at the artifact's commit stamps the kustomization", revision: "main@sha1:" + testCommit, - commit: testCommit, reason: v1alpha3.ApplyReasonRefusal, - want: OutcomeStampedKustomization, wantKustomStamps: true, - }, - { - // The artifact that holds the commit already woke the Kustomization when it arrived. A - // stamp here would only re-apply it, and if the tip had moved on since, re-apply an - // older commit over a published edit. - name: "foreign commit already fetched stamps nothing", revision: "main/" + testCommit, - commit: testCommit, reason: v1alpha3.ApplyReasonForeignCommit, - want: OutcomeCurrent, - }, - { - name: "catch-up already current stamps nothing", revision: "sha1:" + testCommit, - commit: testCommit, reason: v1alpha3.ApplyReasonCatchUp, - want: OutcomeCurrent, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - client := newSourceClient( - kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), - gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), tt.revision), - ) - got, err := pinnedFlux(client).Request(context.Background(), - types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, testBinding, tt.commit, tt.reason, nil) - require.NoError(t, err) - assert.Equal(t, tt.want, got.Outcome) - - wantStamp := func(want bool) string { - if want { - return stampTime.Format(time.RFC3339Nano) - } - return "" - } - assert.Equal(t, wantStamp(tt.wantSourceStamp), - requestedAt(t, client, gitRepositoryGVR(), fluxSystemNS, "platform")) - assert.Equal(t, wantStamp(tt.wantKustomStamps), - requestedAt(t, client, kustomizationGVR(), fluxSystemNS, "checkout")) - }) - } -} - -// TestFluxRequest_SourceRefNamespace: an omitted sourceRef namespace is the Kustomization's own; a -// set one is followed across namespaces. -// TestFluxRequest_AStaleDecisionStampsNothing is the customer review's first blocker: a request -// decided at one commit, landing after a newer push, stamped the Kustomization of an artifact that -// held the older commit, re-applying it over a published edit. The decision is checked again just -// before the stamp. -func TestFluxRequest_AStaleDecisionStampsNothing(t *testing.T) { - for _, revision := range []string{"main@sha1:" + testCommit, "main@sha1:" + olderCommit} { - client := newSourceClient( - kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), - gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), revision), - ) - _, err := pinnedFlux(client).Request(context.Background(), - types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, testBinding, testCommit, - v1alpha3.ApplyReasonRefusal, func() bool { return false }) - require.ErrorIs(t, err, errStale, revision) - for _, action := range client.Actions() { - assert.NotEqual(t, "patch", action.GetVerb(), "a stale decision stamps nothing") - } - } -} - -func TestFluxRequest_SourceRefNamespace(t *testing.T) { - tests := []struct { - name string - sourceRef map[string]any - wantNamespace string - }{ - {"defaults to the kustomization's", gitRepoSourceRef("platform"), "apps"}, - { - "follows an explicit one", - map[string]any{"kind": kindGitRepository, "name": "platform", "namespace": fluxSystemNS}, - fluxSystemNS, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - client := newSourceClient( - kustomizationObj("apps", "checkout", tt.sourceRef), - gitRepositoryObj(tt.wantNamespace, "platform", testRepoURL, branchRef(testBranch), ""), - ) - kustomization := types.NamespacedName{Namespace: "apps", Name: "checkout"} - got, err := pinnedFlux(client).Request(context.Background(), kustomization, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) - require.NoError(t, err) - assert.Equal(t, OutcomeStampedSource, got.Outcome) - assert.NotEmpty(t, requestedAt(t, client, gitRepositoryGVR(), tt.wantNamespace, "platform")) - }) - } -} - func TestFluxRequest_Failures(t *testing.T) { tests := []struct { name string @@ -299,17 +175,29 @@ func TestFluxRequest_Failures(t *testing.T) { }, wantPermanent: true, }, + { + // A parent Kustomization applies this one's manifest with spec.suspend in it: it would + // resume the Kustomization on its next apply. + name: "spec.suspend is applied by another manager", + objs: []runtime.Object{ + fluxsim.OwnSuspend(t, kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), false, + git.FluxFieldManager, metav1.ManagedFieldsOperationApply), + gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), ""), + }, + wantPermanent: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := newSourceClient(tt.objs...) _, err := pinnedFlux(client).Request(context.Background(), - types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, testBinding, testCommit, - v1alpha3.ApplyReasonForeignCommit, nil) + types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, Ask{ + Want: testBinding, Commit: testCommit, Reason: v1alpha3.ApplyReasonForeignCommit, Token: "T1", + }) require.Error(t, err) assert.Equal(t, tt.wantPermanent, isPermanent(err)) for _, action := range client.Actions() { - assert.NotEqual(t, "patch", action.GetVerb(), "a failed request stamps nothing") + assert.NotEqual(t, "patch", action.GetVerb(), "a failed request writes nothing") } }) } @@ -324,7 +212,7 @@ func denyOn(resource string) AccessCheck { } } -// TestFluxValidate_Reasons covers every ApplierConfigured answer. +// TestFluxValidate_Reasons covers every Configured answer. func TestFluxValidate_Reasons(t *testing.T) { healthyKustomization := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) repo := func(url string, ref map[string]any) runtime.Object { @@ -372,47 +260,41 @@ func TestFluxValidate_Reasons(t *testing.T) { name: "git repository missing", objs: []runtime.Object{healthyKustomization}, wantReason: ReasonApplierNotFound, - wantMessage: "GitRepository flux-system/platform not found in the source cluster (the sourceRef of", + wantMessage: "GitRepository flux-system/platform not found in the applier's cluster (the sourceRef of", }, { name: "source is a bucket", objs: []runtime.Object{kustomizationObj(fluxSystemNS, "checkout", map[string]any{"kind": "Bucket", "name": "platform"})}, - wantReason: ReasonSourceMismatch, + wantReason: ReasonUnsupportedSource, wantMessage: "applies from a Bucket, not a GitRepository", }, { - name: "another repository", + name: "another repository and branch, which only a GitTarget can judge", objs: []runtime.Object{ healthyKustomization, - repo("https://github.com/acme/other", branchRef(testBranch)), + repo("https://github.com/acme/other", branchRef("release")), }, - wantReason: ReasonSourceMismatch, - wantMessage: "fetches https://github.com/acme/other, not https://github.com/acme/platform", - }, - { - name: "another branch", - objs: []runtime.Object{healthyKustomization, repo(testRepoURL, branchRef("release"))}, - wantReason: ReasonSourceMismatch, - wantMessage: `fetches branch "release", not main`, + wantReason: ReasonSucceeded, + wantMessage: "fetches branch release of https://github.com/acme/other", }, { name: "no ref", objs: []runtime.Object{healthyKustomization, repo(testRepoURL, nil)}, - wantReason: ReasonSourceMismatch, + wantReason: ReasonUnsupportedSource, wantMessage: "names no spec.ref.branch", }, { name: "pinned tag", objs: []runtime.Object{healthyKustomization, repo(testRepoURL, map[string]any{"tag": "v1.0.0"})}, - wantReason: ReasonSourceMismatch, + wantReason: ReasonUnsupportedSource, wantMessage: `pins spec.ref.tag "v1.0.0"`, }, { name: "pinned commit beside the branch", objs: []runtime.Object{healthyKustomization, repo(testRepoURL, map[string]any{"branch": testBranch, "commit": testCommit})}, - wantReason: ReasonSourceMismatch, + wantReason: ReasonUnsupportedSource, wantMessage: "pins spec.ref.commit", }, { @@ -469,7 +351,7 @@ func TestFluxValidate_Reasons(t *testing.T) { } got := flux.Validate(context.Background(), - types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, testBinding) + types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}) assert.Equal(t, tt.wantReason, got.Reason, got.Message) assert.Equal(t, tt.wantReason == ReasonSucceeded, got.OK) assert.Contains(t, got.Message, tt.wantMessage) @@ -478,36 +360,62 @@ func TestFluxValidate_Reasons(t *testing.T) { } } -// TestFluxValidate_URLEquivalences: the GitRepository's url and the GitProvider's are compared as -// repositories, not as strings. -func TestFluxValidate_URLEquivalences(t *testing.T) { +// TestFluxValidate_ReportsTheSource: what the Kustomization applies is reported whatever the +// verdict, once the GitRepository could be read, so a GitTarget can judge it. +func TestFluxValidate_ReportsTheSource(t *testing.T) { + client := newSourceClient( + kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), + gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), ""), + ) + flux := pinnedFlux(client) + flux.CanI = denyOn("kustomizations") + got := flux.Validate(context.Background(), types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}) + assert.Equal(t, ReasonForbidden, got.Reason) + assert.Equal(t, &Source{ + GitRepository: types.NamespacedName{Namespace: fluxSystemNS, Name: "platform"}, + URL: testRepoURL, Revision: testBranch, + }, got.Source) +} + +// TestBindingMismatch_ComparesRepositoriesAndBranches: a trigger's source and the GitProvider's url +// are compared as repositories, not as strings, and the branch exactly. +func TestBindingMismatch_ComparesRepositoriesAndBranches(t *testing.T) { tests := []struct { name string sourceURL string + branch string provider string - wantOK bool + want string }{ - {"identical", testRepoURL, testRepoURL, true}, - {"trailing .git", testRepoURL + ".git", testRepoURL, true}, - {"scp-style ssh", "git@github.com:acme/platform.git", testRepoURL, true}, - {"ssh url", "ssh://git@github.com/acme/platform", "git@github.com:acme/platform.git", true}, - {"case and trailing slash", "https://GitHub.com/Acme/Platform/", testRepoURL, true}, - {"default port", "https://github.com:443/acme/platform", testRepoURL, true}, - {"another org", "https://github.com/other/platform", testRepoURL, false}, - {"another host", "https://gitlab.com/acme/platform", testRepoURL, false}, + {"identical", testRepoURL, testBranch, testRepoURL, ""}, + {"trailing .git", testRepoURL + ".git", testBranch, testRepoURL, ""}, + {"scp-style ssh", "git@github.com:acme/platform.git", testBranch, testRepoURL, ""}, + {"ssh url", "ssh://git@github.com/acme/platform", testBranch, "git@github.com:acme/platform.git", ""}, + {"case and trailing slash", "https://GitHub.com/Acme/Platform/", testBranch, testRepoURL, ""}, + {"default port", "https://github.com:443/acme/platform", testBranch, testRepoURL, ""}, + { + "another org", + "https://github.com/other/platform", + testBranch, + testRepoURL, + "applies https://github.com/other", + }, + {"another host", "https://gitlab.com/acme/platform", testBranch, testRepoURL, "applies https://gitlab.com"}, + {"another branch", testRepoURL, "release", testRepoURL, `applies branch "release", not main`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := newSourceClient( - kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")), - gitRepositoryObj(fluxSystemNS, "platform", tt.sourceURL, branchRef(testBranch), ""), - ) - flux := pinnedFlux(client) - flux.CanI = allowAll - got := flux.Validate(context.Background(), - types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"}, - Binding{RepositoryURL: tt.provider, Branch: testBranch}) - assert.Equal(t, tt.wantOK, got.OK, got.Message) + trigger := &v1alpha3.FluxTrigger{ObjectMeta: metav1.ObjectMeta{Name: "apps"}} + assert.Empty(t, BindingMismatch(trigger, Binding{RepositoryURL: tt.provider, Branch: testBranch}), + "a trigger that has read nothing yet is not a mismatch") + trigger.Status.Source = &v1alpha3.FluxTriggerSource{URL: tt.sourceURL, Branch: tt.branch} + got := BindingMismatch(trigger, Binding{RepositoryURL: tt.provider, Branch: testBranch}) + if tt.want == "" { + assert.Empty(t, got) + return + } + assert.Contains(t, got, tt.want) + assert.Contains(t, got, "FluxTrigger apps") }) } } @@ -548,75 +456,3 @@ func TestSelfSubjectAccessCheck_AsksTheSourceCluster(t *testing.T) { _, err = check(context.Background(), authorizationv1.ResourceAttributes{Verb: "patch"}) require.Error(t, err) } - -// setStatus writes fields under an object's status in the fake source cluster, as Flux would. -func setStatus(t *testing.T, client *dynamicfake.FakeDynamicClient, gvr schema.GroupVersionResource, - ref types.NamespacedName, value string, fields ...string, -) { - t.Helper() - resource := client.Resource(gvr).Namespace(ref.Namespace) - obj, err := resource.Get(context.Background(), ref.Name, metav1.GetOptions{}) - require.NoError(t, err) - require.NoError(t, unstructured.SetNestedField(obj.Object, value, append([]string{"status"}, fields...)...)) - _, err = resource.Update(context.Background(), obj, metav1.UpdateOptions{}) - require.NoError(t, err) -} - -// TestFlux_Handled pins when an apply window closes: when Flux reports it finished the apply the -// stamp asked for, and not before, for each thing a request can stamp. -func TestFlux_Handled(t *testing.T) { - checkout := types.NamespacedName{Namespace: fluxSystemNS, Name: "checkout"} - platform := types.NamespacedName{Namespace: fluxSystemNS, Name: "platform"} - newClient := func() *dynamicfake.FakeDynamicClient { - ks := kustomizationObj(fluxSystemNS, "checkout", gitRepoSourceRef("platform")) - _ = unstructured.SetNestedField(ks.Object, "main@sha1:"+olderCommit, "status", "lastAttemptedRevision") - return newSourceClient(ks, - gitRepositoryObj(fluxSystemNS, "platform", testRepoURL, branchRef(testBranch), "main@sha1:"+olderCommit)) - } - request := func(t *testing.T, client *dynamicfake.FakeDynamicClient, reason v1alpha3.ApplyReason, commit string) Stamped { - t.Helper() - stamped, err := pinnedFlux(client).Request(context.Background(), checkout, testBinding, commit, reason, nil) - require.NoError(t, err) - return stamped - } - handled := func(t *testing.T, client *dynamicfake.FakeDynamicClient, stamped Stamped) bool { - t.Helper() - ok, err := pinnedFlux(client).Handled(context.Background(), checkout, stamped) - require.NoError(t, err) - return ok - } - - t.Run("a stamped Kustomization, once it handled the stamp", func(t *testing.T) { - client := newClient() - stamped := request(t, client, v1alpha3.ApplyReasonRefusal, olderCommit) - require.Equal(t, OutcomeStampedKustomization, stamped.Outcome) - assert.False(t, handled(t, client, stamped)) - setStatus(t, client, kustomizationGVR(), checkout, "an older stamp", "lastHandledReconcileAt") - assert.False(t, handled(t, client, stamped), "another request's answer is not this one's") - setStatus(t, client, kustomizationGVR(), checkout, stamped.Token, "lastHandledReconcileAt") - assert.True(t, handled(t, client, stamped)) - }) - - t.Run("a stamped source, once the Kustomization attempted the new revision", func(t *testing.T) { - client := newClient() - stamped := request(t, client, v1alpha3.ApplyReasonForeignCommit, testCommit) - require.Equal(t, OutcomeStampedSource, stamped.Outcome) - assert.False(t, handled(t, client, stamped)) - setStatus(t, client, kustomizationGVR(), checkout, "main@sha1:"+testCommit, "lastAttemptedRevision") - assert.True(t, handled(t, client, stamped)) - }) - - t.Run("a stamped source whose fetch brought nothing new, once the source handled the stamp", func(t *testing.T) { - client := newClient() - stamped := request(t, client, v1alpha3.ApplyReasonForeignCommit, testCommit) - setStatus(t, client, gitRepositoryGVR(), platform, stamped.Token, "lastHandledReconcileAt") - assert.True(t, handled(t, client, stamped), "no new artifact: nothing for the Kustomization to apply") - - setStatus(t, client, gitRepositoryGVR(), platform, "main@sha1:"+testCommit, "artifact", "revision") - assert.False(t, handled(t, client, stamped), "a new artifact: the Kustomization's apply is still to come") - }) - - t.Run("nothing stamped is nothing to wait for", func(t *testing.T) { - assert.True(t, handled(t, newClient(), Stamped{Outcome: OutcomeCurrent})) - }) -} diff --git a/internal/applier/fluxsim/fluxsim.go b/internal/applier/fluxsim/fluxsim.go new file mode 100644 index 000000000..02ddc6ada --- /dev/null +++ b/internal/applier/fluxsim/fluxsim.go @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Package fluxsim is a fake source cluster running kustomize-controller and source-controller, for +tests of code that drives Flux. It applies the source facts docs/design/apply-trigger.md +lists (kustomize-controller v1.9.6), and nothing more. +*/ +package fluxsim + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" +) + +// Sim is a source cluster running kustomize-controller and source-controller, as far as the +// source facts in docs/design/apply-trigger.md go (kustomize-controller v1.9.6, +// internal/controller/kustomization_controller.go). It is what the Flux client's lifecycle is tested +// against, so each fact it relies on is one this fake applies: +// +// - A reconcile reads the object once, when it starts. finalizeStatus is deferred and copies the +// requestedAt it read to status.lastHandledReconcileAt on every return (:143, :1335-1340). +// - A suspended object returns right after that read, before the source is resolved (:189-192): +// it fetches, builds and applies nothing, and its Ready is left as it was. +// - spec.suspend is read only when a reconcile starts, so a suspend does not stop one running. +// - lastAttemptedRevision is written before the build (:390); lastAppliedRevision and Ready=True +// once the apply finished. A failed apply is Ready=False; a dependency not ready is Ready=False, +// DependencyNotReady (:264-268). +// - observedGeneration is set from the generation read whenever Ready is True at the end (:1345). +// - A reconcile is enqueued by a new generation, a new requestedAt, or a new artifact of its +// source (kustomization_manager.go:139-173), and reconciles of one object never overlap. +// - A reconcile that gets past the suspend check marks Ready Unknown before it fetches, so the +// Ready it ends with has a new lastTransitionTime (:331); one that builds upserts its digest +// in status.history, counting the reconciles of it (:492-563, meta.History.Upsert). A +// suspended return touches neither. +// +// Two of kustomize-controller's circumstances are a test's to set: its source informer lagging +// behind the GitRepository (LagSourceCache), and its queue holding a reconcile back while the +// object changes under it (PauseKustomizations). +// +// Writes go through the fake dynamic client the client under test uses. spec.suspend and the +// requestedAt annotation keep managedFields, by manager and operation, as the API server does for a +// server-side apply (forced) and for a merge patch. A Kustomization's resourceVersion moves on every +// write that changes it, and a server-side apply that carries a resourceVersion or a uid has them +// checked as preconditions, as the API server checks them. +type Sim struct { + t testing.TB + // Client is the fake source cluster's client, which the code under test writes through. + Client *dynamicfake.FakeDynamicClient + // version is the last resourceVersion given out. + version atomic.Int64 + // writes makes each read-modify-write of an object (a patch, a status write) one step, as the + // API server's resourceVersion check does: the tracker does not, so a status write that read + // the object before a resume landed would otherwise write the resume away. + writes sync.Mutex + + mu sync.Mutex + // remote is the revision a GitRepository's fetch finds, by GitRepository. + remote map[types.NamespacedName]string + // seen is what each object's last reconcile started from, by object, to tell whether another + // is due. + seen map[string]seenState + // failing is the Ready=False message of every apply while set; dependencyNotReady fails every + // apply with DependencyNotReady. + failing string + dependencyNotReady bool + // applied lists the revisions applied, in order, as "namespace/name revision". + applied []string + // hold, while armed, stops the next apply after lastAttemptedRevision until it is closed; holding + // is closed once that apply is stopped there. + hold chan struct{} + holding chan struct{} + armed bool + // started counts reconciles started, by object. + started map[string]int + // paused holds every Kustomization reconcile back while set; lagged is the artifact revision + // kustomize-controller's cache still holds for a GitRepository, while it lags. + paused bool + lagged map[types.NamespacedName]string +} + +// seenState is what one reconcile of an object started from. +type seenState struct { + generation int64 + token string + artifact string +} + +// Field paths the fake keeps managedFields for. +const ( + simFieldSuspend = "f:spec.f:suspend" + simFieldRequestedAt = "f:metadata.f:annotations.f:" + fluxmeta.ReconcileRequestAnnotation +) + +// New wraps a fake source cluster's client and starts its controllers, which stop with the +// test. +func New(t testing.TB, client *dynamicfake.FakeDynamicClient) *Sim { + t.Helper() + sim := &Sim{ + t: t, Client: client, + remote: map[types.NamespacedName]string{}, + seen: map[string]seenState{}, + started: map[string]int{}, + lagged: map[types.NamespacedName]string{}, + } + client.PrependReactor("patch", "kustomizations", sim.patchKustomization) + client.PrependReactor("create", "kustomizations", sim.newVersion) + client.PrependReactor("update", "kustomizations", sim.newVersion) + // What the objects hold now is what their last reconcile started from: a test starts from a + // cluster at rest, not from the controllers' first pass over it. + for _, ks := range sim.list(KustomizationGVR()) { + if ks.GetResourceVersion() == "" { + ks.SetResourceVersion(sim.nextVersion()) + require.NoError(t, client.Tracker().Update(KustomizationGVR(), ks, ks.GetNamespace())) + } + sim.seen[types.NamespacedName{Namespace: ks.GetNamespace(), Name: ks.GetName()}.String()] = sim.seenNow(ks) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + sim.run(ctx) + }() + t.Cleanup(func() { + cancel() + sim.Release() + <-done + }) + return sim +} + +// Fetches sets what the next fetch of a GitRepository finds. +func (s *Sim) Fetches(source types.NamespacedName, revision string) { + s.mu.Lock() + defer s.mu.Unlock() + s.remote[source] = revision +} + +// FailApplies makes every apply fail with message, or succeed again when it is empty. +func (s *Sim) FailApplies(message string) { + s.mu.Lock() + defer s.mu.Unlock() + s.failing = message +} + +// BlockDependencies makes every apply wait on a dependency that is not ready. +func (s *Sim) BlockDependencies(blocked bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.dependencyNotReady = blocked +} + +// HoldNextApply stops the next apply after it wrote lastAttemptedRevision, and returns a channel +// that is closed once it is held there. Release lets it finish. +func (s *Sim) HoldNextApply() <-chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + s.hold, s.holding, s.armed = make(chan struct{}), make(chan struct{}), true + return s.holding +} + +// Release lets a held apply finish. +func (s *Sim) Release() { + s.mu.Lock() + defer s.mu.Unlock() + if s.hold != nil { + close(s.hold) + s.hold = nil + } +} + +// PauseKustomizations holds every Kustomization reconcile back until the returned func is called, +// as a busy queue does: a reconcile due meanwhile starts after it, from the object as it is then. +func (s *Sim) PauseKustomizations() func() { + s.mu.Lock() + defer s.mu.Unlock() + s.paused = true + return func() { + s.mu.Lock() + defer s.mu.Unlock() + s.paused = false + } +} + +// LagSourceCache makes kustomize-controller read revision as the artifact of the GitRepository +// source, as its informer does before it catches up with a fetch, until it is called with "". +// Whether a reconcile is due is still decided by the artifact the GitRepository holds. +func (s *Sim) LagSourceCache(source types.NamespacedName, revision string) { + s.mu.Lock() + defer s.mu.Unlock() + if revision == "" { + delete(s.lagged, source) + return + } + s.lagged[source] = revision +} + +// Applied is every apply so far, as "namespace/name revision". +func (s *Sim) Applied() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.applied...) +} + +// run is the two controllers: it reconciles whatever is due, one reconcile per object at a time. +func (s *Sim) run(ctx context.Context) { + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + for _, source := range s.list(GitRepositoryGVR()) { + s.reconcileSource(ctx, source) + } + for _, ks := range s.list(KustomizationGVR()) { + s.reconcileKustomization(ctx, ks) + } + } +} + +func (s *Sim) list(gvr schema.GroupVersionResource) []*unstructured.Unstructured { + list, err := s.Client.Tracker().List(gvr, gvr.GroupVersion().WithKind(itemKind(gvr)), "") + if err != nil { + return nil + } + ulist, ok := list.(*unstructured.UnstructuredList) + if !ok { + return nil + } + out := make([]*unstructured.Unstructured, 0, len(ulist.Items)) + for i := range ulist.Items { + out = append(out, &ulist.Items[i]) + } + return out +} + +// itemKind is the kind of the objects a resource lists, which the tracker names its list after. +func itemKind(gvr schema.GroupVersionResource) string { + if gvr == GitRepositoryGVR() { + return kindGitRepository + } + return kindKustomization +} + +// reconcileSource fetches when a GitRepository's requestedAt is new: its artifact becomes what the +// remote holds, and the request is marked handled. +func (s *Sim) reconcileSource(ctx context.Context, source *unstructured.Unstructured) { + token := source.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation] + if token == "" || token == statusString(source, "lastHandledReconcileAt") { + return + } + key := types.NamespacedName{Namespace: source.GetNamespace(), Name: source.GetName()} + s.mu.Lock() + revision := s.remote[key] + s.mu.Unlock() + s.WriteStatus(ctx, GitRepositoryGVR(), key, func(obj *unstructured.Unstructured) { + if revision != "" { + _ = unstructured.SetNestedField(obj.Object, revision, "status", "artifact", "revision") + } + _ = unstructured.SetNestedField(obj.Object, token, "status", "lastHandledReconcileAt") + }) +} + +// seenNow is what a reconcile of ks starting now would start from. +func (s *Sim) seenNow(ks *unstructured.Unstructured) seenState { + now := seenState{generation: ks.GetGeneration(), token: ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]} + if sourceRef, err := gitRepositoryRef(ks); err == nil { + if source, err := s.Client.Tracker().Get(GitRepositoryGVR(), sourceRef.Namespace, sourceRef.Name); err == nil { + if source, ok := source.(*unstructured.Unstructured); ok { + now.artifact = statusString(source, "artifact", "revision") + } + } + } + return now +} + +// Requeue makes a reconcile of the Kustomization due with nothing changed, as a requeue after +// retryInterval or interval does. +func (s *Sim) Requeue(key types.NamespacedName) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.seen, key.String()) +} + +// Reconciles is how many reconciles of the Kustomization started. +func (s *Sim) Reconciles(key types.NamespacedName) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.started[key.String()] +} + +// start starts a reconcile of ks when one is due and the controller is not paused: it returns what +// the reconcile starts from, and the artifact revision it reads, its source's or the one the +// controller's cache still holds. +func (s *Sim) start(key types.NamespacedName, ks *unstructured.Unstructured) (seenState, string, bool) { + sourceRef, err := gitRepositoryRef(ks) + if err != nil { + return seenState{}, "", false + } + now := s.seenNow(ks) + s.mu.Lock() + defer s.mu.Unlock() + if last, known := s.seen[key.String()]; s.paused || (known && last == now) { + return seenState{}, "", false + } + s.seen[key.String()] = now + s.started[key.String()]++ + artifact := now.artifact + if lagged, ok := s.lagged[sourceRef]; ok { + artifact = lagged + } + return now, artifact, true +} + +// reconcileKustomization runs one reconcile of a Kustomization when one is due. +func (s *Sim) reconcileKustomization(ctx context.Context, ks *unstructured.Unstructured) { + key := types.NamespacedName{Namespace: ks.GetNamespace(), Name: ks.GetName()} + now, artifact, due := s.start(key, ks) + if !due { + return + } + s.mu.Lock() + failing, blocked := s.failing, s.dependencyNotReady + s.mu.Unlock() + + suspended, _, _ := unstructured.NestedBool(ks.Object, "spec", "suspend") + finalize := func(change func(*unstructured.Unstructured)) { + s.WriteStatus(ctx, KustomizationGVR(), key, func(obj *unstructured.Unstructured) { + if change != nil { + change(obj) + } + if now.token != "" { + _ = unstructured.SetNestedField(obj.Object, now.token, "status", "lastHandledReconcileAt") + } + if readyStatus(obj) == string(metav1.ConditionTrue) { + _ = unstructured.SetNestedField(obj.Object, now.generation, "status", "observedGeneration") + } + }) + } + switch { + case suspended: + finalize(nil) + return + case blocked: + finalize(func(obj *unstructured.Unstructured) { + SetReady( + obj, + metav1.ConditionFalse, + fluxmeta.DependencyNotReadyReason, + "dependency 'apps/infra' is not ready", + ) + }) + return + } + + s.WriteStatus(ctx, KustomizationGVR(), key, func(obj *unstructured.Unstructured) { + _ = unstructured.SetNestedField(obj.Object, artifact, "status", "lastAttemptedRevision") + }) + s.mu.Lock() + hold, holding, armed := s.hold, s.holding, s.armed + s.armed = false + s.mu.Unlock() + if armed { + close(holding) + select { + case <-hold: + case <-ctx.Done(): + return + } + } + if failing != "" { + finalize(func(obj *unstructured.Unstructured) { + SetReady(obj, metav1.ConditionFalse, "ReconciliationFailed", failing) + reconciled(obj, artifact, "ReconciliationFailed") + }) + return + } + s.mu.Lock() + s.applied = append(s.applied, key.String()+" "+artifact) + s.mu.Unlock() + finalize(func(obj *unstructured.Unstructured) { + _ = unstructured.SetNestedField(obj.Object, artifact, "status", "lastAppliedRevision") + SetReady(obj, metav1.ConditionTrue, fluxmeta.ReconciliationSucceededReason, "Applied revision: "+artifact) + reconciled(obj, artifact, fluxmeta.ReconciliationSucceededReason) + }) +} + +// reconciled records what a reconcile that built leaves besides its outcome: Ready's +// lastTransitionTime is now, since the reconcile marked it Unknown first, and status.history counts +// one more reconcile of the digest built (meta.History.Upsert: the snapshot of the same digest and +// status is moved to the front and counted, or a new one is put there). The digest stands for the +// build of revision. +func reconciled(obj *unstructured.Unstructured, revision, status string) { + now := time.Now().UTC().Format(time.RFC3339) + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, c := range conditions { + if cond, ok := c.(map[string]any); ok && cond["type"] == fluxmeta.ReadyCondition { + cond["lastTransitionTime"] = now + } + } + _ = unstructured.SetNestedSlice(obj.Object, conditions, "status", "conditions") + + digest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(revision))) + history, _, _ := unstructured.NestedSlice(obj.Object, "status", "history") + snapshot := map[string]any{ + "digest": digest, "firstReconciled": now, "lastReconciledStatus": status, + "totalReconciliations": int64(0), "metadata": map[string]any{"revision": revision}, + } + rest := make([]any, 0, len(history)+1) + for _, h := range history { + if old, ok := h.(map[string]any); ok && old["digest"] == digest && old["lastReconciledStatus"] == status { + snapshot = old + continue + } + rest = append(rest, h) + } + total, _ := snapshot["totalReconciliations"].(int64) + snapshot["totalReconciliations"], snapshot["lastReconciled"] = total+1, now + _ = unstructured.SetNestedSlice(obj.Object, append([]any{snapshot}, rest...), "status", "history") +} + +// WriteStatus changes an object's status as the controller's patcher does: on the object as it is +// now, leaving its spec and metadata alone. A write that changes the object moves its +// resourceVersion. +func (s *Sim) WriteStatus(_ context.Context, gvr schema.GroupVersionResource, key types.NamespacedName, + change func(*unstructured.Unstructured)) { + s.writes.Lock() + defer s.writes.Unlock() + current, err := s.Client.Tracker().Get(gvr, key.Namespace, key.Name) + if err != nil { + return + } + held, ok := current.(*unstructured.Unstructured) + if !ok { + return + } + obj := held.DeepCopy() + change(obj) + if !equality.Semantic.DeepEqual(held.Object, obj.Object) { + obj.SetResourceVersion(s.nextVersion()) + } + _ = s.Client.Tracker().Update(gvr, obj, key.Namespace) +} + +// Edit changes a Kustomization as another writer's update does, on the object as it is now. It +// writes through the tracker rather than the client, so a test's reactor, which runs while the +// client is locked, can call it. +func (s *Sim) Edit(key types.NamespacedName, change func(*unstructured.Unstructured)) { + s.WriteStatus(context.Background(), KustomizationGVR(), key, change) +} + +// nextVersion gives out a new resourceVersion. +func (s *Sim) nextVersion() string { + return strconv.FormatInt(s.version.Add(1), 10) +} + +// newVersion gives the Kustomization a create or an update writes a new resourceVersion, and lets +// the tracker write it. +func (s *Sim) newVersion(action k8stesting.Action) (bool, runtime.Object, error) { + var obj runtime.Object + switch action := action.(type) { + case k8stesting.CreateActionImpl: + obj = action.GetObject() + case k8stesting.UpdateActionImpl: + obj = action.GetObject() + } + if ks, ok := obj.(*unstructured.Unstructured); ok { + ks.SetResourceVersion(s.nextVersion()) + } + return false, nil, nil +} + +// applyPreconditions is the API server's answer to a server-side apply that carries a +// resourceVersion or a uid (measured in envtest): a resourceVersion other than the object's is a +// Conflict, another object's uid is Invalid, and a uid with no object is a Conflict rather than a +// create. current is nil when there is no object. +func applyPreconditions(current *unstructured.Unstructured, body map[string]any, name string) error { + resourceVersion, _, _ := unstructured.NestedString(body, "metadata", "resourceVersion") + uid, _, _ := unstructured.NestedString(body, "metadata", "uid") + resource := KustomizationGVR().GroupResource() + switch { + case current == nil && uid != "": + return apierrors.NewConflict(resource, name, fmt.Errorf("uid mismatch: the provided object specified "+ + "uid %s, and no existing object was found", uid)) + case current == nil: + return nil + case resourceVersion != "" && resourceVersion != current.GetResourceVersion(): + return apierrors.NewConflict(resource, name, errors.New( + "the object has been modified; please apply your changes to the latest version and try again")) + case uid != "" && uid != string(current.GetUID()): + return apierrors.NewInvalid(schema.GroupKind{Group: KustomizationGVR().Group, Kind: kindKustomization}, name, + field.ErrorList{field.Invalid(field.NewPath("metadata", "uid"), uid, "field is immutable")}) + } + return nil +} + +// readyStatus is a Kustomization's Ready status, or "". +func readyStatus(obj *unstructured.Unstructured) string { + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, c := range conditions { + cond, _ := c.(map[string]any) + if cond["type"] == fluxmeta.ReadyCondition { + status, _ := cond["status"].(string) + return status + } + } + return "" +} + +// SetReady sets a Kustomization's Ready condition. +func SetReady(obj *unstructured.Unstructured, status metav1.ConditionStatus, reason, message string) { + _ = unstructured.SetNestedSlice(obj.Object, []any{map[string]any{ + "type": fluxmeta.ReadyCondition, "status": string(status), "reason": reason, "message": message, + }}, "status", "conditions") +} + +// patchKustomization serves a patch of a Kustomization: a server-side apply (always forced, as the +// client under test applies) or a merge patch, keeping managedFields for spec.suspend and the +// requestedAt annotation, and the generation for a spec change. Ownership follows what a real API +// server did (measured in envtest, internal/controller/fluxtrigger_suspend_apiserver_test.go): a +// write that changes a field's value takes it from every other owner; an update that writes the +// value the field holds takes nothing; an apply that does shares the field with its owners. +func (s *Sim) patchKustomization(action k8stesting.Action) (bool, runtime.Object, error) { + patch, ok := action.(k8stesting.PatchActionImpl) + if !ok { + return false, nil, nil + } + var body map[string]any + if err := json.Unmarshal(patch.Patch, &body); err != nil { + return true, nil, err + } + apply := patch.PatchType == types.ApplyPatchType + s.writes.Lock() + defer s.writes.Unlock() + held, err := s.patched(patch, body, apply) + if err != nil { + return true, nil, err + } + obj := held.DeepCopy() + manager := patch.PatchOptions.FieldManager + if manager == "" { + manager = "unnamed" + } + writer := owner{manager: manager, operation: metav1.ManagedFieldsOperationUpdate} + if apply { + writer.operation = metav1.ManagedFieldsOperationApply + } + byField := owners(obj) + wasSuspended, _, _ := unstructured.NestedBool(obj.Object, "spec", "suspend") + for _, field := range keptFields() { + writeField(obj, byField, field, body, writer) + } + if isSuspended, _, _ := unstructured.NestedBool(obj.Object, "spec", "suspend"); isSuspended != wasSuspended { + obj.SetGeneration(obj.GetGeneration() + 1) + } + setOwners(obj, byField) + if !equality.Semantic.DeepEqual(held.Object, obj.Object) { + obj.SetResourceVersion(s.nextVersion()) + } + if err := s.Client.Tracker().Update(KustomizationGVR(), obj, obj.GetNamespace()); err != nil { + return true, nil, err + } + return true, obj, nil +} + +// patched reads the Kustomization a patch writes, and checks an apply's preconditions against it. +func (s *Sim) patched( + patch k8stesting.PatchActionImpl, body map[string]any, apply bool, +) (*unstructured.Unstructured, error) { + current, err := s.Client.Tracker().Get(KustomizationGVR(), patch.GetNamespace(), patch.GetName()) + if err != nil { + if apply && apierrors.IsNotFound(err) { + if err := applyPreconditions(nil, body, patch.GetName()); err != nil { + return nil, err + } + } + return nil, err + } + held, ok := current.(*unstructured.Unstructured) + if !ok { + return nil, fmt.Errorf("a Kustomization the tracker holds as %T", current) + } + if apply { + if err := applyPreconditions(held, body, patch.GetName()); err != nil { + return nil, err + } + } + return held, nil +} + +// writeField applies one kept field of a patch body by writer to obj and its owners, byField. +func writeField(obj *unstructured.Unstructured, byField map[string][]owner, field keptField, body map[string]any, + writer owner) { + value, present, _ := unstructured.NestedFieldNoCopy(body, field.path...) + old, held, _ := unstructured.NestedFieldNoCopy(obj.Object, field.path...) + unchanged := held && fmt.Sprint(old) == fmt.Sprint(value) + switch { + case present && unchanged && writer.operation == metav1.ManagedFieldsOperationUpdate: + // An update owns only what it changed. + case present && unchanged: + // An apply of the value the field holds shares it with its owners, forced or not. + byField[field.name] = withOwner(byField[field.name], writer) + case present: + setField(obj, field.path, value) + byField[field.name] = []owner{writer} + case writer.operation == metav1.ManagedFieldsOperationApply && slices.Contains(byField[field.name], writer): + // An apply that leaves out a field its manager owned gives it up, and removes it when nobody + // else owns it. + byField[field.name] = slices.DeleteFunc(byField[field.name], func(o owner) bool { return o == writer }) + if len(byField[field.name]) == 0 { + setField(obj, field.path, nil) + delete(byField, field.name) + } + } +} + +// keptField is a field the fake keeps managedFields for. +type keptField struct { + name string + path []string +} + +// keptFields are spec.suspend and the requestedAt annotation. +func keptFields() []keptField { + return []keptField{ + {name: simFieldSuspend, path: []string{"spec", "suspend"}}, + {name: simFieldRequestedAt, path: []string{"metadata", "annotations", fluxmeta.ReconcileRequestAnnotation}}, + } +} + +// managedPath is a kept field's path in a managedFields entry's fieldsV1. +func managedPath(field keptField) []string { + path := make([]string, 0, len(field.path)) + for _, step := range field.path { + path = append(path, "f:"+step) + } + return path +} + +// setField sets the field at path, or removes it when value is nil. +func setField(obj *unstructured.Unstructured, path []string, value any) { + if value == nil { + unstructured.RemoveNestedField(obj.Object, path...) + return + } + if path[0] == "metadata" { + value = fmt.Sprint(value) + } + _ = unstructured.SetNestedField(obj.Object, value, path...) +} + +// owner is a managedFields entry's identity. +type owner struct { + manager string + operation metav1.ManagedFieldsOperationType +} + +// owners reads which entries own each of the fields the fake keeps. A field applied with the value +// it holds is shared: the API server keeps every owner, as it does for an apply (forced or not) of a +// value an update set. +func owners(obj *unstructured.Unstructured) map[string][]owner { + owners := map[string][]owner{} + for _, entry := range obj.GetManagedFields() { + if entry.FieldsV1 == nil { + continue + } + var fields map[string]any + if json.Unmarshal(entry.FieldsV1.GetRawBytes(), &fields) != nil { + continue + } + owner := owner{manager: entry.Manager, operation: entry.Operation} + if _, ok, _ := unstructured.NestedFieldNoCopy(fields, "f:spec", "f:suspend"); ok { + owners[simFieldSuspend] = withOwner(owners[simFieldSuspend], owner) + } + if _, ok, _ := unstructured.NestedFieldNoCopy(fields, "f:metadata", "f:annotations", + "f:"+fluxmeta.ReconcileRequestAnnotation); ok { + owners[simFieldRequestedAt] = withOwner(owners[simFieldRequestedAt], owner) + } + } + return owners +} + +// withOwner adds owner to a field's owners, once. +func withOwner(owners []owner, owner owner) []owner { + if slices.Contains(owners, owner) { + return owners + } + return append(owners, owner) +} + +// setOwners writes one managedFields entry per owner. +func setOwners(obj *unstructured.Unstructured, byField map[string][]owner) { + byOwner := map[owner]map[string]any{} + var order []owner + for _, field := range keptFields() { + for _, owner := range byField[field.name] { + fields := byOwner[owner] + if fields == nil { + fields = map[string]any{} + byOwner[owner] = fields + order = append(order, owner) + } + _ = unstructured.SetNestedField(fields, map[string]any{}, managedPath(field)...) + } + } + entries := make([]metav1.ManagedFieldsEntry, 0, len(byOwner)) + for _, owner := range order { + fields := byOwner[owner] + raw, _ := json.Marshal(fields) + entries = append(entries, metav1.ManagedFieldsEntry{ + Manager: owner.manager, Operation: owner.operation, APIVersion: "kustomize.toolkit.fluxcd.io/v1", + FieldsType: "FieldsV1", FieldsV1: metav1.NewFieldsV1(string(raw)), + }) + } + obj.SetManagedFields(entries) +} + +// OwnSuspend sets a Kustomization's spec.suspend, owned by manager with operation, as a manifest +// applied by something else, or a `flux suspend`, leaves it. +func OwnSuspend(t testing.TB, obj *unstructured.Unstructured, suspend bool, manager string, + operation metav1.ManagedFieldsOperationType) *unstructured.Unstructured { + t.Helper() + require.NoError(t, unstructured.SetNestedField(obj.Object, suspend, "spec", "suspend")) + owners := owners(obj) + owners[simFieldSuspend] = []owner{{manager: manager, operation: operation}} + setOwners(obj, owners) + return obj +} + +// Kustomization reads a Kustomization from the fake. +func (s *Sim) Kustomization(key types.NamespacedName) *unstructured.Unstructured { + s.t.Helper() + obj, err := s.Client.Resource(KustomizationGVR()).Namespace(key.Namespace). + Get(context.Background(), key.Name, metav1.GetOptions{}) + require.NoError(s.t, err) + return obj +} + +// TickingClock starts at start and moves on a millisecond on every read, so every token taken from +// it is new, as `flux reconcile`'s are. +func TickingClock(start time.Time) func() time.Time { + var mu sync.Mutex + now := start + return func() time.Time { + mu.Lock() + defer mu.Unlock() + read := now + now = now.Add(time.Millisecond) + return read + } +} + +// KustomizationGVR is Flux's Kustomization. +func KustomizationGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "kustomize.toolkit.fluxcd.io", Version: "v1", Resource: "kustomizations"} +} + +// GitRepositoryGVR is Flux's GitRepository. +func GitRepositoryGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "gitrepositories"} +} + +const ( + kindKustomization = "Kustomization" + kindGitRepository = "GitRepository" +) + +// gitRepositoryRef is a Kustomization's GitRepository, its namespace defaulted to its own. +func gitRepositoryRef(ks *unstructured.Unstructured) (types.NamespacedName, error) { + kind, _, _ := unstructured.NestedString(ks.Object, "spec", "sourceRef", "kind") + name, _, _ := unstructured.NestedString(ks.Object, "spec", "sourceRef", "name") + namespace, _, _ := unstructured.NestedString(ks.Object, "spec", "sourceRef", "namespace") + if kind != kindGitRepository || name == "" { + return types.NamespacedName{}, fmt.Errorf("%s/%s applies no GitRepository", ks.GetNamespace(), ks.GetName()) + } + if namespace == "" { + namespace = ks.GetNamespace() + } + return types.NamespacedName{Namespace: namespace, Name: name}, nil +} + +// statusString reads a string under an object's status. +func statusString(obj *unstructured.Unstructured, fields ...string) string { + value, _, _ := unstructured.NestedString(obj.Object, append([]string{"status"}, fields...)...) + return value +} diff --git a/internal/applier/fluxsim/fluxsim_test.go b/internal/applier/fluxsim/fluxsim_test.go new file mode 100644 index 000000000..d7ed8be58 --- /dev/null +++ b/internal/applier/fluxsim/fluxsim_test.go @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: Apache-2.0 + +package fluxsim + +import ( + "context" + "testing" + "time" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +// The fake is what every test of the suspended lifecycle trusts, so it is held to the source facts +// it claims, one test each. + +var ( + ksKey = types.NamespacedName{Namespace: "flux-system", Name: "apps"} + sourceKey = types.NamespacedName{Namespace: "flux-system", Name: "repo"} +) + +const ( + oldRevision = "main@sha1:0000000000000000000000000000000000000001" + newRevision = "main@sha1:4f2c1ab9d0e1f2a3b4c5d6e7f8091a2b3c4d5e6f" + manager = "gitops-reverser" +) + +func newSim(t *testing.T, suspended bool) *Sim { + t.Helper() + ks := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kustomize.toolkit.fluxcd.io/v1", "kind": kindKustomization, + "metadata": map[string]any{"namespace": ksKey.Namespace, "name": ksKey.Name}, + "spec": map[string]any{"sourceRef": map[string]any{"kind": kindGitRepository, "name": sourceKey.Name}}, + }} + ks.SetGeneration(1) + if suspended { + ks = OwnSuspend(t, ks, true, manager, metav1.ManagedFieldsOperationApply) + } + SetReady(ks, metav1.ConditionTrue, fluxmeta.ReconciliationSucceededReason, "applied") + source := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "source.toolkit.fluxcd.io/v1", "kind": kindGitRepository, + "metadata": map[string]any{"namespace": sourceKey.Namespace, "name": sourceKey.Name}, + "status": map[string]any{"artifact": map[string]any{"revision": oldRevision}}, + }} + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + KustomizationGVR(): "KustomizationList", GitRepositoryGVR(): "GitRepositoryList", + }, ks, source) + sim := New(t, client) + sim.Fetches(sourceKey, newRevision) + return sim +} + +func (s *Sim) apply(t *testing.T, suspend *bool, token string) { + t.Helper() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("kustomize.toolkit.fluxcd.io/v1") + obj.SetKind(kindKustomization) + obj.SetNamespace(ksKey.Namespace) + obj.SetName(ksKey.Name) + if token != "" { + obj.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: token}) + } + if suspend != nil { + require.NoError(t, unstructured.SetNestedField(obj.Object, *suspend, "spec", "suspend")) + } + _, err := s.Client.Resource(KustomizationGVR()).Namespace(ksKey.Namespace). + Apply(context.Background(), ksKey.Name, obj, metav1.ApplyOptions{FieldManager: manager, Force: true}) + require.NoError(t, err) +} + +func (s *Sim) stampSource(t *testing.T, token string) { + t.Helper() + patch := []byte(`{"metadata":{"annotations":{"reconcile.fluxcd.io/requestedAt":"` + token + `"}}}`) + _, err := s.Client.Resource(GitRepositoryGVR()).Namespace(sourceKey.Namespace). + Patch(context.Background(), sourceKey.Name, types.MergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, err) +} + +func handled(s *Sim, token string) func() bool { + return func() bool { + return statusString(s.Kustomization(ksKey), "lastHandledReconcileAt") == token + } +} + +func eventually(t *testing.T, cond func() bool, why string) { + t.Helper() + require.Eventually(t, cond, 5*time.Second, time.Millisecond, why) +} + +func ptr(b bool) *bool { return &b } + +// A suspended object returns before it resolves its source, and finalizeStatus still marks the +// token it read handled (:189-192, :143). Its Ready is left as it was. +func TestSim_ASuspendedReconcileHandlesItsTokenAndAppliesNothing(t *testing.T) { + sim := newSim(t, true) + sim.stampSource(t, "S1") + eventually(t, func() bool { + return statusString(sim.Kustomization(ksKey), "lastHandledReconcileAt") == "" && + sim.Reconciles(ksKey) > 0 + }, "a new artifact enqueues the consumer") + sim.apply(t, ptr(true), "T1") + eventually(t, handled(sim, "T1"), "the token is handled") + assert.Empty(t, sim.Applied(), "nothing applied") + assert.Equal(t, string(metav1.ConditionTrue), readyStatus(sim.Kustomization(ksKey))) +} + +// One write that sets suspend: false and a token enqueues a reconcile that reads both, and +// applies whatever the artifact holds then; the generation it read is observed on success. +func TestSim_AResumeAppliesTheArtifact(t *testing.T) { + sim := newSim(t, true) + sim.apply(t, ptr(false), "T1") + eventually(t, handled(sim, "T1"), "the token is handled") + assert.Equal(t, []string{"flux-system/apps " + oldRevision}, sim.Applied()) + ks := sim.Kustomization(ksKey) + assert.Equal(t, oldRevision, statusString(ks, "lastAppliedRevision")) + observed, _, _ := unstructured.NestedInt64(ks.Object, "status", "observedGeneration") + assert.Equal(t, ks.GetGeneration(), observed) +} + +// spec.suspend is read when a reconcile starts: a suspend written while an apply runs does not +// stop it, and the next reconcile, which reads the suspend, handles its token after it. +func TestSim_ASuspendDoesNotStopARunningApply(t *testing.T) { + sim := newSim(t, true) + held := sim.HoldNextApply() + sim.apply(t, ptr(false), "T1") + <-held + assert.Equal(t, oldRevision, statusString(sim.Kustomization(ksKey), "lastAttemptedRevision"), + "attempted before it builds") + sim.apply(t, ptr(true), "T2") + sim.Release() + eventually(t, handled(sim, "T2"), "the suspend's token is handled after the apply") + assert.Len(t, sim.Applied(), 1, "the running apply finished") +} + +// A failed apply and a dependency that is not ready are Ready=False, each with its reason, and +// mark the token handled all the same; a requeue into a suspended object applies nothing. +func TestSim_FailuresAndRequeues(t *testing.T) { + sim := newSim(t, true) + sim.FailApplies("dry-run failed") + sim.apply(t, ptr(false), "T1") + eventually(t, handled(sim, "T1"), "a failure handles the token") + assert.Equal(t, string(metav1.ConditionFalse), readyStatus(sim.Kustomization(ksKey))) + + sim.FailApplies("") + sim.BlockDependencies(true) + sim.apply(t, ptr(false), "T2") + eventually(t, handled(sim, "T2"), "so does a dependency not ready") + conditions, _, _ := unstructured.NestedSlice(sim.Kustomization(ksKey).Object, "status", "conditions") + require.Len(t, conditions, 1) + reason, _ := conditions[0].(map[string]any)["reason"].(string) + assert.Equal(t, fluxmeta.DependencyNotReadyReason, reason) + + sim.BlockDependencies(false) + sim.apply(t, ptr(true), "T3") + eventually(t, handled(sim, "T3"), "suspended") + before := sim.Reconciles(ksKey) + sim.Requeue(ksKey) + eventually(t, func() bool { return sim.Reconciles(ksKey) > before }, "the requeue runs") + assert.Empty(t, sim.Applied()) +} + +// The ownership a real API server keeps (measured in envtest, +// internal/controller/fluxtrigger_suspend_apiserver_test.go): a merge patch owns what it changes, +// as an update; an apply of the value a field holds shares it, even forced; a write that changes +// the value takes it from every other owner; an apply that leaves a field out gives up its share, +// and removes the field only when nobody else owns it. +func TestSim_ManagedFields(t *testing.T) { + sim := newSim(t, false) + update := owner{manager: "flux", operation: metav1.ManagedFieldsOperationUpdate} + apply := owner{manager: manager, operation: metav1.ManagedFieldsOperationApply} + mergePatch := func(suspend string) { + t.Helper() + _, err := sim.Client.Resource(KustomizationGVR()).Namespace(ksKey.Namespace).Patch(context.Background(), + ksKey.Name, types.MergePatchType, []byte(`{"spec":{"suspend":`+suspend+`}}`), + metav1.PatchOptions{FieldManager: "flux"}) + require.NoError(t, err) + } + suspendOwners := func() []owner { return owners(sim.Kustomization(ksKey))[simFieldSuspend] } + + mergePatch("true") + assert.Equal(t, []owner{update}, suspendOwners()) + + sim.apply(t, ptr(true), "T1") + assert.ElementsMatch(t, []owner{update, apply}, suspendOwners(), "the same value is shared, forced or not") + + sim.apply(t, nil, "") + ks := sim.Kustomization(ksKey) + assert.True(t, suspended(ks), "the other owner keeps it") + assert.Equal(t, []owner{update}, suspendOwners()) + assert.Empty(t, ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]) + + sim.apply(t, ptr(false), "T2") + assert.Equal(t, []owner{apply}, suspendOwners(), "another value is taken from every owner") + mergePatch("false") + assert.Equal(t, []owner{apply}, suspendOwners(), "an update of the value the field holds takes nothing") + + sim.apply(t, nil, "") + ks = sim.Kustomization(ksKey) + _, found, _ := unstructured.NestedFieldNoCopy(ks.Object, "spec", "suspend") + assert.False(t, found, "released") + assert.Empty(t, owners(ks)) +} + +// A status write and a patch are each one step, as the API server's resourceVersion check makes +// them: a resume that arrives while a reconcile is writing its status lands after that write, and is +// not written away by it. CI once lost a resume this way, and the operation never ended. +func TestSim_AStatusWriteNeverWritesAResumeAway(t *testing.T) { + sim := newSim(t, true) + defer sim.PauseKustomizations()() + + read, release := make(chan struct{}), make(chan struct{}) + written := make(chan struct{}) + go func() { + defer close(written) + sim.WriteStatus(context.Background(), KustomizationGVR(), ksKey, func(ks *unstructured.Unstructured) { + close(read) + <-release + _ = unstructured.SetNestedField(ks.Object, "T0", "status", "lastHandledReconcileAt") + }) + }() + <-read + + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("kustomize.toolkit.fluxcd.io/v1") + obj.SetKind(kindKustomization) + obj.SetNamespace(ksKey.Namespace) + obj.SetName(ksKey.Name) + obj.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: "T1"}) + require.NoError(t, unstructured.SetNestedField(obj.Object, false, "spec", "suspend")) + resumed := make(chan error, 1) + go func() { + _, err := sim.Client.Resource(KustomizationGVR()).Namespace(ksKey.Namespace).Apply(context.Background(), + ksKey.Name, obj, metav1.ApplyOptions{FieldManager: manager, Force: true}) + resumed <- err + }() + select { + case err := <-resumed: + t.Fatalf("the resume landed inside the status write (err %v)", err) + case <-time.After(50 * time.Millisecond): + } + close(release) + <-written + require.NoError(t, <-resumed) + + ks := sim.Kustomization(ksKey) + assert.False(t, suspended(ks), "the resume stands") + assert.Equal(t, "T1", ks.GetAnnotations()[fluxmeta.ReconcileRequestAnnotation]) + handled, _, _ := unstructured.NestedString(ks.Object, "status", "lastHandledReconcileAt") + assert.Equal(t, "T0", handled, "and so does the status write") +} + +// A write that changes the object moves its resourceVersion, and one that changes nothing does +// not. A server-side apply that carries a resourceVersion or a uid has them checked as the API +// server checks them (measured in envtest): a resourceVersion that moved is a Conflict, another +// object's uid is Invalid, and a uid with no object is a Conflict rather than a create. +func TestSim_ApplyPreconditions(t *testing.T) { + sim := newSim(t, true) + sim.Edit(ksKey, func(ks *unstructured.Unstructured) { ks.SetUID("apps-1") }) + read := sim.Kustomization(ksKey) + require.NotEmpty(t, read.GetResourceVersion()) + sim.Edit(ksKey, func(*unstructured.Unstructured) {}) + assert.Equal(t, read.GetResourceVersion(), sim.Kustomization(ksKey).GetResourceVersion(), + "a write that changes nothing") + + resources := sim.Client.Resource(KustomizationGVR()).Namespace(ksKey.Namespace) + applyOver := func(resourceVersion, uid string) error { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("kustomize.toolkit.fluxcd.io/v1") + obj.SetKind(kindKustomization) + obj.SetNamespace(ksKey.Namespace) + obj.SetName(ksKey.Name) + obj.SetResourceVersion(resourceVersion) + obj.SetUID(types.UID(uid)) + require.NoError(t, unstructured.SetNestedField(obj.Object, false, "spec", "suspend")) + _, err := resources.Apply(context.Background(), ksKey.Name, obj, + metav1.ApplyOptions{FieldManager: manager, Force: true}) + return err + } + + sim.Edit(ksKey, func(ks *unstructured.Unstructured) { ks.SetLabels(map[string]string{"team": "a"}) }) + require.True(t, apierrors.IsConflict(applyOver(read.GetResourceVersion(), "apps-1"))) + assert.True(t, suspended(sim.Kustomization(ksKey)), "not applied") + read = sim.Kustomization(ksKey) + require.True(t, apierrors.IsInvalid(applyOver(read.GetResourceVersion(), "apps-2"))) + require.NoError(t, applyOver(read.GetResourceVersion(), "apps-1")) + assert.False(t, suspended(sim.Kustomization(ksKey))) + assert.NotEqual(t, read.GetResourceVersion(), sim.Kustomization(ksKey).GetResourceVersion()) + + require.NoError(t, resources.Delete(context.Background(), ksKey.Name, metav1.DeleteOptions{})) + require.True(t, apierrors.IsConflict(applyOver("", "apps-1")), "no create") +} + +func suspended(ks *unstructured.Unstructured) bool { + value, _, _ := unstructured.NestedBool(ks.Object, "spec", "suspend") + return value +} + +func TestTickingClock_NeverRepeats(t *testing.T) { + start := time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC) + clock := TickingClock(start) + assert.Equal(t, start, clock()) + assert.Equal(t, start.Add(time.Millisecond), clock()) +} diff --git a/internal/applier/holder.go b/internal/applier/holder.go new file mode 100644 index 000000000..ced395597 --- /dev/null +++ b/internal/applier/holder.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "fmt" + "slices" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// One applier, one trigger: two triggers holding the same Kustomization would both suspend it under +// the same field manager, each would take the other's resume for an external one and suspend it +// mid-apply, and deleting either would hand back the hold the other still needs. Two triggers on one +// Application would both start syncs. So exactly one trigger holds an applier, by a rule every party +// computes alike (precedes, over the triggers in line): the trigger reconciler, which holds and +// hands back only for the holder, and the dispatcher, which asks the applier only for it. Both +// confirm it uncached before they write to a Kustomization. +// +// Only a trigger in line can hold: one an eligible GitTarget references (references.go), or a +// FluxTrigger that carries its finalizer, which it does from before its first adoption write until +// it has handed the Kustomization back or over. So an older trigger nothing eligible references +// keeps no younger one out, and one that is retiring stays the holder until its release is done. + +// ReasonHeldByAnotherTrigger is Configured=False on a trigger whose applier another trigger of its +// kind holds: one in line that names the same object in the same ClusterProvider's cluster, and has +// an apply in flight, holds the Kustomization already, or was created first. The trigger takes the +// applier over once it no longer comes first. +const ReasonHeldByAnotherTrigger = "HeldByAnotherTrigger" + +// sameApplier reports whether two triggers name one applier: the same kind of object, in the same +// ClusterProvider's cluster, with the same namespace and name. +func sameApplier(a, b v1alpha3.ApplyTrigger) bool { + return a.TriggerKind() == b.TriggerKind() && a.ClusterProvider() == b.ClusterProvider() && + a.Applier() == b.Applier() +} + +// sameTrigger reports whether a and b are the same trigger, by kind, namespace and name. +func sameTrigger(a, b v1alpha3.ApplyTrigger) bool { + return a.TriggerKind() == b.TriggerKind() && a.GetNamespace() == b.GetNamespace() && a.GetName() == b.GetName() +} + +// precedes reports whether a comes before b in line for an applier. A FluxTrigger with an operation +// in flight comes first: its worker follows that operation until the Kustomization is suspended +// again, and a trigger that took the Kustomization over meanwhile would take that resume for an +// external one and suspend it mid-apply. So a Kustomization changes hands only between operations, +// whatever triggers appear; a trigger created in the same second as the holder, which can come +// before it by name, waits for the holder's apply to end. An Application has no such hold: a sync +// somebody else started is never replaced, so an ArgoCDTrigger's operation does not count. Then a +// FluxTrigger that holds the Kustomization already (it carries the finalizer) comes first: a trigger +// that becomes eligible later takes it over only once the holder lets go, so a Kustomization is +// never taken from a holder still suspending or releasing it. Then the trigger created first comes +// first, and then by namespace and name. +func precedes(a, b v1alpha3.ApplyTrigger) bool { + if applying, other := holdsAnApply(a), holdsAnApply(b); applying != other { + return applying + } + if holding, other := Holding(a), Holding(b); holding != other { + return holding + } + return createdBefore(a, b) +} + +// Holding reports whether trigger is a FluxTrigger that carries its finalizer: it holds its +// Kustomization, or hands it back or over. +func Holding(trigger v1alpha3.ApplyTrigger) bool { + return trigger.TriggerKind() == v1alpha3.ApplyTriggerKindFlux && + slices.Contains(trigger.GetFinalizers(), v1alpha3.FluxTriggerFinalizer) +} + +// holdsAnApply reports whether trigger is a FluxTrigger with an operation in flight. +func holdsAnApply(trigger v1alpha3.ApplyTrigger) bool { + operation := trigger.Operation() + return trigger.TriggerKind() == v1alpha3.ApplyTriggerKindFlux && operation != nil && !operation.Phase.Completed() +} + +// createdBefore orders triggers by creation, then by namespace, then by name. +func createdBefore(a, b v1alpha3.ApplyTrigger) bool { + at, bt := a.GetCreationTimestamp(), b.GetCreationTimestamp() + switch { + case !at.Equal(&bt): + return at.Before(&bt) + case a.GetNamespace() != b.GetNamespace(): + return a.GetNamespace() < b.GetNamespace() + default: + return a.GetName() < b.GetName() + } +} + +// firstInLine is the trigger that holds trigger's applier: of the triggers of its kind in line for +// the same applier, the one first (precedes). trigger itself counts as in line. A FluxTrigger being +// deleted still holds it, until it is gone: its finalizer is handing the applier back. It is trigger +// itself when no other comes before it. +func firstInLine( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, +) (v1alpha3.ApplyTrigger, error) { + others, err := TriggersOfApplier(ctx, reader, trigger) + if err != nil { + return nil, err + } + source := publishedSource(trigger) + first := trigger + for _, other := range others { + if !precedes(other, first) { + continue + } + inLine, err := inLine(ctx, reader, other, source) + if err != nil { + return nil, err + } + if inLine { + first = other + } + } + return first, nil +} + +// inLine reports whether other is in line for the applier whose source the caller read: it holds +// it (Holding), or an eligible GitTarget references it. +func inLine(ctx context.Context, reader client.Reader, other v1alpha3.ApplyTrigger, source *Source) (bool, error) { + if Holding(other) { + return true, nil + } + return eligible(ctx, reader, other, source) +} + +// publishedSource is the source trigger last read, which every trigger naming its applier shares, +// or nil when it has read none. +func publishedSource(trigger v1alpha3.ApplyTrigger) *Source { + url, branch, known := trigger.SourceBinding() + if !known { + return nil + } + return &Source{URL: url, Revision: branch} +} + +// HeldByAnother is the trigger that holds trigger's applier when it is another, or nil when trigger +// holds it itself. +func HeldByAnother( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, +) (v1alpha3.ApplyTrigger, error) { + first, err := firstInLine(ctx, reader, trigger) + if err != nil || sameTrigger(first, trigger) { + return nil, err + } + return first, nil +} + +// TriggersOfApplier lists the triggers other than trigger that name its applier. +func TriggersOfApplier( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, +) ([]v1alpha3.ApplyTrigger, error) { + triggers, err := ListApplyTriggers(ctx, reader, trigger.TriggerKind()) + if err != nil { + return nil, err + } + var others []v1alpha3.ApplyTrigger + for _, other := range triggers { + if !sameTrigger(other, trigger) && sameApplier(other, trigger) { + others = append(others, other) + } + } + return others, nil +} + +// ListApplyTriggers lists the triggers of kind, with opts (everywhere without a namespace option). +func ListApplyTriggers( + ctx context.Context, reader client.Reader, kind string, opts ...client.ListOption, +) ([]v1alpha3.ApplyTrigger, error) { + var triggers []v1alpha3.ApplyTrigger + switch kind { + case v1alpha3.ApplyTriggerKindFlux: + var list v1alpha3.FluxTriggerList + if err := reader.List(ctx, &list, opts...); err != nil { + return nil, fmt.Errorf("list %ss: %w", kind, err) + } + for i := range list.Items { + triggers = append(triggers, &list.Items[i]) + } + case v1alpha3.ApplyTriggerKindArgoCD: + var list v1alpha3.ArgoCDTriggerList + if err := reader.List(ctx, &list, opts...); err != nil { + return nil, fmt.Errorf("list %ss: %w", kind, err) + } + for i := range list.Items { + triggers = append(triggers, &list.Items[i]) + } + default: + return nil, fmt.Errorf("no apply trigger kind %q", kind) + } + return triggers, nil +} + +// HeldByAnotherMessage is the Configured message of a trigger whose applier holder holds. +func HeldByAnotherMessage(trigger, holder v1alpha3.ApplyTrigger) string { + held := fmt.Sprintf("%s %s in the cluster of ClusterProvider %q is held by %s %s/%s", trigger.ApplierKind(), + trigger.Applier(), trigger.ClusterProvider(), holder.TriggerKind(), holder.GetNamespace(), holder.GetName()) + switch { + case holdsAnApply(holder) && !holdsAnApply(trigger): + return held + ", whose apply is in flight; this trigger takes it over once that apply ends" + case Holding(holder) && !Holding(trigger): + return held + ", which holds it now; this trigger takes it over once that one lets go of it" + } + return held + ", which was created first; this trigger takes it over once that one is deleted or no " + + "eligible GitTarget references it" +} diff --git a/internal/applier/holder_test.go b/internal/applier/holder_test.go new file mode 100644 index 000000000..730dbeddd --- /dev/null +++ b/internal/applier/holder_test.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "testing" + "time" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// rival is a trigger of kind in namespace team-b naming the same applier as the one the dispatcher +// fixture made for the GitTarget, in the same cluster, created an hour before it (older) or after. +func rival(t *testing.T, f *dispatcherFixture, kind, name string, older bool) { + t.Helper() + own := v1alpha3.NewApplyTrigger(kind) + require.NoError(t, f.control.Get(context.Background(), client.ObjectKey{Namespace: targetNS, Name: name}, own)) + own.SetCreationTimestamp(metav1.NewTime(time.Unix(1_000_000, 0))) + require.NoError(t, f.control.Update(context.Background(), own.(client.Object))) + created := own.GetCreationTimestamp().Add(time.Hour) + if older { + created = own.GetCreationTimestamp().Add(-time.Hour) + } + other := own.DeepCopyObject().(v1alpha3.ApplyTrigger) + other.SetNamespace("team-b") + other.SetResourceVersion("") + other.SetCreationTimestamp(metav1.NewTime(created)) + // The fixture's own trigger names its applier in its own namespace's terms; spell it out. + applier := own.Applier() + ref := fluxmeta.NamespacedObjectReference{Namespace: applier.Namespace, Name: applier.Name} + switch typed := other.(type) { + case *v1alpha3.FluxTrigger: + typed.Spec.KustomizationRef = ref + case *v1alpha3.ArgoCDTrigger: + typed.Spec.ApplicationRef = ref + } + require.NoError(t, f.control.Create(context.Background(), other.(client.Object))) + referenceFrom(t, f.control, other) +} + +// referenceFrom puts a GitTarget in trigger's namespace that names it, and admits that namespace +// on the ClusterProviders, so the trigger is in line for its applier. +func referenceFrom(t *testing.T, c client.Client, trigger v1alpha3.ApplyTrigger) { + t.Helper() + ctx := context.Background() + target := gitTarget("names-"+trigger.GetName(), nil, "") + target.Namespace = trigger.GetNamespace() + target.Spec.ApplyTriggerRef = &v1alpha3.ApplyTriggerReference{Kind: trigger.TriggerKind(), Name: trigger.GetName()} + require.NoError(t, c.Create(ctx, target)) + for _, name := range []string{v1alpha3.DefaultClusterProviderName, "prod-eu-1", trigger.ClusterProvider()} { + provider := &v1alpha3.ClusterProvider{} + if err := c.Get(ctx, client.ObjectKey{Name: name}, provider); err != nil { + require.NoError(t, c.Create(ctx, clusterProvider(name, targetNS, trigger.GetNamespace()))) + continue + } + provider.Spec.AccessFrom.Names = append(provider.Spec.AccessFrom.Names, trigger.GetNamespace()) + require.NoError(t, c.Update(ctx, provider)) + } +} + +// TestDispatcher_OnlyTheHolderAsksItsApplier is review 5's finding 5, the dispatcher's half: two +// triggers naming one applier both asked it. Now only the one first in line (created first) does; +// a GitTarget naming the other asks nothing. +func TestDispatcher_OnlyTheHolderAsksItsApplier(t *testing.T) { + for name, tc := range map[string]struct { + target func() *v1alpha3.GitTarget + kind string + trigger string + applier git.ApplierRef + }{ + "FluxTrigger": { + target: func() *v1alpha3.GitTarget { return gitTarget("a", checkoutRef(), "") }, + kind: v1alpha3.ApplyTriggerKindFlux, + trigger: "checkout", applier: checkout, + }, + "ArgoCDTrigger": { + target: argoTarget, kind: v1alpha3.ApplyTriggerKindArgoCD, + trigger: "platform", applier: argoPlatform, + }, + } { + t.Run(name, func(t *testing.T) { + t.Run("another trigger was first", func(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{tc.target()}, append(behindSource(), manualApp())...) + rival(t, f, tc.kind, tc.trigger, true) + + res := f.attempt(request(tc.applier, v1alpha3.ApplyReasonForeignCommit, "a")) + + assert.Equal(t, git.ApplyDropped, res.Outcome) + assert.Empty(t, f.patches(), "nothing is resumed") + assert.Empty(t, f.updates(), "no sync is set") + }) + t.Run("this trigger was first", func(t *testing.T) { + f := newDispatcherFixture(t, []client.Object{tc.target()}, append(behindSource(), manualApp())...) + rival(t, f, tc.kind, tc.trigger, false) + + res := f.attempt(request(tc.applier, v1alpha3.ApplyReasonForeignCommit, "a")) + + assert.Equal(t, git.ApplyRequested, res.Outcome, res.Err) + }) + }) + } +} + +// TestHolder_FirstInLine: the holder is the trigger in line created first, then by namespace and +// name; one being deleted still holds until it is gone, one no eligible GitTarget references is not +// in line, and neither is a trigger naming another applier, or the same one in another cluster. +func TestHolder_FirstInLine(t *testing.T) { + at := func(minutes int) metav1.Time { + return metav1.NewTime(time.Unix(0, 0).Add(time.Duration(minutes) * time.Minute)) + } + trigger := func(namespace, name, cluster, ks string, created metav1.Time) *v1alpha3.FluxTrigger { + return &v1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name, CreationTimestamp: created}, + Spec: v1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &fluxmeta.LocalObjectReference{Name: cluster}, + KustomizationRef: fluxmeta.NamespacedObjectReference{Namespace: "flux-system", Name: ks}, + }, + } + } + self := trigger("team-c", "mine", "default", "checkout", at(10)) + deleting := trigger("team-z", "old", "default", "checkout", at(5)) + deleting.Finalizers = []string{v1alpha3.FluxTriggerFinalizer} + deleting.DeletionTimestamp = &metav1.Time{Time: time.Unix(100, 0)} + objects := []client.Object{ + self, + trigger("team-a", "elsewhere", "hub", "checkout", at(1)), + trigger("team-a", "other", "default", "infra", at(1)), + trigger("team-b", "same-second", "default", "checkout", at(10)), + } + unreferenced := trigger("team-a", "stray", "default", "checkout", at(1)) + f := newDispatcherFixture(t, nil) + for _, obj := range append(objects, unreferenced) { + require.NoError(t, f.control.Create(context.Background(), obj)) + } + + holder, err := firstInLine(context.Background(), f.control, self) + require.NoError(t, err) + assert.Equal(t, "team-c/mine", holder.GetNamespace()+"/"+holder.GetName(), + "older triggers nothing eligible references keep no younger one out") + + referenceFrom(t, f.control, objects[3].(*v1alpha3.FluxTrigger)) + holder, err = firstInLine(context.Background(), f.control, self) + require.NoError(t, err) + assert.Equal(t, "team-b/same-second", holder.GetNamespace()+"/"+holder.GetName(), + "created in the same second: the namespace decides") + + require.NoError(t, f.control.Create(context.Background(), deleting)) + holder, err = HeldByAnother(context.Background(), f.control, self) + require.NoError(t, err) + require.NotNil(t, holder) + assert.Equal(t, "team-z/old", holder.GetNamespace()+"/"+holder.GetName(), "a trigger being deleted still holds") + + others, err := TriggersOfApplier(context.Background(), f.control, self) + require.NoError(t, err) + assert.Len(t, others, 3, "the same Kustomization in another cluster, or another one, is not in line") +} + +// TestHolder_AnApplyInFlightComesFirst is review 5's P2: a FluxTrigger with an operation in flight +// comes first in line, so a trigger that comes before it by creation and name waits for that apply +// to end. An ArgoCDTrigger's operation does not count: a sync somebody else started is never +// replaced, so the Application can change hands mid-sync. +func TestHolder_AnApplyInFlightComesFirst(t *testing.T) { + created := metav1.NewTime(time.Unix(1_000_000, 0)) + applying := &v1alpha3.ApplyOperation{Phase: v1alpha3.ApplyOperationRunning, Token: "T1"} + flux := func(namespace string, operation *v1alpha3.ApplyOperation) *v1alpha3.FluxTrigger { + return &v1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "checkout", CreationTimestamp: created}, + Status: v1alpha3.FluxTriggerStatus{Operation: operation}, + } + } + argo := func(namespace string, operation *v1alpha3.ApplyOperation) *v1alpha3.ArgoCDTrigger { + return &v1alpha3.ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "platform", CreationTimestamp: created}, + Status: v1alpha3.ArgoCDTriggerStatus{Operation: operation}, + } + } + + assert.True(t, precedes(flux("team-a", nil), flux("team-b", nil)), "by namespace, with nothing in flight") + assert.True(t, precedes(flux("team-b", applying), flux("team-a", nil)), "the apply in flight first") + assert.False(t, precedes(flux("team-a", nil), flux("team-b", applying))) + done := applying.DeepCopy() + done.Phase = v1alpha3.ApplyOperationSucceeded + assert.True(t, precedes(flux("team-a", nil), flux("team-b", done)), "a completed one does not count") + assert.True(t, precedes(argo("team-a", nil), argo("team-b", applying)), "nor an Argo CD sync") + assert.Contains(t, HeldByAnotherMessage(flux("team-a", nil), flux("team-b", applying)), + "is held by FluxTrigger team-b/checkout, whose apply is in flight; this trigger takes it over once that "+ + "apply ends") +} diff --git a/internal/applier/references.go b/internal/applier/references.go new file mode 100644 index 000000000..89bd4f0e6 --- /dev/null +++ b/internal/applier/references.go @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "fmt" + "sort" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/authz" + "github.com/ConfigButler/gitops-reverser/internal/repourl" +) + +// A trigger acts only while an eligible GitTarget references it: one that names it, is admitted by +// its ClusterProviders, and writes the repository and branch the applier fetches. A trigger nothing +// eligible names serves nobody, so it reads and writes nothing in the applier's cluster, and is not +// in line for its applier. Eligibility is judged from those inputs alone, never from the +// GitTarget's Ready, which reads the trigger's own Ready back (ApplyTriggerReady) and would make +// the two circular. + +// ReasonNotReferenced is Configured=Unknown (and Ready=True) on a trigger no GitTarget names: it is +// asked nothing, so it does nothing. +const ReasonNotReferenced = "NotReferenced" + +// ReasonNotBound is Configured on a trigger whose naming GitTargets are none of them eligible: +// Unknown while one cannot be judged yet, False once each is refused for different reasons. +const ReasonNotBound = "NotBound" + +// ReferenceState is how one GitTarget naming a trigger stands with it. +type ReferenceState int + +const ( + // ReferenceEligible names the trigger, is admitted, and writes what the applier fetches. + ReferenceEligible ReferenceState = iota + // ReferenceUndecided cannot be judged yet: its GitProvider or a ClusterProvider is missing, or + // the applier's source has not been read. + ReferenceUndecided + // ReferenceRefused is not admitted, or writes another repository or branch. + ReferenceRefused +) + +// Reference is one GitTarget naming a trigger, judged. +type Reference struct { + Target types.NamespacedName + State ReferenceState + // Reason is the refusal's condition reason (ReasonSourceMismatch, + // authz.ReasonNamespaceNotAuthorized), set only for a refused reference. + Reason string + // Message says why a reference is not eligible. + Message string +} + +// References are the GitTargets naming one trigger, judged. +type References []Reference + +// Named reports whether any GitTarget names the trigger. +func (refs References) Named() bool { return len(refs) > 0 } + +// Eligible reports whether at least one GitTarget naming the trigger is eligible. +func (refs References) Eligible() bool { + for _, ref := range refs { + if ref.State == ReferenceEligible { + return true + } + } + return false +} + +// Refused reports whether every GitTarget naming the trigger is conclusively refused: there is at +// least one, and none is eligible or may yet be. +func (refs References) Refused() bool { + for _, ref := range refs { + if ref.State != ReferenceRefused { + return false + } + } + return refs.Named() +} + +// Reason is the Configured reason of references that are not eligible: the refusal reason they +// share when every one is refused for the same one, and ReasonNotBound otherwise. +func (refs References) Reason() string { + reason := "" + for _, ref := range refs { + switch { + case ref.State != ReferenceRefused: + return ReasonNotBound + case reason == "": + reason = ref.Reason + case reason != ref.Reason: + return ReasonNotBound + } + } + if reason == "" { + return ReasonNotBound + } + return reason +} + +// Message lists each GitTarget that is not eligible, and why. +func (refs References) Message() string { + var parts []string + for _, ref := range refs { + if ref.State != ReferenceEligible { + parts = append(parts, fmt.Sprintf("GitTarget %s: %s", ref.Target, ref.Message)) + } + } + return "no GitTarget naming this trigger is eligible: " + strings.Join(parts, "; ") +} + +// NamingTargets lists the GitTargets that name trigger, in its namespace, and are not being +// deleted, by name. +func NamingTargets( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, +) ([]*v1alpha3.GitTarget, error) { + var list v1alpha3.GitTargetList + if err := reader.List(ctx, &list, client.InNamespace(trigger.GetNamespace())); err != nil { + return nil, fmt.Errorf("list GitTargets: %w", err) + } + var targets []*v1alpha3.GitTarget + for i := range list.Items { + target := &list.Items[i] + if target.DeletionTimestamp.IsZero() && Names(target, trigger) { + targets = append(targets, target) + } + } + sort.Slice(targets, func(i, j int) bool { return targets[i].Name < targets[j].Name }) + return targets, nil +} + +// Names reports whether target's spec.applyTriggerRef names trigger. +func Names(target *v1alpha3.GitTarget, trigger v1alpha3.ApplyTrigger) bool { + ref := target.Spec.ApplyTriggerRef + return ref != nil && target.Namespace == trigger.GetNamespace() && ref.Kind == trigger.TriggerKind() && + ref.Name == trigger.GetName() +} + +// JudgeReferences judges each of targets, which name trigger, against the source its applier +// fetches, or with source nil, on admission alone: a reference that is admitted is then undecided. +// A read that fails with anything but NotFound is an error, so the caller keeps what it published. +func JudgeReferences( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, targets []*v1alpha3.GitTarget, + source *Source, +) (References, error) { + refs := make(References, 0, len(targets)) + for _, target := range targets { + ref, err := judgeReference(ctx, reader, trigger, target, source) + if err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, nil +} + +func judgeReference( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, target *v1alpha3.GitTarget, + source *Source, +) (Reference, error) { + ref := Reference{Target: types.NamespacedName{Namespace: target.Namespace, Name: target.Name}} + decision, err := authz.ApplierAdmitted(ctx, reader, target, trigger) + switch { + case err != nil: + return ref, fmt.Errorf("check GitTarget %s against its ClusterProviders: %w", ref.Target, err) + case decision.Allowed: + case decision.Reason == authz.ReasonClusterProviderNotFound: + // A provider may be on its way, as the trigger's own ClusterProviderNotFound may. + ref.State, ref.Message = ReferenceUndecided, decision.Message + return ref, nil + default: + ref.State, ref.Reason, ref.Message = ReferenceRefused, decision.Reason, decision.Message + return ref, nil + } + + provider := &v1alpha3.GitProvider{} + key := client.ObjectKey{Namespace: target.Namespace, Name: target.Spec.GitProviderRef.Name} + if err := reader.Get(ctx, key, provider); err != nil { + if !apierrors.IsNotFound(err) { + return ref, fmt.Errorf("read GitProvider %s: %w", key, err) + } + ref.State, ref.Message = ReferenceUndecided, fmt.Sprintf("GitProvider %s not found", key) + return ref, nil + } + if source == nil { + ref.State, ref.Message = ReferenceUndecided, "the applier's source has not been read yet" + return ref, nil + } + want := Binding{RepositoryURL: provider.Spec.URL, Branch: target.Spec.Branch} + if mismatch := bindingMismatch(trigger, source.URL, source.Revision, want); mismatch != "" { + ref.State, ref.Reason, ref.Message = ReferenceRefused, ReasonSourceMismatch, mismatch + return ref, nil + } + return ref, nil +} + +// bindingMismatch returns why an applier fetching url and branch does not apply what want writes, +// or "" when it does. +func bindingMismatch(trigger v1alpha3.ApplyTrigger, url, branch string, want Binding) string { + name := trigger.TriggerKind() + " " + trigger.GetName() + if !repourl.Same(url, want.RepositoryURL) { + return fmt.Sprintf("the applier of %s applies %s, not %s", name, url, want.RepositoryURL) + } + if branch != want.Branch { + return fmt.Sprintf("the applier of %s applies branch %q, not %s", name, branch, want.Branch) + } + return "" +} + +// eligible reports whether at least one GitTarget naming trigger is eligible, judged against +// source, the applier's source as the caller read it: triggers in line for one applier share it. +// With no source read, a reference that is not refused counts: a trigger that may be eligible is +// taken to be, so a caller that cannot tell waits rather than acts. +func eligible( + ctx context.Context, reader client.Reader, trigger v1alpha3.ApplyTrigger, source *Source, +) (bool, error) { + if !trigger.GetDeletionTimestamp().IsZero() { + return false, nil + } + targets, err := NamingTargets(ctx, reader, trigger) + if err != nil || len(targets) == 0 { + return false, err + } + refs, err := JudgeReferences(ctx, reader, trigger, targets, source) + if err != nil || source != nil { + return refs.Eligible(), err + } + for _, ref := range refs { + if ref.State != ReferenceRefused { + return true, nil + } + } + return false, nil +} diff --git a/internal/applier/status_writer.go b/internal/applier/status_writer.go new file mode 100644 index 000000000..8b7a42214 --- /dev/null +++ b/internal/applier/status_writer.go @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// DefaultStatusRetryInterval is how soon a trigger status write that failed is tried again. +const DefaultStatusRetryInterval = 5 * time.Second + +// maxStatusConflicts bounds how often one write re-reads a trigger after losing a race with its +// reconciler, which writes the conditions and the source of the same status. +const maxStatusConflicts = 5 + +// StatusWriter writes what the branch workers publish on their apply triggers: status.operation +// and status.repairPending. It implements git.ApplyTriggerStatusWriter, and runs as a manager.Runnable +// beside the branch workers, on the leader as they are. +// +// A publication never blocks the worker. It is held until written, and a later publication for the +// same trigger replaces one not yet written, so the status catches up with the worker's latest view +// rather than replaying each step. A write that fails is tried again after RetryInterval, unless a +// newer publication replaced it in between. +// +// An operation an attempt starts reaches the trigger first through RecordApplyOperation, which the +// dispatcher calls just before the write that starts the apply, so the trigger never shows an +// operation that nothing could have applied. Writes are made one at a time, each of the latest +// publication, so a publication is never written over a later one. +// +// The status may hold an operation somebody else recorded: an earlier process, or the trigger's +// reconciler for an apply started outside GitOps Reverser. A publication replaces status.operation +// only on a status that holds none, or holds the same operation (sameOperation) at an earlier +// point, so it never writes over a record that is not the one it carries. An operation the worker +// withdraws, because its attempt was recorded and then asked for no apply, is replaced by what the +// status held before it was recorded. +// +// The rest of the status (the conditions, the source) belongs to the trigger's reconciler. Each +// write reads the trigger, sets these two fields and updates the status with the resourceVersion it +// read, so a status the reconciler wrote in between is never overwritten: the update conflicts and +// the write reads again. +type StatusWriter struct { + reader client.Reader + writer client.StatusClient + log logr.Logger + + // RetryInterval is how soon a failed write is tried again. Set before Start. + RetryInterval time.Duration + + // writing is held for every write, and for the whole of a batch, so writes land in the order + // their publications were made. + writing sync.Mutex + // recorded is the operation RecordApplyOperation last wrote on each trigger, with what the + // status held before it. It is read and written under writing. + recorded map[git.TriggerRef]recordedOperation + + mu sync.Mutex + pending map[git.TriggerRef]publication + wake chan struct{} +} + +// publication is what is held for one trigger until it is written: the worker's latest status, and +// every operation withdrawn since the last write, which a later publication never takes back. +type publication struct { + status git.ApplyTriggerStatus + withdrawn []*v1alpha3.ApplyOperation +} + +// merged is newer, withdrawing what p withdrew as well. +func (p publication) merged(newer publication) publication { + for _, operation := range p.withdrawn { + if !holdsOperation(newer.withdrawn, operation) { + newer.withdrawn = append(newer.withdrawn, operation) + } + } + return newer +} + +// recordedOperation is an operation RecordApplyOperation wrote, and the operation it replaced. +type recordedOperation struct { + operation *v1alpha3.ApplyOperation + displaced *v1alpha3.ApplyOperation +} + +var ( + // errTriggerReleased is a write to a trigger that is gone, or being deleted. + errTriggerReleased = errors.New("the trigger is gone or being deleted") + // errOperationInFlight is a record refused because the trigger's status holds another operation + // that has not completed. + errOperationInFlight = errors.New("the trigger records another operation that has not completed") +) + +var ( + _ git.ApplyTriggerStatusWriter = (*StatusWriter)(nil) + _ manager.Runnable = (*StatusWriter)(nil) + _ manager.LeaderElectionRunnable = (*StatusWriter)(nil) +) + +// NewStatusWriter returns a StatusWriter that reads triggers through reader, uncached so a write +// does not lose its race to a lagging cache, and writes their status through writer. +func NewStatusWriter(reader client.Reader, writer client.StatusClient, log logr.Logger) *StatusWriter { + return &StatusWriter{ + reader: reader, + writer: writer, + log: log, + RetryInterval: DefaultStatusRetryInterval, + recorded: map[git.TriggerRef]recordedOperation{}, + pending: map[git.TriggerRef]publication{}, + wake: make(chan struct{}, 1), + } +} + +// NeedLeaderElection is true: the branch workers that publish run only on the leader. +func (w *StatusWriter) NeedLeaderElection() bool { return true } + +// PublishApplyTriggerStatus holds status for trigger until it is written, replacing any not yet +// written, but not what that one withdrew. It does not block. +func (w *StatusWriter) PublishApplyTriggerStatus(trigger git.TriggerRef, status git.ApplyTriggerStatus) { + next := publication{status: status} + if status.Operation != nil { + next.status.Operation = stored(status.Operation) + } + if status.Withdrawn != nil { + next.withdrawn = []*v1alpha3.ApplyOperation{stored(status.Withdrawn)} + } + w.mu.Lock() + w.pending[trigger] = w.pending[trigger].merged(next) + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +// stored is a copy of operation as the API server stores it: its times to the second, so a status +// that already holds it compares equal and is not written again. +func stored(operation *v1alpha3.ApplyOperation) *v1alpha3.ApplyOperation { + operation = operation.DeepCopy() + operation.RequestedAt = operation.RequestedAt.Rfc3339Copy() + if operation.FinishedAt != nil { + finished := operation.FinishedAt.Rfc3339Copy() + operation.FinishedAt = &finished + } + return operation +} + +// Start writes what is published until ctx ends. +func (w *StatusWriter) Start(ctx context.Context) error { + var retry <-chan time.Time + for { + select { + case <-ctx.Done(): + return nil + case <-w.wake: + case <-retry: + } + retry = nil + if w.writePending(ctx) { + retry = time.After(w.RetryInterval) + } + } +} + +// RecordApplyOperation writes operation on the trigger now, and returns once it is written. It +// first writes what is held for the trigger, so the operation it replaces is the worker's latest +// view of it. It fails, so that nothing starts the apply, when the trigger is gone or being +// deleted, and when its status holds another operation that has not completed: an apply started +// outside GitOps Reverser, which the trigger's reconciler recorded, or one an earlier process +// recorded, which the worker follows before it asks for anything. +func (w *StatusWriter) RecordApplyOperation( + ctx context.Context, ref git.TriggerRef, operation *v1alpha3.ApplyOperation, +) error { + w.writing.Lock() + defer w.writing.Unlock() + if err := w.writeHeld(ctx, ref); err != nil { + return err + } + + operation = stored(operation) + for range maxStatusConflicts { + trigger, err := w.read(ctx, ref) + if err != nil { + return err + } + current := trigger.Operation() + switch { + case current != nil && sameOperation(current, operation): + return nil + case current != nil && !current.Phase.Completed(): + return fmt.Errorf("record the operation on %s: %w (%s, requested %s)", ref, errOperationInFlight, + current.Phase, current.RequestedAt.UTC().Format(time.RFC3339)) + } + // Kept before the update: an update that reports an error may still have landed. + w.recorded[ref] = recordedOperation{operation: operation, displaced: current.DeepCopy()} + trigger.SetOperation(operation, trigger.RepairPending()) + err = w.writer.Status().Update(ctx, trigger) + switch { + case err == nil: + return nil + case apierrors.IsNotFound(err): + return errTriggerReleased + case apierrors.IsConflict(err): + continue + default: + return fmt.Errorf("record the operation on %s: %w", ref, err) + } + } + return fmt.Errorf("record the operation on %s: it kept changing under the write", ref) +} + +// writeHeld writes what is held for one trigger, if anything, and holds it again should the write +// fail. The caller holds writing. +func (w *StatusWriter) writeHeld(ctx context.Context, ref git.TriggerRef) error { + w.mu.Lock() + held, ok := w.pending[ref] + delete(w.pending, ref) + w.mu.Unlock() + if !ok { + return nil + } + err := w.write(ctx, ref, held) + if err != nil && !errors.Is(err, errTriggerReleased) { + w.holdAgain(ref, held) + } + return err +} + +// writePending writes every publication held, and reports whether one failed and is held again. +func (w *StatusWriter) writePending(ctx context.Context) bool { + w.writing.Lock() + defer w.writing.Unlock() + w.mu.Lock() + batch := w.pending + w.pending = map[git.TriggerRef]publication{} + w.mu.Unlock() + + failed := false + for trigger, held := range batch { + err := w.write(ctx, trigger, held) + if err == nil || errors.Is(err, errTriggerReleased) { + continue + } + w.log.Info("could not write an apply trigger's status; trying again", "trigger", trigger.String(), + "error", err.Error()) + if w.holdAgain(trigger, held) { + failed = true + } + } + return failed +} + +// holdAgain holds a publication whose write failed, under any made since, and reports whether none +// was: only then is a retry due, because a newer publication wakes the writer itself. +func (w *StatusWriter) holdAgain(ref git.TriggerRef, failed publication) bool { + w.mu.Lock() + defer w.mu.Unlock() + newer, ok := w.pending[ref] + if !ok { + w.pending[ref] = failed + return true + } + w.pending[ref] = failed.merged(newer) + return false +} + +// write sets one trigger's status.operation, as operationToWrite says, and status.repairPending. A +// trigger that is gone, or being deleted, takes the publication with it: errTriggerReleased. +func (w *StatusWriter) write(ctx context.Context, ref git.TriggerRef, held publication) error { + for range maxStatusConflicts { + trigger, err := w.read(ctx, ref) + if err != nil { + return err + } + operation := w.operationToWrite(ref, held, trigger.Operation()) + repairPending := held.status.RepairPending + if trigger.RepairPending() == repairPending && equality.Semantic.DeepEqual(trigger.Operation(), operation) { + return nil + } + trigger.SetOperation(operation, repairPending) + err = w.writer.Status().Update(ctx, trigger) + switch { + case err == nil: + return nil + case apierrors.IsNotFound(err): + return errTriggerReleased + case apierrors.IsConflict(err): + continue + default: + return fmt.Errorf("write the status of %s: %w", ref, err) + } + } + return fmt.Errorf("write the status of %s: it kept changing under the write", ref) +} + +// read reads a trigger to write, and errTriggerReleased when it is gone or being deleted. +func (w *StatusWriter) read(ctx context.Context, ref git.TriggerRef) (v1alpha3.ApplyTrigger, error) { + trigger := v1alpha3.NewApplyTrigger(ref.Kind) + if trigger == nil { + return nil, errTriggerReleased + } + err := w.reader.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, trigger) + switch { + case apierrors.IsNotFound(err): + return nil, errTriggerReleased + case err != nil: + return nil, fmt.Errorf("read %s: %w", ref, err) + case !trigger.GetDeletionTimestamp().IsZero(): + return nil, errTriggerReleased + } + return trigger, nil +} + +// operationToWrite is the operation a write of held leaves on a trigger whose status holds current. +// An operation held withdraws is replaced by what the status held before this writer recorded it, +// or by none. Then the published operation is written over none, or over the same operation at an +// earlier point; never over another, which somebody else recorded, and never in flight over the +// same one completed: a worker that recovered an external resume publishes it as it was listed, +// and the trigger's reconciler can have completed it since. +func (w *StatusWriter) operationToWrite( + ref git.TriggerRef, held publication, current *v1alpha3.ApplyOperation, +) *v1alpha3.ApplyOperation { + operation := current + if current != nil && holdsOperation(held.withdrawn, current) { + operation = nil + if recorded, ok := w.recorded[ref]; ok && sameOperation(recorded.operation, current) { + operation = recorded.displaced + } + } + published := held.status.Operation + switch { + case published == nil: + case operation == nil: + operation = published + case sameOperation(operation, published) && (!operation.Phase.Completed() || published.Phase.Completed()): + operation = published + } + return operation +} + +// sameOperation reports whether a and b are one operation, perhaps at different points: an +// operation keeps the token and requestedAt it was recorded with. The requestedAt tells apart the +// operations the trigger's reconciler records, which carry no token. +func sameOperation(a, b *v1alpha3.ApplyOperation) bool { + return a.Token == b.Token && a.RequestedAt.Equal(&b.RequestedAt) +} + +// holdsOperation reports whether operations holds operation, by sameOperation. +func holdsOperation(operations []*v1alpha3.ApplyOperation, operation *v1alpha3.ApplyOperation) bool { + for _, held := range operations { + if sameOperation(held, operation) { + return true + } + } + return false +} diff --git a/internal/applier/status_writer_test.go b/internal/applier/status_writer_test.go new file mode 100644 index 000000000..f90e61db2 --- /dev/null +++ b/internal/applier/status_writer_test.go @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applier + +import ( + "context" + "errors" + "testing" + "time" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +var appsTrigger = git.TriggerRef{Kind: v1alpha3.ApplyTriggerKindFlux, Namespace: targetNS, Name: "apps"} + +// statusWriterFixture is a StatusWriter over a fake control cluster holding the FluxTrigger "apps", +// whose reconciler has already published Configured. +func statusWriterFixture(t *testing.T, funcs interceptor.Funcs) (*StatusWriter, client.Client) { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha3.AddToScheme(scheme)) + trigger := &v1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: targetNS, Name: "apps"}, + Spec: v1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &fluxmeta.LocalObjectReference{Name: v1alpha3.DefaultClusterProviderName}, + KustomizationRef: fluxmeta.NamespacedObjectReference{Name: "apps"}, + }, + Status: v1alpha3.FluxTriggerStatus{Conditions: []metav1.Condition{{ + Type: "Configured", Status: metav1.ConditionTrue, Reason: "Succeeded", + LastTransitionTime: metav1.Now(), + }}}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(trigger). + WithStatusSubresource(&v1alpha3.FluxTrigger{}).WithInterceptorFuncs(funcs).Build() + writer := NewStatusWriter(c, c, logr.Discard()) + writer.RetryInterval = time.Millisecond + return writer, c +} + +func readTrigger(t *testing.T, c client.Client) *v1alpha3.FluxTrigger { + t.Helper() + trigger := &v1alpha3.FluxTrigger{} + require.NoError(t, c.Get(context.Background(), client.ObjectKey{Namespace: targetNS, Name: "apps"}, trigger)) + return trigger +} + +func operationAt(phase v1alpha3.ApplyOperationPhase) *v1alpha3.ApplyOperation { + return &v1alpha3.ApplyOperation{ + Phase: phase, Reason: v1alpha3.ApplyReasonRefusal, Commit: testCommit, Token: "T1", + RequestedAt: metav1.NewTime(stampTime), + } +} + +// TestStatusWriter_WritesTheOperationAndKeepsTheRest: the worker's two fields land, and the +// reconciler's conditions stay as they were. +func TestStatusWriter_WritesTheOperationAndKeepsTheRest(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{ + Operation: operationAt(v1alpha3.ApplyOperationRunning), RepairPending: true, + }) + assert.False(t, writer.writePending(context.Background())) + + trigger := readTrigger(t, c) + require.NotNil(t, trigger.Status.Operation) + assert.Equal(t, v1alpha3.ApplyOperationRunning, trigger.Status.Operation.Phase) + assert.Equal(t, "T1", trigger.Status.Operation.Token) + assert.True(t, trigger.Status.RepairPending) + require.Len(t, trigger.Status.Conditions, 1, "the reconciler's conditions are kept") + + // No operation in the publication leaves the recorded one. + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{RepairPending: false}) + assert.False(t, writer.writePending(context.Background())) + trigger = readTrigger(t, c) + require.NotNil(t, trigger.Status.Operation, "a worker with nothing to say about it keeps it") + assert.False(t, trigger.Status.RepairPending) +} + +// TestStatusWriter_TheLatestPublicationWins: publications not yet written are replaced, not +// replayed. +func TestStatusWriter_TheLatestPublicationWins(t *testing.T) { + updates := 0 + writer, c := statusWriterFixture(t, interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, + opts ...client.SubResourceUpdateOption) error { + updates++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + for _, phase := range []v1alpha3.ApplyOperationPhase{ + v1alpha3.ApplyOperationRequested, v1alpha3.ApplyOperationRunning, v1alpha3.ApplyOperationSucceeded, + } { + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{Operation: operationAt(phase)}) + } + writer.writePending(context.Background()) + + assert.Equal(t, 1, updates) + assert.Equal(t, v1alpha3.ApplyOperationSucceeded, readTrigger(t, c).Status.Operation.Phase) + + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{ + Operation: operationAt(v1alpha3.ApplyOperationSucceeded), + }) + writer.writePending(context.Background()) + assert.Equal(t, 1, updates, "a status that already holds it is not written again") +} + +// TestStatusWriter_AConflictReadsAgain: a write that lost to the reconciler reads the trigger again +// and writes over what the reconciler wrote, never over it blind. +func TestStatusWriter_AConflictReadsAgain(t *testing.T) { + conflicts := 1 + writer, c := statusWriterFixture(t, interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, + opts ...client.SubResourceUpdateOption) error { + if conflicts > 0 { + conflicts-- + return apierrors.NewConflict(schema.GroupResource{Resource: "fluxtriggers"}, "apps", + errors.New("the object has been modified")) + } + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + writer.PublishApplyTriggerStatus( + appsTrigger, + git.ApplyTriggerStatus{Operation: operationAt(v1alpha3.ApplyOperationFailed)}, + ) + assert.False(t, writer.writePending(context.Background())) + assert.Equal(t, v1alpha3.ApplyOperationFailed, readTrigger(t, c).Status.Operation.Phase) +} + +// TestStatusWriter_AFailedWriteIsHeldUntilItLands: a write that failed is tried again, unless a newer +// publication took its place; a trigger that is gone takes its publication with it. +func TestStatusWriter_AFailedWriteIsHeldUntilItLands(t *testing.T) { + failing := true + writer, c := statusWriterFixture(t, interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, + opts ...client.SubResourceUpdateOption) error { + if failing { + return errors.New("the API server is away") + } + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + writer.PublishApplyTriggerStatus( + appsTrigger, + git.ApplyTriggerStatus{Operation: operationAt(v1alpha3.ApplyOperationRunning)}, + ) + assert.True(t, writer.writePending(context.Background()), "held for a retry") + + failing = false + assert.False(t, writer.writePending(context.Background())) + assert.Equal(t, v1alpha3.ApplyOperationRunning, readTrigger(t, c).Status.Operation.Phase) + + gone := git.TriggerRef{Kind: v1alpha3.ApplyTriggerKindFlux, Namespace: targetNS, Name: "gone"} + writer.PublishApplyTriggerStatus(gone, git.ApplyTriggerStatus{RepairPending: true}) + assert.False(t, writer.writePending(context.Background()), "nothing to hold for a trigger that is gone") +} + +// TestStatusWriter_StartWritesWhatIsPublished: the runnable writes a publication without being asked +// twice, and stops with its context. +func TestStatusWriter_StartWritesWhatIsPublished(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- writer.Start(ctx) }() + + writer.PublishApplyTriggerStatus( + appsTrigger, + git.ApplyTriggerStatus{Operation: operationAt(v1alpha3.ApplyOperationRequested)}, + ) + require.Eventually(t, func() bool { + operation := readTrigger(t, c).Status.Operation + return operation != nil && operation.Phase == v1alpha3.ApplyOperationRequested + }, 5*time.Second, 5*time.Millisecond) + + cancel() + require.NoError(t, <-done) + assert.True(t, writer.NeedLeaderElection()) +} + +// setOperation gives the trigger's status an operation, as another writer would. +func setOperation(t *testing.T, c client.Client, operation *v1alpha3.ApplyOperation) { + t.Helper() + trigger := readTrigger(t, c) + trigger.Status.Operation = operation + require.NoError(t, c.Status().Update(context.Background(), trigger)) +} + +// operationRequested is a worker's operation with its own token, requested after operationAt's. +func operationRequested( + phase v1alpha3.ApplyOperationPhase, token string, after time.Duration, +) *v1alpha3.ApplyOperation { + operation := operationAt(phase) + operation.Token, operation.RequestedAt = token, metav1.NewTime(stampTime.Add(after)) + return operation +} + +// assertOperation asserts the status holds want, as the API server stores it. +func assertOperation(t *testing.T, want, got *v1alpha3.ApplyOperation) { + t.Helper() + if want == nil { + assert.Nil(t, got) + return + } + assert.True(t, equality.Semantic.DeepEqual(stored(want), got), "want %+v, got %+v", want, got) +} + +// countingUpdates counts the status updates a fixture makes. +func countingUpdates(updates *int) interceptor.Funcs { + return interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, + opts ...client.SubResourceUpdateOption) error { + *updates++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + } +} + +// TestStatusWriter_ARecordIsWrittenThroughBeforeItReturns: the dispatcher records the operation +// just before it starts the apply. It is on the trigger when RecordApplyOperation returns, over the +// completed operation before it, and the reconciler's conditions and the recorded repairPending stay. +func TestStatusWriter_ARecordIsWrittenThroughBeforeItReturns(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + setOperation(t, c, operationAt(v1alpha3.ApplyOperationSucceeded)) + writer.PublishApplyTriggerStatus(appsTrigger, + git.ApplyTriggerStatus{Operation: operationAt(v1alpha3.ApplyOperationSucceeded), RepairPending: true}) + + requested := operationRequested(v1alpha3.ApplyOperationRequested, "T2", time.Minute) + require.NoError(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested)) + recorded := readTrigger(t, c) + require.NotNil(t, recorded.Status.Operation) + assert.Equal(t, "T2", recorded.Status.Operation.Token) + assert.Equal(t, v1alpha3.ApplyOperationRequested, recorded.Status.Operation.Phase) + assert.True(t, recorded.Status.RepairPending, "what was held is written first") + require.Len(t, recorded.Status.Conditions, 1, "the reconciler's conditions are kept") + + assert.False(t, writer.writePending(context.Background())) + assert.Equal(t, recorded.ResourceVersion, readTrigger(t, c).ResourceVersion, "nothing left to write") + require.NoError(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested), + "an operation recorded already is recorded") +} + +// TestStatusWriter_ARecordWritesTheLatestViewFirst: the operation before the one recorded completed +// on the worker, and its completion is still held. The record writes it first, so it replaces a +// completed operation, not one in flight that it would refuse. +func TestStatusWriter_ARecordWritesTheLatestViewFirst(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + setOperation(t, c, operationAt(v1alpha3.ApplyOperationRunning)) + writer.PublishApplyTriggerStatus(appsTrigger, + git.ApplyTriggerStatus{Operation: operationAt(v1alpha3.ApplyOperationSucceeded)}) + + requested := operationRequested(v1alpha3.ApplyOperationRequested, "T2", time.Minute) + require.NoError(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested)) + assert.Equal(t, "T2", readTrigger(t, c).Status.Operation.Token) +} + +// TestStatusWriter_ARecordNeverReplacesAnotherOperationInFlight is the fourth review's first +// finding. The trigger's reconciler recorded an external resume (no token, requested after the +// worker sent its attempt), and the record used to keep it, write nothing, and return nil, so the +// resume went ahead with no record of its own. The record fails, and leaves the status as it is. +func TestStatusWriter_ARecordNeverReplacesAnotherOperationInFlight(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + external := &v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationUnresolved, SuspendToken: "F1", + RequestedAt: metav1.NewTime(stampTime.Add(time.Hour)), + } + setOperation(t, c, external) + + requested := operationRequested(v1alpha3.ApplyOperationRequested, "T2", time.Minute) + err := writer.RecordApplyOperation(context.Background(), appsTrigger, requested) + require.ErrorIs(t, err, errOperationInFlight) + assertOperation(t, external, readTrigger(t, c).Status.Operation) + + // An operation an earlier process left in flight is refused the same way: the worker follows it + // before it asks for anything. + setOperation(t, c, operationRequested(v1alpha3.ApplyOperationRunning, "T0", -time.Hour)) + require.ErrorIs(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested), + errOperationInFlight) +} + +// TestStatusWriter_ARecordForAReleasedTriggerFails: a trigger that is being deleted is an error, so +// nothing is started for it, also when the background writer already dropped what was held for it. +func TestStatusWriter_ARecordForAReleasedTriggerFails(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + trigger := readTrigger(t, c) + trigger.Finalizers = []string{v1alpha3.FluxTriggerFinalizer} + require.NoError(t, c.Update(context.Background(), trigger)) + require.NoError(t, c.Delete(context.Background(), trigger)) + requested := operationRequested(v1alpha3.ApplyOperationRequested, "T2", time.Minute) + + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{RepairPending: true}) + require.ErrorIs(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested), + errTriggerReleased, "with a publication held") + + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{RepairPending: true}) + assert.False(t, writer.writePending(context.Background()), "the background writer drops it") + require.ErrorIs(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested), + errTriggerReleased, "with nothing held") + assert.Nil(t, readTrigger(t, c).Status.Operation) +} + +// TestStatusWriter_AWithdrawnRecordIsPutBack: the operation was recorded, and the apply then never +// started (the resume failed). The worker withdraws it, and the status shows what it held before +// the record again: the completed operation before it, or none. +func TestStatusWriter_AWithdrawnRecordIsPutBack(t *testing.T) { + for name, before := range map[string]*v1alpha3.ApplyOperation{ + "a completed operation": operationAt(v1alpha3.ApplyOperationSucceeded), + "none": nil, + } { + t.Run(name, func(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + if before != nil { + setOperation(t, c, before) + } + requested := operationRequested(v1alpha3.ApplyOperationRequested, "T2", time.Minute) + require.NoError(t, writer.RecordApplyOperation(context.Background(), appsTrigger, requested)) + require.Equal(t, "T2", readTrigger(t, c).Status.Operation.Token) + + // The worker knows nothing older than the record: it withdraws it with no operation. + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{Withdrawn: requested}) + // A later publication that withdraws nothing does not take the withdrawal back. + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{RepairPending: true}) + assert.False(t, writer.writePending(context.Background())) + + written := readTrigger(t, c) + assertOperation(t, before, written.Status.Operation) + assert.True(t, written.Status.RepairPending) + }) + } +} + +// TestStatusWriter_AWithdrawalOfAnOperationNotHeldChangesNothing: an attempt that ended before its +// record withdraws an operation the status never held, and the status is not written. +func TestStatusWriter_AWithdrawalOfAnOperationNotHeldChangesNothing(t *testing.T) { + updates := 0 + writer, c := statusWriterFixture(t, countingUpdates(&updates)) + setOperation(t, c, operationAt(v1alpha3.ApplyOperationSucceeded)) + updates = 0 + + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{ + Operation: operationAt(v1alpha3.ApplyOperationSucceeded), + Withdrawn: operationRequested(v1alpha3.ApplyOperationRequested, "T2", time.Minute), + }) + assert.False(t, writer.writePending(context.Background())) + assert.Zero(t, updates) + assert.Equal(t, "T1", readTrigger(t, c).Status.Operation.Token) +} + +// TestStatusWriter_NeverWritesOverAnotherOperation: the reconciler records an external resume as an +// operation of its own. A publication of the worker's operation does not replace it, whether it was +// requested before it or after it; the rest of the publication still lands. +func TestStatusWriter_NeverWritesOverAnotherOperation(t *testing.T) { + for name, requested := range map[string]time.Duration{"before": -time.Minute, "after": time.Minute} { + t.Run(name, func(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + external := &v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationUnresolved, SuspendToken: "F1", RequestedAt: metav1.NewTime(stampTime), + } + setOperation(t, c, external) + + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{ + Operation: operationRequested(v1alpha3.ApplyOperationSucceeded, "T2", requested), + RepairPending: true, + }) + assert.False(t, writer.writePending(context.Background())) + written := readTrigger(t, c) + assertOperation(t, external, written.Status.Operation) + assert.True(t, written.Status.RepairPending, "the rest of the publication still lands") + }) + } +} + +// TestStatusWriter_NeverReopensACompletedOperation is review 6: a worker that recovers the +// reconciler's record of an external resume publishes it as it was listed, Unresolved; the +// reconciler can have completed it since. The same operation is never written back in flight. +func TestStatusWriter_NeverReopensACompletedOperation(t *testing.T) { + writer, c := statusWriterFixture(t, interceptor.Funcs{}) + listed := &v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationUnresolved, SuspendToken: "F1", RequestedAt: metav1.NewTime(stampTime), + } + completed := listed.DeepCopy() + completed.Phase = v1alpha3.ApplyOperationSucceeded + setOperation(t, c, completed) + + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{Operation: listed}) + assert.False(t, writer.writePending(context.Background())) + assertOperation(t, completed, readTrigger(t, c).Status.Operation) + + failed := listed.DeepCopy() + failed.Phase = v1alpha3.ApplyOperationFailed + writer.PublishApplyTriggerStatus(appsTrigger, git.ApplyTriggerStatus{Operation: failed}) + assert.False(t, writer.writePending(context.Background())) + assertOperation(t, failed, readTrigger(t, c).Status.Operation) // a completion still replaces a completion +} diff --git a/internal/audit/outcome/outcome.go b/internal/audit/outcome/outcome.go index 9d31e1a29..d27930876 100644 --- a/internal/audit/outcome/outcome.go +++ b/internal/audit/outcome/outcome.go @@ -91,7 +91,7 @@ const ( // none. // // It is Dropped rather than Error: nothing failed, and the events are exactly the population - // that was previously counted queued (which claimed an append that was never owed), so this + // that was previously counted queued (which claimed an append that was never expected), so this // population was invisible. The handler is the only place it can be counted at all — the event // is rejected before publication, so no fact-side counter ever sees it. NoAttributionFact Outcome = "no_attribution_fact" diff --git a/internal/audit/outcome/outcome_test.go b/internal/audit/outcome/outcome_test.go index eefeba9fb..2826b7fe9 100644 --- a/internal/audit/outcome/outcome_test.go +++ b/internal/audit/outcome/outcome_test.go @@ -33,7 +33,7 @@ func TestOutcomeCategory(t *testing.T) { OlderThanHighWater: Dropped, NonNumericRV: Dropped, MissingClusterAnnotation: Dropped, - // Accepted, and no fact was owed for it: dropped, never error, so the e2e invariant that + // Accepted, and no fact was expected for it: dropped, never error, so the e2e invariant that // gates on category="error" being zero is untouched by a type audit cannot identify. NoAttributionFact: Dropped, WriteError: Error, diff --git a/internal/authz/clusterprovider_admission.go b/internal/authz/clusterprovider_admission.go index 9f50124b8..edfbe0c4e 100644 --- a/internal/authz/clusterprovider_admission.go +++ b/internal/authz/clusterprovider_admission.go @@ -37,6 +37,11 @@ const ( // admitted by its ClusterProvider's spec.accessFrom — including the case where that // policy carries a selector the apiserver accepted but that does not convert to a selector. ReasonNamespaceNotAuthorized = "NamespaceNotAuthorized" + + // ReasonApplierNamespaceNotAllowed is the denial reason when a trigger names an applier in + // another namespace than its own and its ClusterProvider does not set + // spec.allowCrossNamespace.appliers. + ReasonApplierNamespaceNotAllowed = "ApplierNamespaceNotAllowed" ) // Decision is the outcome of a ClusterProvider namespace-admission check. A denial always carries @@ -68,55 +73,87 @@ func GitTargetAdmitted( target *configv1alpha3.GitTarget, ) (Decision, error) { providerName := target.SourceCluster() - return namespaceAdmitted(ctx, reader, target.Namespace, providerName, fmt.Sprintf( + decision, _, err := namespaceAdmitted(ctx, reader, target.Namespace, providerName, fmt.Sprintf( "referenced ClusterProvider %q was not found; a GitTarget may mirror a source cluster "+ "only through an existing ClusterProvider. The operator never creates one: declare it "+ "yourself, or let the chart render %q via clusterProvider.createDefault", providerName, configv1alpha3.DefaultClusterProviderName)) + return decision, err +} + +// ApplyTriggerAdmitted reports whether a trigger's namespace is admitted by the ClusterProvider it +// names (spec.clusterProviderRef), and whether it may reach its applier there. The applier is +// reached with that provider's credential, so the provider's accessFrom decides who may wield it, +// exactly as for mirroring, and its allowCrossNamespace.appliers whether a namespace may suspend +// and apply another namespace's applier. The trigger's own namespace is always allowed; in a remote +// cluster that is the namespace of the same name, as it is for mirroring. +func ApplyTriggerAdmitted( + ctx context.Context, + reader client.Reader, + trigger configv1alpha3.ApplyTrigger, +) (Decision, error) { + providerName := trigger.ClusterProvider() + decision, provider, err := namespaceAdmitted(ctx, reader, trigger.GetNamespace(), providerName, fmt.Sprintf( + "ClusterProvider %q named by %s %s/%s spec.clusterProviderRef was not found", + providerName, trigger.TriggerKind(), trigger.GetNamespace(), trigger.GetName())) + if err != nil || !decision.Allowed { + return decision, err + } + applier := trigger.Applier() + if applier.Namespace != trigger.GetNamespace() && !provider.AllowsCrossNamespaceAppliers() { + return Decision{ + Reason: ReasonApplierNamespaceNotAllowed, + Message: fmt.Sprintf( + "%s %s is in another namespace than the trigger's own %q, and ClusterProvider %q does not "+ + "set spec.allowCrossNamespace.appliers", trigger.ApplierKind(), applier, trigger.GetNamespace(), + providerName), + }, nil + } + return decision, nil } -// ApplierAdmitted reports whether target may ask its applier to apply: its namespace is admitted -// by its own ClusterProvider (GitTargetAdmitted) and, when spec.applier.clusterProviderRef names -// another one, by that one too. The applier is reached with that provider's credential, so the -// provider's accessFrom decides who may wield it, exactly as for mirroring. Call it only with -// spec.applier set. +// ApplierAdmitted reports whether target may ask the applier its trigger holds to apply: its +// namespace is admitted by its own ClusterProvider (GitTargetAdmitted) and by the trigger's +// (ApplyTriggerAdmitted). A GitTarget names a trigger in its own namespace only, so the second +// answers for the GitTarget's namespace too. A ClusterProvider's client is shared by every object +// that uses the cluster, so a client being available is no permission to use it. func ApplierAdmitted( ctx context.Context, reader client.Reader, target *configv1alpha3.GitTarget, + trigger configv1alpha3.ApplyTrigger, ) (Decision, error) { decision, err := GitTargetAdmitted(ctx, reader, target) if err != nil || !decision.Allowed { return decision, err } - providerName := target.ApplierCluster() - if providerName == target.SourceCluster() { + if trigger.ClusterProvider() == target.SourceCluster() && trigger.GetNamespace() == target.Namespace { return decision, nil } - return namespaceAdmitted(ctx, reader, target.Namespace, providerName, fmt.Sprintf( - "ClusterProvider %q named by spec.applier.clusterProviderRef was not found", providerName)) + return ApplyTriggerAdmitted(ctx, reader, trigger) } // namespaceAdmitted is the check both admissions share: the provider exists, and its accessFrom -// admits namespace. notFound is the denial message for a provider that does not exist. +// admits namespace. notFound is the denial message for a provider that does not exist. It returns +// the provider it read once it exists. func namespaceAdmitted( ctx context.Context, reader client.Reader, namespace, providerName, notFound string, -) (Decision, error) { - var provider configv1alpha3.ClusterProvider - if err := reader.Get(ctx, k8stypes.NamespacedName{Name: providerName}, &provider); err != nil { +) (Decision, *configv1alpha3.ClusterProvider, error) { + provider := &configv1alpha3.ClusterProvider{} + if err := reader.Get(ctx, k8stypes.NamespacedName{Name: providerName}, provider); err != nil { if apierrors.IsNotFound(err) { - return Decision{Reason: ReasonClusterProviderNotFound, Message: notFound}, nil + return Decision{Reason: ReasonClusterProviderNotFound, Message: notFound}, nil, nil } - return Decision{}, fmt.Errorf("read ClusterProvider %q: %w", providerName, err) + return Decision{}, nil, fmt.Errorf("read ClusterProvider %q: %w", providerName, err) } nsLabels := map[string]string{} var ns corev1.Namespace if err := reader.Get(ctx, k8stypes.NamespacedName{Name: namespace}, &ns); err != nil { if !apierrors.IsNotFound(err) { - return Decision{}, fmt.Errorf("read namespace %q: %w", namespace, err) + return Decision{}, nil, fmt.Errorf("read namespace %q: %w", namespace, err) } } else { nsLabels = ns.Labels @@ -128,7 +165,7 @@ func namespaceAdmitted( Reason: ReasonNamespaceNotAuthorized, Message: fmt.Sprintf( "ClusterProvider %q accessFrom selector is invalid: %v", providerName, selErr), - }, nil + }, provider, nil } if !allowed { return Decision{ @@ -136,8 +173,8 @@ func namespaceAdmitted( Message: fmt.Sprintf( "namespace %q is not permitted to reference ClusterProvider %q (spec.accessFrom)", namespace, providerName), - }, nil + }, provider, nil } - return Decision{Allowed: true}, nil + return Decision{Allowed: true}, provider, nil } diff --git a/internal/authz/clusterprovider_admission_test.go b/internal/authz/clusterprovider_admission_test.go index deb835c08..e41eb40df 100644 --- a/internal/authz/clusterprovider_admission_test.go +++ b/internal/authz/clusterprovider_admission_test.go @@ -301,58 +301,56 @@ func TestGitTargetAdmitted_ReadErrorsSurfaceAsError(t *testing.T) { } } -// TestApplierAdmitted: an applier in another cluster is reached with that ClusterProvider's -// credential, so its accessFrom must admit the GitTarget too, and the source's denial comes first. +// TestApplierAdmitted: an applier is reached with its trigger's ClusterProvider's credential, so +// that provider's accessFrom must admit the namespace too, and the source's denial comes first. func TestApplierAdmitted(t *testing.T) { admits := func(name string, namespaces ...string) *configv1alpha3.ClusterProvider { return providerNamed(name, &configv1alpha3.NamespaceMatcher{Names: namespaces}) } - withApplier := func(source, applierCluster string) *configv1alpha3.GitTarget { - target := targetIn(source) - target.Spec.Applier = &configv1alpha3.GitTargetApplier{} - if applierCluster != "" { - target.Spec.Applier.ClusterProviderRef = &meta.LocalObjectReference{Name: applierCluster} + triggerOn := func(cluster string) configv1alpha3.ApplyTrigger { + return &configv1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNS, Name: "apps"}, + Spec: configv1alpha3.FluxTriggerSpec{ClusterProviderRef: &meta.LocalObjectReference{Name: cluster}}, } - return target } tests := []struct { name string objects []client.Object - target *configv1alpha3.GitTarget + trigger configv1alpha3.ApplyTrigger wantAllowed bool wantReason string wantMessage string }{ { - name: "omitted, the applier's provider is the source's", + name: "the trigger's provider is the source's", objects: []client.Object{admits("prod-eu-1", testNS)}, - target: withApplier("prod-eu-1", ""), + trigger: triggerOn("prod-eu-1"), wantAllowed: true, }, { name: "another provider that admits the namespace", objects: []client.Object{admits("prod-eu-1", testNS), admits("hub", testNS)}, - target: withApplier("prod-eu-1", "hub"), + trigger: triggerOn("hub"), wantAllowed: true, }, { name: "another provider that does not admit the namespace", objects: []client.Object{admits("prod-eu-1", testNS), admits("hub", "elsewhere")}, - target: withApplier("prod-eu-1", "hub"), + trigger: triggerOn("hub"), wantReason: ReasonNamespaceNotAuthorized, wantMessage: `ClusterProvider "hub"`, }, { name: "another provider that does not exist", objects: []client.Object{admits("prod-eu-1", testNS)}, - target: withApplier("prod-eu-1", "hub"), + trigger: triggerOn("hub"), wantReason: ReasonClusterProviderNotFound, - wantMessage: "spec.applier.clusterProviderRef", + wantMessage: "FluxTrigger " + testNS + "/apps spec.clusterProviderRef", }, { name: "the source's denial comes first", objects: []client.Object{admits("prod-eu-1", "elsewhere"), admits("hub", testNS)}, - target: withApplier("prod-eu-1", "hub"), + trigger: triggerOn("hub"), wantReason: ReasonNamespaceNotAuthorized, wantMessage: `ClusterProvider "prod-eu-1"`, }, @@ -360,7 +358,7 @@ func TestApplierAdmitted(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { reader := fake.NewClientBuilder().WithScheme(admissionScheme(t)).WithObjects(tc.objects...).Build() - decision, err := ApplierAdmitted(context.Background(), reader, tc.target) + decision, err := ApplierAdmitted(context.Background(), reader, targetIn("prod-eu-1"), tc.trigger) require.NoError(t, err) assert.Equal(t, tc.wantAllowed, decision.Allowed, decision.Message) assert.Equal(t, tc.wantReason, decision.Reason) diff --git a/internal/authz/source_namespace.go b/internal/authz/source_namespace.go index 829a1c679..c7b4e532e 100644 --- a/internal/authz/source_namespace.go +++ b/internal/authz/source_namespace.go @@ -37,7 +37,7 @@ const ( // ReasonSourceNamespaceNotAllowed is the TERMINAL False reason for a refusal: the referenced // ClusterProvider does not exist, does not admit this GitTarget's namespace, or does not set - // spec.allowAnySourceNamespace. It is a decision, never an inability to decide. + // spec.allowCrossNamespace.sources. It is a decision, never an inability to decide. ReasonSourceNamespaceNotAllowed = "SourceNamespaceNotAllowed" ) @@ -118,7 +118,7 @@ func (s ResolvedSourceScope) Fingerprint() string { // on upgrade. // 2. Any OTHER namespace — a name, or "*" for every namespace the credential can read — // additionally requires the GitTarget's namespace to be admitted by its ClusterProvider, and -// that provider to set spec.allowAnySourceNamespace. +// that provider to set spec.allowCrossNamespace.sources. // // There is no third rule. GitTarget.spec.allowedSourceNamespaces used to supply one, and what it // bounded is now bounded by the source credential's own RBAC: a request this gate permits still @@ -193,7 +193,7 @@ func (g *itemGate) decide( // named-namespace collection on the same type, never a replacement for one. base.Namespaces = []string{""} base.Message = fmt.Sprintf( - "%s: ClusterProvider %q sets spec.allowAnySourceNamespace, so this item watches every "+ + "%s: ClusterProvider %q sets spec.allowCrossNamespace.sources, so this item watches every "+ "namespace its source credential can read, as one cluster-wide stream", g.describeItem(index, item), g.target.SourceCluster()) return base, nil @@ -202,7 +202,7 @@ func (g *itemGate) decide( candidate := item.EffectiveSourceNamespace(g.rule.Namespace) base.Namespaces = []string{candidate} base.Message = fmt.Sprintf( - "%s: ClusterProvider %q sets spec.allowAnySourceNamespace, so this item may watch source "+ + "%s: ClusterProvider %q sets spec.allowCrossNamespace.sources, so this item may watch source "+ "namespace %q; whether it can is decided by that credential's RBAC", g.describeItem(index, item), g.target.SourceCluster(), candidate) return base, nil @@ -251,7 +251,7 @@ func (g *itemGate) evaluateDelegation(ctx context.Context) (*SourceNamespaceDeci Message: fmt.Sprintf( "referenced ClusterProvider %q was not found, so it delegates nothing; a "+ "WatchRule may watch a namespace other than its own only through an "+ - "existing provider that sets spec.allowAnySourceNamespace", + "existing provider that sets spec.allowCrossNamespace.sources", providerName), }, nil } @@ -273,11 +273,11 @@ func (g *itemGate) evaluateDelegation(ctx context.Context) (*SourceNamespaceDeci }, nil } - if !provider.AllowsAnySourceNamespace() { + if !provider.AllowsCrossNamespaceSources() { return &SourceNamespaceDecision{ Reason: ReasonSourceNamespaceNotAllowed, Message: fmt.Sprintf( - "ClusterProvider %q does not set spec.allowAnySourceNamespace; a WatchRule may "+ + "ClusterProvider %q does not set spec.allowCrossNamespace.sources; a WatchRule may "+ "watch only its own namespace %q until a platform admin delegates that choice", providerName, g.rule.Namespace), }, nil diff --git a/internal/authz/source_namespace_test.go b/internal/authz/source_namespace_test.go index 222a0f151..71645d9a3 100644 --- a/internal/authz/source_namespace_test.go +++ b/internal/authz/source_namespace_test.go @@ -75,8 +75,8 @@ func snClusterProvider(delegate bool) *configv1alpha3.ClusterProvider { return &configv1alpha3.ClusterProvider{ ObjectMeta: metav1.ObjectMeta{Name: snProvider}, Spec: configv1alpha3.ClusterProviderSpec{ - AccessFrom: &configv1alpha3.NamespaceMatcher{Names: []string{snTenantNS}}, - AllowAnySourceNamespace: delegate, + AccessFrom: &configv1alpha3.NamespaceMatcher{Names: []string{snTenantNS}}, + AllowCrossNamespace: &configv1alpha3.ClusterProviderCrossNamespace{Sources: delegate}, }, } } diff --git a/internal/controller/applytrigger_controller.go b/internal/controller/applytrigger_controller.go new file mode 100644 index 000000000..4fddbac54 --- /dev/null +++ b/internal/controller/applytrigger_controller.go @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/authz" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// ApplyTriggerConditionConfigured reports whether a trigger's applier is one GitOps Reverser can +// drive: it exists in its cluster, applies a source that follows a branch, may be written, and (an +// Argo CD Application) has automated sync off. The trigger's ClusterProvider must admit its +// namespace first, and no trigger in line before it may name the same applier +// (applier.ReasonHeldByAnotherTrigger). +const ApplyTriggerConditionConfigured = "Configured" + +// ReasonOperationUnresolved is Ready=False on a trigger whose operation has not completed in the +// time it was given. It clears when the operation completes. +const ReasonOperationUnresolved = "OperationUnresolved" + +// ApplyTriggerConditionHeld reports whether a FluxTrigger holds its Kustomization: suspended +// under GitOps Reverser's field manager, with the last suspend's token handled, so nothing that +// started before the suspend is still running, or resumed for an apply of GitOps Reverser's own. A +// FluxTrigger is Ready only once it is. +const ApplyTriggerConditionHeld = applier.TriggerConditionHeld + +// Held reasons, and the Events of the suspend lifecycle on a FluxTrigger. +const ( + // ReasonSuspended is Held=True: the Kustomization is suspended, the last suspend handled, and + // nothing applies. + ReasonSuspended = "Suspended" + // ReasonApplying is Held=True while an operation of a branch worker is in flight and followed: + // the Kustomization is resumed for it, and suspended again when it ends. + ReasonApplying = "Applying" + // ReasonSuspending is Held=False while the token of the suspend is not handled yet. + ReasonSuspending = "Suspending" + // ReasonNotConfigured is Held=Unknown on a trigger that is not Configured: nothing is held, or + // the Kustomization is left suspended as the trigger last held it (the message says which). + ReasonNotConfigured = "NotConfigured" + // ReasonReleasing is Held=False on a trigger that held its Kustomization and is handing it back: + // it is being deleted, or no eligible GitTarget references it any more. Held=False with + // applier.ReasonNotReferenced is a trigger with nothing to hold. + ReasonReleasing = "Releasing" + // EventReasonAdopted is a Kustomization suspended for the first time. + EventReasonAdopted = "Adopted" + // EventReasonResumedElsewhere is a held Kustomization found resumed by somebody else, such as + // `flux resume`, and suspended again. + EventReasonResumedElsewhere = "ResumedElsewhere" + // EventReasonReleased is a Kustomization handed back to Flux, or left suspended for the trigger + // next in line, when its trigger was deleted or no eligible GitTarget references it any more. + EventReasonReleased = "Released" + // EventReasonReleaseFailed is a retiring trigger that could not hand its Kustomization back. + EventReasonReleaseFailed = "ReleaseFailed" +) + +// fluxTriggerSteadyInterval is how often a FluxTrigger that is Ready reads its Kustomization again: +// that read is what finds a `flux resume`, so it is more often than other kinds settle. +const fluxTriggerSteadyInterval = time.Minute + +// applyTriggerSuspendingInterval is how soon a trigger waiting on its Kustomization's suspend is +// read again: a suspended reconcile handles the token in tens of milliseconds. +const applyTriggerSuspendingInterval = 2 * time.Second + +// applierCheckTimeout bounds the Configured check: at most two GETs and two access reviews +// against the applier's cluster. Its client has no request timeout, because it also carries the +// watches, and a cluster that accepts the connection and never answers would otherwise hold this +// controller's worker, not only this trigger. +const applierCheckTimeout = 10 * time.Second + +// Why the trigger reconciler reads the applier on every reconcile rather than watching it: Flux or +// Argo CD may not be installed in the applier's cluster at all, and an informer on a kind that is +// not served would be a failure to report rather than a check that answers "not found". The +// cadence is its own: RequeueSteadyInterval (fluxTriggerSteadyInterval for a FluxTrigger), +// applyTriggerSuspendingInterval while a Kustomization has not handled its suspend yet, or +// applyTriggerSettleInterval while something is expected to clear on its own, such as an applier +// created after its trigger. + +// applyTriggerSettleInterval is how soon a trigger that is not Ready, and not stalled, is read again. +const applyTriggerSettleInterval = 15 * time.Second + +// ApplyTriggerReconciler reconciles one apply trigger kind: it checks that the applier the trigger +// holds is one GitOps Reverser can drive, and publishes what it applies (status.source) for the +// GitTargets naming the trigger. For a FluxTrigger it also holds the Kustomization suspended, and +// hands it back once the trigger is deleted (fluxtrigger_hold.go). It never starts an apply: the +// branch workers decide when, and write status.operation and status.repairPending. It writes +// status.operation only for what no worker follows: an external resume, and an operation nobody +// follows any more. +type ApplyTriggerReconciler struct { + client.Client + + // Kind is the trigger kind this reconciler serves (v1alpha3.ApplyTriggerKind*). + Kind string + Recorder record.EventRecorder + + // ApplierClients resolves the dynamic client of a trigger's cluster. Nil leaves Configured + // Unknown (tests, and the CLI). + ApplierClients applier.Clients + // APIReader reads a trigger uncached before its Kustomization is suspended, so an operation a + // branch worker recorded a moment ago is never taken for an external resume. Nil reads through + // Client. + APIReader client.Reader + // Follower says which branch-worker operations this process follows (the apply dispatcher). A + // FluxTrigger's Kustomization is left resumed for an operation in flight only while it is + // followed, or for UnfollowedGrace (DefaultUnfollowedGrace when zero) after it was first seen + // followed by nobody. Nil follows nothing, so only the grace applies. + Follower OperationFollower + UnfollowedGrace time.Duration + unfollowed unfollowedOperations + // newApplier builds the client that checks an applier of a kind; nil is applier.NewClient. A + // test sets it to answer the access review without a real API server. + newApplier func(kind string, client dynamic.Interface) applier.Client +} + +// +kubebuilder:rbac:groups=configbutler.ai,resources=fluxtriggers;argocdtriggers,verbs=get;list;watch +// +kubebuilder:rbac:groups=configbutler.ai,resources=fluxtriggers,verbs=update;patch +// +kubebuilder:rbac:groups=configbutler.ai,resources=fluxtriggers/finalizers,verbs=update +// +kubebuilder:rbac:groups=configbutler.ai,resources=fluxtriggers/status;argocdtriggers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=configbutler.ai,resources=clusterproviders,verbs=get;list;watch +// +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets;gitproviders,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile publishes one trigger's Configured condition, its source and the readiness trio. +func (r *ApplyTriggerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + trigger := configbutleraiv1alpha3.NewApplyTrigger(r.Kind) + if trigger == nil { + return ctrl.Result{}, fmt.Errorf("no apply trigger kind %q", r.Kind) + } + if err := r.Get(ctx, req.NamespacedName, trigger); err != nil { + if apierrors.IsNotFound(err) { + telemetry.ForgetResourceConditions(r.Kind, req.Namespace, req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + st := beginStatus(r.Client, r.Recorder, trigger) + if flux, holds := trigger.(*configbutleraiv1alpha3.FluxTrigger); holds { + return r.reconcileFlux(ctx, st, flux) + } + check := r.configuredVerdict(ctx, trigger) + publishCheck(st, trigger, check) + return r.commitTrigger(ctx, st, r.triggerReadiness(trigger, check.configured, nil), nil, RequeueSteadyInterval) +} + +// reconcileFlux is Reconcile for a FluxTrigger: one being deleted, or that no eligible GitTarget +// references any more, hands its Kustomization back; any other holds it while it is Configured. +func (r *ApplyTriggerReconciler) reconcileFlux( + ctx context.Context, st *reconcileStatus, trigger *configbutleraiv1alpha3.FluxTrigger, +) (ctrl.Result, error) { + if !trigger.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(trigger, configbutleraiv1alpha3.FluxTriggerFinalizer) { + return ctrl.Result{}, nil + } + return r.retire(ctx, st, trigger, nil) + } + check := r.configuredVerdict(ctx, trigger) + publishCheck(st, trigger, check) + if check.retiring(trigger) { + return r.retire(ctx, st, trigger, &check) + } + held, err := r.holdFlux(ctx, st, trigger, check) + if err != nil { + return ctrl.Result{}, err + } + st.setValue(ApplyTriggerConditionHeld, held) + return r.commitTrigger(ctx, st, r.triggerReadiness(trigger, check.configured, &held), &held, + fluxTriggerSteadyInterval) +} + +// publishCheck sets Configured, and the source when the check was made. +func publishCheck(st *reconcileStatus, trigger configbutleraiv1alpha3.ApplyTrigger, check triggerCheck) { + st.setValue(ApplyTriggerConditionConfigured, check.configured) + if check.checked { + setTriggerSource(trigger, check.source) + } +} + +// commitTrigger writes the trio from rd and the status, and returns the cadence to read the +// trigger again on: steady, or sooner while a suspend is waited for or something may clear on its +// own. +func (r *ApplyTriggerReconciler) commitTrigger( + ctx context.Context, st *reconcileStatus, rd *readiness, held *conditionValue, steady time.Duration, +) (ctrl.Result, error) { + st.applyReadiness(rd) + if err := st.commit(ctx); err != nil { + return ctrl.Result{}, err + } + requeue := steady + switch { + case held != nil && held.Reason == ReasonSuspending: + requeue = applyTriggerSuspendingInterval + case rd.converging(): + requeue = applyTriggerSettleInterval + } + return ctrl.Result{RequeueAfter: st.requeueAfter(requeue)}, nil +} + +// holdFlux returns a FluxTrigger's Held condition. A Configured trigger holds its Kustomization, +// with its finalizer put in place before the first adoption write, and then publishes its source +// consumers: prefetch also needs the Kustomization held. One that is not Configured holds nothing, +// and one in line behind another holder lets go of a finalizer it carries, handing nothing back. +func (r *ApplyTriggerReconciler) holdFlux( + ctx context.Context, st *reconcileStatus, trigger *configbutleraiv1alpha3.FluxTrigger, check triggerCheck, +) (conditionValue, error) { + configured := check.configured + ref := trigger.Applier() + switch { + case configured.Status == metav1.ConditionTrue: + if err := r.setFinalizer(ctx, st, trigger, true); err != nil { + return conditionValue{}, err + } + held := r.holdKustomization(ctx, trigger) + if check.checked { + r.publishConsumers(ctx, trigger, held) + } + return held, nil + case configured.Reason == applier.ReasonNotReferenced: + return conditionValue{Status: metav1.ConditionFalse, Reason: applier.ReasonNotReferenced, + Message: "nothing to hold: no GitTarget names this trigger"}, nil + case configured.Reason == applier.ReasonHeldByAnotherTrigger && applier.Holding(trigger): + // The holder took it, or two adopted at once: spec.suspend is held under the same field + // manager either way, so there is nothing of this trigger's to hand back. + if err := r.setFinalizer(ctx, st, trigger, false); err != nil { + return conditionValue{}, err + } + case applier.Holding(trigger): + return conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonNotConfigured, Message: fmt.Sprintf( + "Kustomization %s is left suspended under GitOps Reverser's field manager, as this trigger last held "+ + "it: it is held again once the trigger is Configured, and handed back once no eligible GitTarget "+ + "references the trigger or the trigger is deleted", ref)}, nil + } + return conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonNotConfigured, + Message: "the Kustomization is held once the trigger is Configured"}, nil +} + +// triggerCheck is what configuredVerdict found. +type triggerCheck struct { + // configured is the Configured condition. + configured conditionValue + // source is what the applier applies, published only while the binding is satisfied. + source *applier.Source + // checked is false when the check could not be made: the published verdict and source stay. + checked bool + // refs are the GitTargets naming the trigger, judged; judged is false when they could not be. + refs applier.References + judged bool +} + +// retiring reports whether trigger holds its Kustomization and no eligible GitTarget references it +// any more, conclusively: none names it, or each is refused. A reference that cannot be judged yet +// keeps the hold, since a release is not undone: Flux applies at once. +func (c triggerCheck) retiring(trigger *configbutleraiv1alpha3.FluxTrigger) bool { + return applier.Holding(trigger) && c.judged && (!c.refs.Named() || c.refs.Refused()) +} + +// configuredVerdict judges the GitTargets naming the trigger, and, while one may be eligible, +// checks the trigger's applier in its cluster and the binding against what it read. A trigger no +// GitTarget names is NotReferenced, and reads nothing in the applier's cluster. A check that could +// not be made keeps the verdict and the source already published, since a blip reading that cluster +// says nothing about the applier; with nothing published it is Unknown. +func (r *ApplyTriggerReconciler) configuredVerdict( + ctx context.Context, trigger configbutleraiv1alpha3.ApplyTrigger, +) triggerCheck { + cluster := trigger.ClusterProvider() + ctx, cancel := context.WithTimeout(ctx, applierCheckTimeout) + defer cancel() + kept := func(message string) triggerCheck { return keptCheck(trigger, message) } + + targets, err := applier.NamingTargets(ctx, r.Client, trigger) + if err != nil { + return kept(fmt.Sprintf("Cannot list the GitTargets naming the trigger: %v", err)) + } + if len(targets) == 0 { + return triggerCheck{configured: conditionValue{ + Status: metav1.ConditionUnknown, Reason: applier.ReasonNotReferenced, + Message: "no GitTarget names this trigger; it reads and writes nothing until one does", + }, checked: true, judged: true} + } + refs, err := applier.JudgeReferences(ctx, r.Client, trigger, targets, nil) + if err != nil { + return kept(err.Error()) + } + + decision, err := authz.ApplyTriggerAdmitted(ctx, r.Client, trigger) + switch { + case err != nil: + return kept(fmt.Sprintf("Cannot check ClusterProvider %q: %v", cluster, err)) + case !decision.Allowed: + return falseCheck(decision.Reason, decision.Message, refs) + case refs.Refused(): + return falseCheck(refs.Reason(), refs.Message(), refs) + } + return r.applierVerdict(ctx, trigger, targets, refs) +} + +// applierVerdict checks the trigger's applier in its cluster, and judges the GitTargets naming the +// trigger against the source it read: refs is their judgement on admission alone. +func (r *ApplyTriggerReconciler) applierVerdict( + ctx context.Context, trigger configbutleraiv1alpha3.ApplyTrigger, targets []*configbutleraiv1alpha3.GitTarget, + refs applier.References, +) triggerCheck { + kind, ref, cluster := trigger.ApplierKind(), trigger.Applier(), trigger.ClusterProvider() + if r.ApplierClients == nil { + return triggerCheck{configured: conditionValue{ + Status: metav1.ConditionUnknown, + Reason: GitTargetReasonNotChecked, + Message: fmt.Sprintf("%s %s has not been checked: no cluster client is wired", kind, ref), + }} + } + dynamicClient, err := r.ApplierClients(ctx, cluster) + if err != nil { + return keptCheck(trigger, fmt.Sprintf("Cannot reach cluster %q to check %s %s: %v", cluster, kind, ref, err)) + } + newApplier := r.newApplier + if newApplier == nil { + newApplier = applier.NewClient + } + verdict := newApplier(kind, dynamicClient).Validate(ctx, ref) + switch { + case verdict.Err != nil: + return keptCheck(trigger, verdict.Message) + case !verdict.OK: + return falseCheck(verdict.Reason, verdict.Message, refs) + } + + refs, err = applier.JudgeReferences(ctx, r.Client, trigger, targets, verdict.Source) + switch { + case err != nil: + return keptCheck(trigger, err.Error()) + case refs.Eligible(): + return r.heldVerdict(ctx, trigger, verdict, refs) + case refs.Refused(): + return falseCheck(refs.Reason(), refs.Message(), refs) + } + return triggerCheck{configured: conditionValue{ + Status: metav1.ConditionUnknown, Reason: applier.ReasonNotBound, Message: refs.Message(), + }, checked: true, refs: refs, judged: true} +} + +// keptCheck is a check that could not be made: the published verdict and source stay. +func keptCheck(trigger configbutleraiv1alpha3.ApplyTrigger, message string) triggerCheck { + return triggerCheck{configured: keptCondition(*trigger.StatusConditions(), ApplyTriggerConditionConfigured, + message)} +} + +// falseCheck is Configured=False for reason, with the GitTargets naming the trigger as judged. +func falseCheck(reason, message string, refs applier.References) triggerCheck { + return triggerCheck{configured: conditionValue{Status: metav1.ConditionFalse, Reason: reason, Message: message}, + checked: true, refs: refs, judged: true} +} + +// heldVerdict is Configured for a trigger whose applier passed every other check and an eligible +// GitTarget references: True when this trigger holds it, False (HeldByAnotherTrigger) when another +// trigger in line comes first. Checked last, so a trigger in line for an applier is one that can +// take it over as it is. +func (r *ApplyTriggerReconciler) heldVerdict( + ctx context.Context, trigger configbutleraiv1alpha3.ApplyTrigger, verdict applier.Verdict, + refs applier.References, +) triggerCheck { + // The triggers in line are judged against the source just read, which they share. + setTriggerSource(trigger, verdict.Source) + holder, err := applier.HeldByAnother(ctx, r.Client, trigger) + switch { + case err != nil: + return triggerCheck{configured: keptCondition(*trigger.StatusConditions(), ApplyTriggerConditionConfigured, + fmt.Sprintf("Cannot check which trigger holds %s %s: %v", trigger.ApplierKind(), trigger.Applier(), err)), + source: verdict.Source, checked: true, refs: refs, judged: true} + case holder != nil: + return triggerCheck{configured: conditionValue{ + Status: metav1.ConditionFalse, Reason: applier.ReasonHeldByAnotherTrigger, + Message: applier.HeldByAnotherMessage(trigger, holder), + }, source: verdict.Source, checked: true, refs: refs, judged: true} + } + return triggerCheck{ + configured: conditionValue{Status: metav1.ConditionTrue, Reason: ReasonSucceeded, Message: verdict.Message}, + source: verdict.Source, checked: true, refs: refs, judged: true, + } +} + +// setTriggerSource publishes what the applier applies, or clears it when the check read nothing. +func setTriggerSource(trigger configbutleraiv1alpha3.ApplyTrigger, source *applier.Source) { + switch t := trigger.(type) { + case *configbutleraiv1alpha3.FluxTrigger: + t.Status.Source = nil + if source != nil { + t.Status.Source = &configbutleraiv1alpha3.FluxTriggerSource{ + GitRepository: meta.NamespacedObjectReference{ + Namespace: source.GitRepository.Namespace, Name: source.GitRepository.Name, + }, + URL: source.URL, + Branch: source.Revision, + } + } + case *configbutleraiv1alpha3.ArgoCDTrigger: + t.Status.Source = nil + if source != nil { + t.Status.Source = &configbutleraiv1alpha3.ArgoCDTriggerSource{ + RepoURL: source.URL, TargetRevision: source.Revision, + } + } + } +} + +// triggerSourceSummary names what an applier applies, as the trigger publishes it, for the Ready +// message. +func triggerSourceSummary(trigger configbutleraiv1alpha3.ApplyTrigger) string { + url, revision, known := trigger.SourceBinding() + if !known { + return "" + } + return fmt.Sprintf("(revision %q of %s)", revision, url) +} + +// triggerReadiness is the trio's accumulator for a trigger, its gates contributed. +func (r *ApplyTriggerReconciler) triggerReadiness( + trigger configbutleraiv1alpha3.ApplyTrigger, configured conditionValue, held *conditionValue, +) *readiness { + rd := newReadiness(fmt.Sprintf("%s %s %s can be asked to apply", trigger.ApplierKind(), trigger.Applier(), + triggerSourceSummary(trigger)), trigger.TriggerKind()+" is not stalled") + applyTriggerReadinessGates(rd, configured, held, trigger.Operation()) + return rd +} + +// applyTriggerReadinessGates contributes Configured, the operation, and a FluxTrigger's Held +// to the trio, in that order. +// An applier that is missing, or a ClusterProvider that is, may be on its way (Flux and Argo CD +// objects are often created after the trigger); anything else Configured=False needs a human. An +// operation that has not completed in time clears on its own once it does. A trigger no GitTarget +// names is Ready (NotReferenced): it does all it is asked to, which is nothing, and a Flux health +// check that waits on it must not wait for the GitTarget that will name it. +func applyTriggerReadinessGates( + rd *readiness, configured conditionValue, held *conditionValue, operation *configbutleraiv1alpha3.ApplyOperation, +) { + if configured.Reason == applier.ReasonNotReferenced { + rd.convergesAs(conditionValue{Status: metav1.ConditionTrue, Reason: applier.ReasonNotReferenced, + Message: "no GitTarget names this trigger, so it does nothing"}) + } + waits := configured.Reason == applier.ReasonApplierNotFound || + configured.Reason == authz.ReasonClusterProviderNotFound + rd.stalledIf(configured.Status == metav1.ConditionFalse && !waits, configured.Reason, configured.Message) + rd.progressingIf(configured.Status == metav1.ConditionFalse, metav1.ConditionFalse, + configured.Reason, configured.Message) + rd.progressingIf(configured.Status == metav1.ConditionUnknown && configured.Reason != applier.ReasonNotReferenced, + metav1.ConditionUnknown, configured.Reason, configured.Message) + if operation != nil && operation.Phase == configbutleraiv1alpha3.ApplyOperationUnresolved { + what := fmt.Sprintf("the %s apply of %s, asked for at", operation.Reason, + applier.DescribeCommit(operation.Commit)) + if operation.Reason == "" { + what = "an apply GitOps Reverser did not start, seen at" + } + rd.progressing(metav1.ConditionFalse, ReasonOperationUnresolved, fmt.Sprintf( + "%s %s, has not completed: %s", what, operation.RequestedAt.UTC().Format(time.RFC3339), operation.Message)) + } + if held != nil { + rd.progressingIf(held.Status != metav1.ConditionTrue && held.Reason != applier.ReasonNotReferenced, + held.Status, held.Reason, held.Message) + } +} + +// SetupWithManager sets up the controller for its kind with the Manager. +func (r *ApplyTriggerReconciler) SetupWithManager(mgr ctrl.Manager) error { + trigger := configbutleraiv1alpha3.NewApplyTrigger(r.Kind) + if trigger == nil { + return fmt.Errorf("no apply trigger kind %q", r.Kind) + } + return ctrl.NewControllerManagedBy(mgr). + // The status is written by two parties: this reconciler (conditions, source) and the branch + // workers (operation, repairPending). A status-only update is filtered, as everywhere, except + // one that moves the operation's phase, which Ready reads. + For(trigger, builder.WithPredicates(applyTriggerSpecOrOperationChanged())). + // What makes a GitTarget eligible re-runs the check of the trigger it names, and of the one it + // named before: the GitTarget appearing, going, or changing what it names or writes. + Watches(&configbutleraiv1alpha3.GitTarget{}, r.gitTargetToTriggers()). + // A GitProvider's URL is what its GitTargets write. + Watches( + &configbutleraiv1alpha3.GitProvider{}, + handler.EnqueueRequestsFromMapFunc(r.gitProviderToTriggers), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). + // A ClusterProvider admits the trigger's namespace, and the namespaces of the GitTargets naming + // triggers, so a provider created, deleted, or with its spec changed re-runs every check. + Watches( + &configbutleraiv1alpha3.ClusterProvider{}, + handler.EnqueueRequestsFromMapFunc(r.clusterProviderToTriggers), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). + // An accessFrom selector reads namespace labels. + Watches( + &corev1.Namespace{}, + handler.EnqueueRequestsFromMapFunc(r.namespaceToTriggers), + builder.WithPredicates(predicate.LabelChangedPredicate{}), + ). + // One trigger holds an applier; the others naming it wait in line. A holder that is gone, or + // lets go, and a trigger that joins or leaves the line, re-run the others' check: nothing else + // changes on them to reconcile them. + Watches( + trigger, + handler.EnqueueRequestsFromMapFunc(r.triggersInLine), + builder.WithPredicates(triggerLineChanged()), + ). + Named(r.Kind). + Complete(r) +} + +// applyTriggerSpecOrOperationChanged fires on create and delete, on a spec change, on a change of +// the reconcile-request annotation, and on a change of status.operation.phase. +func applyTriggerSpecOrOperationChanged() predicate.Predicate { + requested := reconcileRequestedOrSpecChanged() + return predicate.Funcs{ + CreateFunc: requested.Create, + DeleteFunc: requested.Delete, + GenericFunc: requested.Generic, + UpdateFunc: func(e event.UpdateEvent) bool { + if requested.Update(e) { + return true + } + before, ok1 := e.ObjectOld.(configbutleraiv1alpha3.ApplyTrigger) + after, ok2 := e.ObjectNew.(configbutleraiv1alpha3.ApplyTrigger) + return !ok1 || !ok2 || operationPhase(before) != operationPhase(after) + }, + } +} + +// operationPhase is a trigger's status.operation.phase, or "" without one. +func operationPhase(trigger configbutleraiv1alpha3.ApplyTrigger) configbutleraiv1alpha3.ApplyOperationPhase { + if operation := trigger.Operation(); operation != nil { + return operation.Phase + } + return "" +} + +// listTriggers lists this reconciler's kind, in namespace or everywhere when it is empty. +func (r *ApplyTriggerReconciler) listTriggers( + ctx context.Context, namespace string, +) ([]configbutleraiv1alpha3.ApplyTrigger, error) { + var opts []client.ListOption + if namespace != "" { + opts = append(opts, client.InNamespace(namespace)) + } + return applier.ListApplyTriggers(ctx, r.Client, r.Kind, opts...) +} + +// triggerLineChanged passes the event of a trigger that is gone, and an update that can move it in +// or out of line for its applier: its finalizer, its Configured condition, or its Held one. +func triggerLineChanged() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + before, ok1 := e.ObjectOld.(configbutleraiv1alpha3.ApplyTrigger) + after, ok2 := e.ObjectNew.(configbutleraiv1alpha3.ApplyTrigger) + return !ok1 || !ok2 || lineSummary(before) != lineSummary(after) + }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, + } +} + +// lineSummary is what of a trigger decides its place in line for its applier. +func lineSummary(trigger configbutleraiv1alpha3.ApplyTrigger) string { + var summary strings.Builder + summary.WriteString(strconv.FormatBool(applier.Holding(trigger))) + for _, conditionType := range []string{ApplyTriggerConditionConfigured, ApplyTriggerConditionHeld} { + if cond := findCondition(*trigger.StatusConditions(), conditionType); cond != nil { + summary.WriteString("|" + string(cond.Status) + "/" + cond.Reason) + } + } + return summary.String() +} + +// gitTargetToTriggers enqueues the triggers of this kind a GitTarget names, before and after an +// update, when it appears, goes, or changes anything eligibility reads: its spec, or its deletion. +func (r *ApplyTriggerReconciler) gitTargetToTriggers() handler.EventHandler { + named := func(obj client.Object) []ctrlreconcile.Request { + target, ok := obj.(*configbutleraiv1alpha3.GitTarget) + if !ok || target.Spec.ApplyTriggerRef == nil || target.Spec.ApplyTriggerRef.Kind != r.Kind { + return nil + } + return []ctrlreconcile.Request{{NamespacedName: k8stypes.NamespacedName{ + Namespace: target.Namespace, Name: target.Spec.ApplyTriggerRef.Name, + }}} + } + add := func(q workqueue.TypedRateLimitingInterface[ctrlreconcile.Request], objs ...client.Object) { + for _, obj := range objs { + for _, request := range named(obj) { + q.Add(request) + } + } + } + return handler.Funcs{ + CreateFunc: func(_ context.Context, e event.CreateEvent, + q workqueue.TypedRateLimitingInterface[ctrlreconcile.Request]) { + add(q, e.Object) + }, + UpdateFunc: func(_ context.Context, e event.UpdateEvent, + q workqueue.TypedRateLimitingInterface[ctrlreconcile.Request]) { + if e.ObjectOld.GetGeneration() == e.ObjectNew.GetGeneration() && + e.ObjectOld.GetDeletionTimestamp().IsZero() == e.ObjectNew.GetDeletionTimestamp().IsZero() { + return + } + add(q, e.ObjectOld, e.ObjectNew) + }, + DeleteFunc: func(_ context.Context, e event.DeleteEvent, + q workqueue.TypedRateLimitingInterface[ctrlreconcile.Request]) { + add(q, e.Object) + }, + } +} + +// gitProviderToTriggers maps a GitProvider event to the triggers of this kind named by the +// GitTargets that write through it. +func (r *ApplyTriggerReconciler) gitProviderToTriggers(ctx context.Context, obj client.Object) []ctrlreconcile.Request { + var targets configbutleraiv1alpha3.GitTargetList + if err := r.List(ctx, &targets, client.InNamespace(obj.GetNamespace())); err != nil { + logDependencyListError(ctx, err, "GitTargets", obj) + return nil + } + seen := map[k8stypes.NamespacedName]bool{} + var requests []ctrlreconcile.Request + for i := range targets.Items { + target := &targets.Items[i] + ref := target.Spec.ApplyTriggerRef + if ref == nil || ref.Kind != r.Kind || target.Spec.GitProviderRef.Name != obj.GetName() { + continue + } + key := k8stypes.NamespacedName{Namespace: target.Namespace, Name: ref.Name} + if !seen[key] { + seen[key] = true + requests = append(requests, ctrlreconcile.Request{NamespacedName: key}) + } + } + return requests +} + +// triggersInLine maps a trigger that is gone, or that moved in line, to the other triggers of this +// kind naming its applier, so the next in line takes it over. +func (r *ApplyTriggerReconciler) triggersInLine(ctx context.Context, obj client.Object) []ctrlreconcile.Request { + gone, ok := obj.(configbutleraiv1alpha3.ApplyTrigger) + if !ok || gone.TriggerKind() != r.Kind { + return nil + } + others, err := applier.TriggersOfApplier(ctx, r.Client, gone) + if err != nil { + logDependencyListError(ctx, err, r.Kind+"s", obj) + return nil + } + requests := make([]ctrlreconcile.Request, 0, len(others)) + for _, other := range others { + requests = append(requests, triggerRequest(other)) + } + return requests +} + +// clusterProviderToTriggers maps a ClusterProvider event to every trigger of this kind: a provider +// admits the triggers that name it, and the GitTargets that mirror through it, which may name any +// trigger in their namespace. +func (r *ApplyTriggerReconciler) clusterProviderToTriggers( + ctx context.Context, obj client.Object, +) []ctrlreconcile.Request { + triggers, err := r.listTriggers(ctx, "") + if err != nil { + logDependencyListError(ctx, err, r.Kind+"s", obj) + return nil + } + requests := make([]ctrlreconcile.Request, 0, len(triggers)) + for _, trigger := range triggers { + requests = append(requests, triggerRequest(trigger)) + } + return requests +} + +// namespaceToTriggers maps a Namespace label change to every trigger of this kind in it. +func (r *ApplyTriggerReconciler) namespaceToTriggers(ctx context.Context, obj client.Object) []ctrlreconcile.Request { + triggers, err := r.listTriggers(ctx, obj.GetName()) + if err != nil { + logDependencyListError(ctx, err, r.Kind+"s", obj) + return nil + } + requests := make([]ctrlreconcile.Request, 0, len(triggers)) + for _, trigger := range triggers { + requests = append(requests, triggerRequest(trigger)) + } + return requests +} + +func triggerRequest(trigger configbutleraiv1alpha3.ApplyTrigger) ctrlreconcile.Request { + return ctrlreconcile.Request{NamespacedName: k8stypes.NamespacedName{ + Namespace: trigger.GetNamespace(), Name: trigger.GetName(), + }} +} diff --git a/internal/controller/applytrigger_controller_test.go b/internal/controller/applytrigger_controller_test.go new file mode 100644 index 000000000..47713fe29 --- /dev/null +++ b/internal/controller/applytrigger_controller_test.go @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" + "github.com/ConfigButler/gitops-reverser/internal/authz" +) + +const applierTestRepo = "https://github.com/acme/platform.git" + +// applierFluxObjects is a Kustomization flux-system/checkout applying a GitRepository on main. +func applierFluxObjects() []runtime.Object { + ks := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kustomize.toolkit.fluxcd.io/v1", + "kind": "Kustomization", + "metadata": map[string]any{"namespace": "flux-system", "name": "checkout"}, + "spec": map[string]any{ + "path": "./apps", + "sourceRef": map[string]any{"kind": "GitRepository", "name": "platform"}, + }, + }} + repo := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "GitRepository", + "metadata": map[string]any{"namespace": "flux-system", "name": "platform"}, + "spec": map[string]any{ + "url": "https://github.com/acme/platform", + "ref": map[string]any{"branch": "main"}, + }, + }} + return []runtime.Object{ks, repo} +} + +// applierArgoApplication is an Application argocd/checkout syncing main of the test repository, +// with a syncPolicy when one is given. +func applierArgoApplication(syncPolicy map[string]any) runtime.Object { + app := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "argoproj.io/v1alpha1", + "kind": "Application", + "metadata": map[string]any{"namespace": "argocd", "name": "checkout"}, + "spec": map[string]any{ + "source": map[string]any{ + "repoURL": "https://github.com/acme/platform", "targetRevision": "main", "path": "apps", + }, + }, + }} + if syncPolicy != nil { + _ = unstructured.SetNestedMap(app.Object, syncPolicy, "spec", "syncPolicy") + } + return app +} + +// applierFakeSource is an applier cluster holding objs, with Flux's controllers simulated in it. +func applierFakeSource(t *testing.T, objs ...runtime.Object) *dynamicfake.FakeDynamicClient { + t.Helper() + return applierSim(t, objs...).Client +} + +// applierSim is an applier cluster holding objs, with Flux's controllers simulated in it. +func applierSim(t *testing.T, objs ...runtime.Object) *fluxsim.Sim { + t.Helper() + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + fluxsim.KustomizationGVR(): "KustomizationList", + fluxsim.GitRepositoryGVR(): "GitRepositoryList", + {Group: "helm.toolkit.fluxcd.io", Version: "v2", Resource: "helmreleases"}: "HelmReleaseList", + {Group: "argoproj.io", Version: "v1alpha1", Resource: "applications"}: "ApplicationList", + }, objs...) + return fluxsim.New(t, client) +} + +// admitting is a ClusterProvider whose accessFrom admits namespaces, and which lets a trigger name +// an applier in another namespace (the test appliers live in flux-system and argocd). +func admitting(name string, namespaces ...string) *configbutleraiv1alpha3.ClusterProvider { + return &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{Names: namespaces}, + AllowCrossNamespace: &configbutleraiv1alpha3.ClusterProviderCrossNamespace{Appliers: true}, + }, + } +} + +// testFluxTrigger is the FluxTrigger team-a/checkout holding flux-system/checkout in cluster. +func testFluxTrigger(cluster string) *configbutleraiv1alpha3.FluxTrigger { + return &configbutleraiv1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: "checkout", Namespace: "team-a", Generation: 1}, + Spec: configbutleraiv1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: cluster}, + KustomizationRef: meta.NamespacedObjectReference{Namespace: "flux-system", Name: "checkout"}, + }, + } +} + +// testArgoCDTrigger is the ArgoCDTrigger team-a/checkout holding argocd/checkout in cluster. +func testArgoCDTrigger(cluster string) *configbutleraiv1alpha3.ArgoCDTrigger { + return &configbutleraiv1alpha3.ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: "checkout", Namespace: "team-a", Generation: 1}, + Spec: configbutleraiv1alpha3.ArgoCDTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: cluster}, + ApplicationRef: meta.NamespacedObjectReference{Namespace: "argocd", Name: "checkout"}, + }, + } +} + +// referencingTarget is a GitTarget team-a/name writing main of the test repository through the +// GitProvider platform, naming the trigger kind/checkout. +func referencingTarget(name, kind string) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "team-a"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "platform"}, + Branch: "main", + Path: name, + ApplyTriggerRef: &configbutleraiv1alpha3.ApplyTriggerReference{Kind: kind, Name: "checkout"}, + }, + } +} + +// platformProvider is the GitProvider team-a/platform of the test repository. +func platformProvider() *configbutleraiv1alpha3.GitProvider { + return &configbutleraiv1alpha3.GitProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "platform", Namespace: "team-a"}, + Spec: configbutleraiv1alpha3.GitProviderSpec{URL: applierTestRepo}, + } +} + +// referencedIn is a GitProvider platform and a GitTarget checkout- naming the trigger +// kind/checkout, both in namespace: what makes a trigger there eligible. +func referencedIn(namespace, kind string) []ctrlclient.Object { + provider := platformProvider() + provider.Namespace = namespace + target := referencingTarget("checkout-"+strings.ToLower(kind), kind) + target.Namespace = namespace + return []ctrlclient.Object{provider, target} +} + +// unreferenced marks a triggerTestClient that holds no GitTarget naming the triggers. +type unreferenced struct{ ctrlclient.Object } + +// triggerTestClient is a control cluster holding objects, the ClusterProviders "default" and "hub" +// admitting team-a unless objects carries its own, the GitProvider platform unless objects carries a +// GitProvider, and a GitTarget naming each trigger kind's checkout unless objects carries a +// GitTarget or unreferenced{}. +func triggerTestClient(t *testing.T, objects ...ctrlclient.Object) ctrlclient.Client { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, configbutleraiv1alpha3.AddToScheme(scheme)) + held := map[string]bool{} + hasTarget, hasProvider := false, false + kept := objects[:0:0] + for _, obj := range objects { + switch obj.(type) { + case *configbutleraiv1alpha3.ClusterProvider: + held[obj.GetName()] = true + case *configbutleraiv1alpha3.GitTarget, unreferenced: + hasTarget = true + case *configbutleraiv1alpha3.GitProvider: + hasProvider = true + } + if _, marker := obj.(unreferenced); !marker { + kept = append(kept, obj) + } + } + objects = kept + for _, name := range []string{configbutleraiv1alpha3.DefaultClusterProviderName, "hub"} { + if !held[name] { + objects = append(objects, admitting(name, "team-a")) + } + } + if !hasProvider { + objects = append(objects, platformProvider()) + } + if !hasTarget { + objects = append(objects, + referencingTarget("checkout-flux", configbutleraiv1alpha3.ApplyTriggerKindFlux), + referencingTarget("checkout-argo", configbutleraiv1alpha3.ApplyTriggerKindArgoCD)) + } + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithStatusSubresource(&configbutleraiv1alpha3.FluxTrigger{}, &configbutleraiv1alpha3.ArgoCDTrigger{}). + Build() +} + +// triggerReconciler checks appliers against a fake cluster whose access review answers allowed. +func triggerReconciler( + c ctrlclient.Client, kind string, source dynamic.Interface, sourceErr error, allowed bool, +) *ApplyTriggerReconciler { + return &ApplyTriggerReconciler{ + Client: c, + Kind: kind, + ApplierClients: func(context.Context, string) (dynamic.Interface, error) { return source, sourceErr }, + newApplier: func(kind string, client dynamic.Interface) applier.Client { + canI := func(context.Context, authorizationv1.ResourceAttributes) (bool, error) { return allowed, nil } + if kind == configbutleraiv1alpha3.ApplierKindApplication { + argo := applier.NewArgoCD(client) + argo.CanI = canI + return argo + } + flux := applier.NewFlux(client) + flux.CanI = canI + return flux + }, + } +} + +// triggerKey is the trigger every test reconciles. +var triggerKey = k8stypes.NamespacedName{Namespace: "team-a", Name: "checkout"} + +// reconcileTrigger reconciles team-a/checkout until it settles and returns the trigger as it is +// then. While its Kustomization's suspend is being handled, a FluxTrigger is read again at the +// suspending interval. Every pass publishes its conditions: the one that adopts too, which also adds +// the finalizer, since nothing re-enqueues a trigger for a metadata-only update. A FluxTrigger +// carries the finalizer exactly while it is Configured: it holds its Kustomization. +func reconcileTrigger(t *testing.T, r *ApplyTriggerReconciler) configbutleraiv1alpha3.ApplyTrigger { + t.Helper() + var trigger configbutleraiv1alpha3.ApplyTrigger + require.Eventually(t, func() bool { + result, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + trigger = configbutleraiv1alpha3.NewApplyTrigger(r.Kind) + require.NoError(t, r.Get(context.Background(), triggerKey, trigger)) + require.NotNil(t, findCondition(*trigger.StatusConditions(), ConditionTypeReady), + "every pass publishes Ready") + flux, holds := trigger.(*configbutleraiv1alpha3.FluxTrigger) + if holds && conditionIsTrue(flux.Status.Conditions, ApplyTriggerConditionConfigured) { + require.True(t, controllerutil.ContainsFinalizer(flux, configbutleraiv1alpha3.FluxTriggerFinalizer), + "a trigger that holds its Kustomization carries the finalizer") + } + if held := findCondition(*trigger.StatusConditions(), ApplyTriggerConditionHeld); held != nil && + held.Reason == ReasonSuspending { + assert.Equal(t, applyTriggerSuspendingInterval, result.RequeueAfter, "a suspend is waited for closely") + return false + } + want := RequeueSteadyInterval + if holds { + want = fluxTriggerSteadyInterval + } + if !conditionIsTrue(*trigger.StatusConditions(), ConditionTypeReady) && + !conditionIsTrue(*trigger.StatusConditions(), ConditionTypeStalled) { + want = applyTriggerSettleInterval + } + assert.Equal(t, want, result.RequeueAfter, "a trigger that may clear on its own is read again soon") + return true + }, 5*time.Second, 5*time.Millisecond) + return trigger +} + +func TestApplyTriggerReconcile_FluxVerdicts(t *testing.T) { + cases := []struct { + name string + objects []runtime.Object + allowed bool + wantStatus metav1.ConditionStatus + wantReason string + wantStalled bool + wantSource *configbutleraiv1alpha3.FluxTriggerSource + }{ + {name: "a Kustomization applying a GitRepository on a branch", objects: applierFluxObjects(), + allowed: true, wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, + wantSource: &configbutleraiv1alpha3.FluxTriggerSource{ + GitRepository: meta.NamespacedObjectReference{Namespace: "flux-system", Name: "platform"}, + URL: "https://github.com/acme/platform", Branch: "main", Prefetch: true, + }}, + {name: "no such Kustomization, which may be on its way", allowed: true, + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonApplierNotFound}, + {name: "an identity that may not patch it, which publishes no source", objects: applierFluxObjects(), + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonForbidden, wantStalled: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := triggerTestClient(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, + applierFakeSource(t, tc.objects...), nil, tc.allowed) + + trigger := reconcileTrigger(t, r).(*configbutleraiv1alpha3.FluxTrigger) + + configured := findCondition(trigger.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, tc.wantStatus, configured.Status, configured.Message) + assert.Equal(t, tc.wantReason, configured.Reason) + ready := findCondition(trigger.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, tc.wantStatus, ready.Status) + assert.Equal(t, tc.wantStalled, conditionIsTrue(trigger.Status.Conditions, ConditionTypeStalled)) + assert.Equal(t, tc.wantSource, trigger.Status.Source) + assert.Equal(t, int64(1), trigger.Status.ObservedGeneration) + }) + } +} + +// TestApplyTriggerReconcile_ArgoCDVerdicts: an ArgoCDTrigger checks an Application, with automated +// sync on as a verdict of its own. +func TestApplyTriggerReconcile_ArgoCDVerdicts(t *testing.T) { + cases := []struct { + name string + objects []runtime.Object + allowed bool + wantStatus metav1.ConditionStatus + wantReason string + wantStalled bool + }{ + {name: "automated sync off", objects: []runtime.Object{applierArgoApplication(nil)}, allowed: true, + wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded}, + {name: "automated sync switched off, keeping prune", allowed: true, + objects: []runtime.Object{applierArgoApplication(map[string]any{ + "automated": map[string]any{"enabled": false, "prune": true}, + })}, + wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded}, + {name: "automated sync on", allowed: true, + objects: []runtime.Object{applierArgoApplication(map[string]any{ + "automated": map[string]any{"prune": true}, + })}, + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonAutomatedSyncEnabled, wantStalled: true}, + {name: "no such Application", allowed: true, + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonApplierNotFound}, + {name: "an identity that may not update it", objects: []runtime.Object{applierArgoApplication(nil)}, + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonForbidden, wantStalled: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := triggerTestClient(t, testArgoCDTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindArgoCD, + applierFakeSource(t, tc.objects...), nil, tc.allowed) + + trigger := reconcileTrigger(t, r).(*configbutleraiv1alpha3.ArgoCDTrigger) + + configured := findCondition(trigger.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, tc.wantStatus, configured.Status, configured.Message) + assert.Equal(t, tc.wantReason, configured.Reason) + assert.Equal(t, tc.wantStalled, conditionIsTrue(trigger.Status.Conditions, ConditionTypeStalled)) + if tc.wantStatus == metav1.ConditionTrue { + assert.Equal(t, &configbutleraiv1alpha3.ArgoCDTriggerSource{ + RepoURL: "https://github.com/acme/platform", TargetRevision: "main", + }, trigger.Status.Source) + } + }) + } +} + +// TestApplyTriggerReconcile_AnUnreachableClusterKeepsThePublishedVerdict: a blip reading the +// applier's cluster says nothing about the applier, so it must not flip a True to Unknown and back on +// every hiccup, nor drop the source the GitTargets bind against. +func TestApplyTriggerReconcile_AnUnreachableClusterKeepsThePublishedVerdict(t *testing.T) { + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + c := triggerTestClient(t, trigger) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, nil, errors.New("connection refused"), true) + got := reconcileTrigger(t, r) + configured := findCondition(*got.StatusConditions(), ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, metav1.ConditionUnknown, configured.Status, "nothing published yet") + + published := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + published.Status.Conditions = []metav1.Condition{{ + Type: ApplyTriggerConditionConfigured, Status: metav1.ConditionTrue, Reason: ReasonSucceeded, + LastTransitionTime: metav1.Now(), + }} + published.Status.Source = &configbutleraiv1alpha3.FluxTriggerSource{URL: applierTestRepo, Branch: "main"} + c = triggerTestClient(t, published) + r = triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, nil, errors.New("connection refused"), true) + got = reconcileTrigger(t, r) + configured = findCondition(*got.StatusConditions(), ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, metav1.ConditionTrue, configured.Status, "the published verdict stands") + url, branch, known := got.SourceBinding() + assert.True(t, known, "and so does the source") + assert.Equal(t, applierTestRepo, url) + assert.Equal(t, "main", branch) +} + +// TestApplyTriggerReconcile_AHangingClusterIsBounded: the applier's client has no request timeout, +// so a cluster that accepts the connection and never answers would hold the reconcile for as long as +// it liked. The check gives up, and a check that could not be made keeps the published verdict. +func TestApplyTriggerReconcile_AHangingClusterIsBounded(t *testing.T) { + var sawDeadline bool + source := applierFakeSource(t, applierFluxObjects()...) + published := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + published.Status.Conditions = []metav1.Condition{{ + Type: ApplyTriggerConditionConfigured, Status: metav1.ConditionTrue, Reason: ReasonSucceeded, + LastTransitionTime: metav1.Now(), + }} + r := &ApplyTriggerReconciler{ + Client: triggerTestClient(t, published), + Kind: configbutleraiv1alpha3.ApplyTriggerKindFlux, + ApplierClients: func(ctx context.Context, _ string) (dynamic.Interface, error) { + _, sawDeadline = ctx.Deadline() + return source, nil + }, + newApplier: func(_ string, client dynamic.Interface) applier.Client { + flux := applier.NewFlux(client) + flux.CanI = func(ctx context.Context, _ authorizationv1.ResourceAttributes) (bool, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok, "the access review must run under a deadline") + assert.LessOrEqual(t, time.Until(deadline), applierCheckTimeout) + return false, context.DeadlineExceeded // what a server that never answers ends in + } + return flux + }, + } + + got := reconcileTrigger(t, r) + + assert.True(t, sawDeadline, "the client lookup must run under a deadline") + configured := findCondition(*got.StatusConditions(), ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, metav1.ConditionTrue, configured.Status, "a timed-out check keeps the published verdict") +} + +// TestApplyTriggerReconcile_TheTriggersClusterProviderAdmitsIt: the applier is reached with the +// trigger's ClusterProvider's credential, so that provider's accessFrom gates the check before the +// cluster is ever asked. +func TestApplyTriggerReconcile_TheTriggersClusterProviderAdmitsIt(t *testing.T) { + cases := []struct { + name string + cluster string + hub *configbutleraiv1alpha3.ClusterProvider + wantStatus metav1.ConditionStatus + wantReason string + wantStalled bool + wantClients []string + }{ + {name: "a provider that admits the namespace is asked", cluster: "hub", hub: admitting("hub", "team-a"), + wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, wantClients: []string{"hub"}}, + {name: "a provider that does not admit it is never asked", cluster: "hub", hub: admitting("hub", "team-b"), + wantStatus: metav1.ConditionFalse, wantReason: authz.ReasonNamespaceNotAuthorized, wantStalled: true}, + {name: "a provider that does not exist is never asked, and may be on its way", cluster: "absent", + wantStatus: metav1.ConditionFalse, wantReason: authz.ReasonClusterProviderNotFound}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + objects := []ctrlclient.Object{testFluxTrigger(tc.cluster)} + if tc.hub != nil { + objects = append(objects, tc.hub) + } + var asked []string + r := triggerReconciler(triggerTestClient(t, objects...), configbutleraiv1alpha3.ApplyTriggerKindFlux, + applierFakeSource(t, applierFluxObjects()...), nil, true) + source := r.ApplierClients + r.ApplierClients = func(ctx context.Context, cluster string) (dynamic.Interface, error) { + asked = append(asked, cluster) + return source(ctx, cluster) + } + + got := reconcileTrigger(t, r) + + configured := findCondition(*got.StatusConditions(), ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, tc.wantStatus, configured.Status, configured.Message) + assert.Equal(t, tc.wantReason, configured.Reason) + assert.Equal(t, tc.wantStalled, conditionIsTrue(*got.StatusConditions(), ConditionTypeStalled)) + assert.Equal(t, tc.wantClients, slices.Compact(asked), "only the trigger's own cluster is asked") + }) + } +} + +// TestApplyTriggerReconcile_AnUnresolvedOperationIsNotReady is the third review's fourth finding, +// seen from the trigger: an operation past its budget keeps its window open, and the trigger says so +// in Ready until the operation completes. It clears on its own, so it is never Stalled. +func TestApplyTriggerReconcile_AnUnresolvedOperationIsNotReady(t *testing.T) { + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + trigger.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationUnresolved, Reason: configbutleraiv1alpha3.ApplyReasonCatchUp, + Commit: "4f2c1ab9d00d", Token: "T1", RequestedAt: metav1.Now(), Message: "still applying", + } + c := triggerTestClient(t, trigger) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, + applierFakeSource(t, applierFluxObjects()...), nil, true) + + got := reconcileTrigger(t, r) + + ready := findCondition(*got.StatusConditions(), ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, ReasonOperationUnresolved, ready.Reason) + assert.Contains(t, ready.Message, "the CatchUp apply of 4f2c1ab9") + assert.False(t, conditionIsTrue(*got.StatusConditions(), ConditionTypeStalled)) + require.NotNil(t, got.Operation(), "the operation is the worker's, and left as it is") + assert.Equal(t, configbutleraiv1alpha3.ApplyOperationUnresolved, got.Operation().Phase) +} + +// TestApplyTriggerSpecOrOperationChanged_ReadsTheOperationPhase: the worker's status writes reach the +// reconciler only when they move the phase Ready reads. +func TestApplyTriggerSpecOrOperationChanged_ReadsTheOperationPhase(t *testing.T) { + p := applyTriggerSpecOrOperationChanged() + at := func(phase configbutleraiv1alpha3.ApplyOperationPhase, generation int64) ctrlclient.Object { + trigger := testFluxTrigger("default") + trigger.Generation = generation + if phase != "" { + trigger.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{Phase: phase} + } + return trigger + } + assert.True(t, p.Update(event.UpdateEvent{ + ObjectOld: at(configbutleraiv1alpha3.ApplyOperationRunning, 1), + ObjectNew: at(configbutleraiv1alpha3.ApplyOperationUnresolved, 1), + })) + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: at("", 1), ObjectNew: at("", 2)}), "a spec change") + assert.False(t, p.Update(event.UpdateEvent{ + ObjectOld: at(configbutleraiv1alpha3.ApplyOperationRunning, 1), + ObjectNew: at(configbutleraiv1alpha3.ApplyOperationRunning, 1), + }), "a status write that moves nothing Ready reads") +} + +// TestApplyTriggerReconcile_ClusterProviderAndNamespaceEventsFindTheirTriggers: a provider event +// re-runs every trigger, since it admits the GitTargets naming triggers too, and a namespace label +// change the triggers in it. +func TestApplyTriggerReconcile_ClusterProviderAndNamespaceEventsFindTheirTriggers(t *testing.T) { + other := testFluxTrigger("other") + other.Name = "elsewhere" + c := triggerTestClient(t, testFluxTrigger("hub"), other) + r := &ApplyTriggerReconciler{Client: c, Kind: configbutleraiv1alpha3.ApplyTriggerKindFlux} + + assert.Len(t, r.clusterProviderToTriggers(context.Background(), admitting("hub")), 2) + + ns := &metav1.PartialObjectMetadata{ObjectMeta: metav1.ObjectMeta{Name: "team-a"}} + assert.Len(t, r.namespaceToTriggers(context.Background(), ns), 2) +} + +// TestApplyTriggerReconcile_ArgoCDEventsFindTheirTriggers: the Argo CD kind lists its own. +func TestApplyTriggerReconcile_ArgoCDEventsFindTheirTriggers(t *testing.T) { + c := triggerTestClient(t, testArgoCDTrigger("hub"), testFluxTrigger("hub")) + r := &ApplyTriggerReconciler{Client: c, Kind: configbutleraiv1alpha3.ApplyTriggerKindArgoCD} + requests := r.clusterProviderToTriggers(context.Background(), admitting("hub")) + assert.Len(t, requests, 1, "a FluxTrigger of the same name is another kind's") +} + +// TestApplyTriggerReconcile_AGoneTriggerIsForgotten: a trigger that is gone has nothing to publish. +func TestApplyTriggerReconcile_AGoneTriggerIsForgotten(t *testing.T) { + r := &ApplyTriggerReconciler{Client: triggerTestClient(t), Kind: configbutleraiv1alpha3.ApplyTriggerKindFlux} + result, err := r.Reconcile(context.Background(), + ctrl.Request{NamespacedName: k8stypes.NamespacedName{Namespace: "team-a", Name: "gone"}}) + require.NoError(t, err) + assert.Zero(t, result.RequeueAfter) + + _, err = (&ApplyTriggerReconciler{Kind: "WebhookTrigger"}).Reconcile(context.Background(), ctrl.Request{}) + require.Error(t, err, "a kind this release does not serve") +} diff --git a/internal/controller/applytrigger_holder_test.go b/internal/controller/applytrigger_holder_test.go new file mode 100644 index 000000000..ca47f7952 --- /dev/null +++ b/internal/controller/applytrigger_holder_test.go @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" +) + +// The second FluxTrigger on flux-system/checkout, in another namespace and created after +// team-a/checkout. +var inLineKey = k8stypes.NamespacedName{Namespace: "team-b", Name: "checkout"} + +// duplicateFixture is the hold fixture with two FluxTriggers naming flux-system/checkout in the +// default cluster: team-a/checkout, created first, and team-b/checkout, an hour later. The default +// ClusterProvider admits both namespaces. +func duplicateFixture(t *testing.T) *holdFixture { + t.Helper() + first := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + first.CreationTimestamp = metav1.NewTime(time.Unix(1_000_000, 0)) + second := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + second.Namespace = inLineKey.Namespace + second.CreationTimestamp = metav1.NewTime(first.CreationTimestamp.Add(time.Hour)) + + sim := applierSim(t, applierFluxObjects()...) + c := inLineClient(t, configbutleraiv1alpha3.ApplyTriggerKindFlux, first, second) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, sim.Client, nil, true) + recorder := record.NewFakeRecorder(64) + r.Recorder = recorder + return &holdFixture{r: r, sim: sim, recorder: recorder} +} + +// inLineClient is a control cluster holding triggers, in team-a and team-b, each referenced by an +// eligible GitTarget of its namespace, with the default ClusterProvider admitting both. +func inLineClient(t *testing.T, kind string, triggers ...ctrlclient.Object) ctrlclient.Client { + t.Helper() + objects := append([]ctrlclient.Object{admitting(configbutleraiv1alpha3.DefaultClusterProviderName, + "team-a", "team-b")}, triggers...) + objects = append(objects, referencedIn("team-a", kind)...) + objects = append(objects, referencedIn("team-b", kind)...) + return triggerTestClient(t, objects...) +} + +// reconcileKey reconciles one trigger a few times, enough to settle a suspend, and returns it. +func (f *holdFixture) reconcileKey(t *testing.T, key k8stypes.NamespacedName) *configbutleraiv1alpha3.FluxTrigger { + t.Helper() + trigger := &configbutleraiv1alpha3.FluxTrigger{} + require.Eventually(t, func() bool { + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, f.r.Get(context.Background(), key, trigger)) + held := findCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld) + return held == nil || held.Reason != ReasonSuspending + }, 5*time.Second, 5*time.Millisecond) + return trigger +} + +// deleteUntilGone deletes a trigger and reconciles it until its finalizer is gone. +func (f *holdFixture) deleteUntilGone(t *testing.T, key k8stypes.NamespacedName) { + t.Helper() + trigger := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), key, trigger)) + require.NoError(t, f.r.Delete(context.Background(), trigger)) + require.Eventually(t, func() bool { + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + return apierrors.IsNotFound(f.r.Get(context.Background(), key, &configbutleraiv1alpha3.FluxTrigger{})) + }, 5*time.Second, 5*time.Millisecond, "the finalizer goes") +} + +// TestFluxTrigger_OneTriggerHoldsAKustomization is review 5's finding 5: two FluxTriggers naming one +// Kustomization both held it, under the same field manager. Each took the other's resume for an +// external one and suspended it mid-apply, and deleting either handed back the hold the other still +// needed. Now the one created first holds it; the other is Configured=False (HeldByAnotherTrigger), +// naming the holder, and does nothing to the Kustomization, ever. +func TestFluxTrigger_OneTriggerHoldsAKustomization(t *testing.T) { + f := duplicateFixture(t) + + second := f.reconcileKey(t, inLineKey) + first := f.reconcileKey(t, triggerKey) + + assert.True(t, conditionIsTrue(first.Status.Conditions, ApplyTriggerConditionHeld), "the first holds it") + assert.True(t, conditionIsTrue(first.Status.Conditions, ConditionTypeReady)) + configured := findCondition(second.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, metav1.ConditionFalse, configured.Status) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, configured.Reason) + assert.Equal(t, "Kustomization flux-system/checkout in the cluster of ClusterProvider \"default\" is held by "+ + "FluxTrigger team-a/checkout, which was created first; this trigger takes it over once that one is deleted "+ + "or no eligible GitTarget references it", configured.Message) + ready := findCondition(second.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, ready.Reason) + assert.True(t, conditionIsTrue(second.Status.Conditions, ConditionTypeStalled), "a human removes one of them") + held := findCondition(second.Status.Conditions, ApplyTriggerConditionHeld) + require.NotNil(t, held) + assert.Equal(t, metav1.ConditionUnknown, held.Status) + assert.Equal(t, ReasonNotConfigured, held.Reason) + + events := f.events() + require.Len(t, events, 1, "one adoption, by the holder: %v", events) + assert.True(t, strings.HasPrefix(events[0], "Normal Adopted"), events[0]) + + // A GitTarget naming the trigger in line says why it is not Ready. + target := triggerTarget(true) + target.Namespace = inLineKey.Namespace + gt := &GitTargetReconciler{Client: f.r.Client} + verdict, _ := gt.observeApplyTrigger(context.Background(), beginStatus(nil, nil, target), target, + triggerProvider()) + assert.Equal(t, metav1.ConditionFalse, verdict.value.Status) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, verdict.value.Reason) + assert.Contains(t, verdict.value.Message, "held by FluxTrigger team-a/checkout") + assert.True(t, verdict.stalled) +} + +// TestFluxTrigger_TheHoldersResumeIsNoExternalResumeToTheOther: a branch worker's apply through the +// holder resumes the Kustomization; the trigger in line does not take that for `flux resume`, and +// leaves it running. +func TestFluxTrigger_TheHoldersResumeIsNoExternalResumeToTheOther(t *testing.T) { + f := duplicateFixture(t) + f.reconcileKey(t, triggerKey) + f.reconcileKey(t, inLineKey) + f.events() + + // The worker records its operation on the holder, and resumes the Kustomization. + holder := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), triggerKey, holder)) + holder.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationRunning, Reason: configbutleraiv1alpha3.ApplyReasonRefusal, + Token: "T1", RequestedAt: metav1.Now(), + } + require.NoError(t, f.r.Status().Update(context.Background(), holder)) + patch := []byte(`{"spec":{"suspend":false}}`) + _, err := f.sim.Client.Resource(fluxsim.KustomizationGVR()).Namespace("flux-system").Patch(context.Background(), + "checkout", k8stypes.MergePatchType, patch, metav1.PatchOptions{FieldManager: applier.SuspendFieldManager}) + require.NoError(t, err) + + second := f.reconcileKey(t, inLineKey) + + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "left running for the holder") + assert.Empty(t, f.events(), "no external resume") + assert.Nil(t, second.Status.Operation, "nothing recorded on the trigger in line") +} + +// TestFluxTrigger_DeletingTheTriggerInLineHandsNothingBack: the trigger in line never held the +// Kustomization, so its finalizer goes without touching it: it stays suspended under GitOps +// Reverser's field manager, for the holder. +func TestFluxTrigger_DeletingTheTriggerInLineHandsNothingBack(t *testing.T) { + f := duplicateFixture(t) + f.reconcileKey(t, triggerKey) + f.reconcileKey(t, inLineKey) + f.events() + + f.deleteUntilGone(t, inLineKey) + + ks := f.sim.Kustomization(checkoutKustomization) + assert.True(t, suspendedSpec(ks), "still held") + assert.Contains(t, kustomizationManagers(ks), applier.SuspendFieldManager) + assert.Empty(t, f.events(), "nothing released") + holder := f.reconcileKey(t, triggerKey) + assert.True(t, conditionIsTrue(holder.Status.Conditions, ApplyTriggerConditionHeld)) +} + +// TestFluxTrigger_DeletingTheHolderHandsOverToTheNextInLine: the holder suspends the Kustomization as +// it goes, and leaves it suspended for the trigger in line instead of releasing it: released, Flux +// would apply it at once, unchecked against the branch, before the next trigger could suspend it +// again (review 6). Its going enqueues the trigger in line (a deletion maps to the others naming the +// applier), which then holds the Kustomization as it was left, with nothing to adopt. +func TestFluxTrigger_DeletingTheHolderHandsOverToTheNextInLine(t *testing.T) { + f := duplicateFixture(t) + f.reconcileKey(t, triggerKey) + f.reconcileKey(t, inLineKey) + f.events() + gone := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), triggerKey, gone)) + applied := len(f.sim.Applied()) + + f.deleteUntilGone(t, triggerKey) + ks := f.sim.Kustomization(checkoutKustomization) + assert.True(t, suspendedSpec(ks), "never released between the two") + assert.Contains(t, kustomizationManagers(ks), applier.SuspendFieldManager) + assert.Len(t, f.sim.Applied(), applied, "so Flux applied nothing on its own") + requests := f.r.triggersInLine(context.Background(), gone) + require.Len(t, requests, 1) + assert.Equal(t, inLineKey, requests[0].NamespacedName) + assert.True(t, triggerLineChanged().Delete(event.DeleteEvent{Object: gone})) + assert.False(t, triggerLineChanged().Update(event.UpdateEvent{ObjectOld: gone, ObjectNew: gone})) + + second := f.reconcileKey(t, inLineKey) + + assert.True(t, conditionIsTrue(second.Status.Conditions, ApplyTriggerConditionConfigured)) + assert.True(t, conditionIsTrue(second.Status.Conditions, ApplyTriggerConditionHeld)) + assert.True(t, conditionIsTrue(second.Status.Conditions, ConditionTypeReady)) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) + assert.Equal(t, []string{"Normal Released left Kustomization flux-system/checkout suspended for FluxTrigger " + + "team-b/checkout, which holds it next"}, f.events()) +} + +// TestFluxTrigger_AHolderWithAnApplyInFlightKeepsIt is review 5's second finding (P2): a trigger +// created in the same second as the holder, which comes before it by name, appears while the +// holder's apply is in flight. It waits: it is Configured=False, leaves the resumed Kustomization to +// the holder's worker, and records nothing. Once that apply ends, the holder still keeps it: a +// trigger that holds its Kustomization comes before any that does not, so the Kustomization changes +// hands only once the holder lets go of it. +func TestFluxTrigger_AHolderWithAnApplyInFlightKeepsIt(t *testing.T) { + holderTrigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + holderTrigger.Namespace = inLineKey.Namespace // team-b: after team-a by name + holderTrigger.CreationTimestamp = metav1.NewTime(time.Unix(1_000_000, 0)) + holderKey := k8stypes.NamespacedName{Namespace: holderTrigger.Namespace, Name: holderTrigger.Name} + sim := applierSim(t, applierFluxObjects()...) + c := inLineClient(t, configbutleraiv1alpha3.ApplyTriggerKindFlux, holderTrigger) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, sim.Client, nil, true) + recorder := record.NewFakeRecorder(64) + r.Recorder = recorder + f := &holdFixture{r: r, sim: sim, recorder: recorder} + f.reconcileKey(t, holderKey) + f.events() + + // The holder's worker records its operation, and resumes the Kustomization. + held := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), holderKey, held)) + held.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationRunning, Reason: configbutleraiv1alpha3.ApplyReasonRefusal, + Token: "T1", RequestedAt: metav1.Now(), + } + require.NoError(t, f.r.Status().Update(context.Background(), held)) + _, err := f.sim.Client.Resource(fluxsim.KustomizationGVR()).Namespace("flux-system").Patch(context.Background(), + "checkout", k8stypes.MergePatchType, []byte(`{"spec":{"suspend":false}}`), + metav1.PatchOptions{FieldManager: applier.SuspendFieldManager}) + require.NoError(t, err) + // The newcomer: same second, team-a. + newcomer := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + newcomer.CreationTimestamp = holderTrigger.CreationTimestamp + require.NoError(t, f.r.Create(context.Background(), newcomer)) + + waiting := f.reconcileKey(t, triggerKey) + + configured := findCondition(waiting.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, configured.Reason) + assert.Contains(t, configured.Message, "whose apply is in flight; this trigger takes it over once that apply ends") + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "left running for the holder") + assert.Nil(t, waiting.Status.Operation, "no external resume recorded") + assert.Empty(t, f.events()) + assert.True(t, conditionIsTrue(f.reconcileKey(t, holderKey).Status.Conditions, ApplyTriggerConditionConfigured), + "the holder keeps it") + + // The worker suspends the Kustomization again and completes its operation. + _, err = f.sim.Client.Resource(fluxsim.KustomizationGVR()).Namespace("flux-system").Patch(context.Background(), + "checkout", k8stypes.MergePatchType, []byte(`{"spec":{"suspend":true}}`), + metav1.PatchOptions{FieldManager: applier.SuspendFieldManager}) + require.NoError(t, err) + require.NoError(t, f.r.Get(context.Background(), holderKey, held)) + finished := metav1.Now() + held.Status.Operation.Phase, held.Status.Operation.FinishedAt = configbutleraiv1alpha3.ApplyOperationSucceeded, + &finished + require.NoError(t, f.r.Status().Update(context.Background(), held)) + + still := findCondition(f.reconcileKey(t, triggerKey).Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, still) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, still.Reason) + assert.Contains(t, still.Message, "which holds it now") + assert.True(t, conditionIsTrue(f.reconcileKey(t, holderKey).Status.Conditions, ApplyTriggerConditionConfigured), + "the holder keeps it between applies too") + assert.Empty(t, f.events(), "and nothing is adopted twice") +} + +// TestArgoCDTrigger_OneTriggerHoldsAnApplication: two ArgoCDTriggers on one Application would both +// start syncs; the one created later is Configured=False (HeldByAnotherTrigger). +func TestArgoCDTrigger_OneTriggerHoldsAnApplication(t *testing.T) { + first := testArgoCDTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + first.CreationTimestamp = metav1.NewTime(time.Unix(1_000_000, 0)) + second := testArgoCDTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + second.Namespace = inLineKey.Namespace + second.CreationTimestamp = first.CreationTimestamp + c := inLineClient(t, configbutleraiv1alpha3.ApplyTriggerKindArgoCD, first, second) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindArgoCD, + applierFakeSource(t, applierArgoApplication(nil)), nil, true) + + for _, key := range []k8stypes.NamespacedName{triggerKey, inLineKey} { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + } + + got := &configbutleraiv1alpha3.ArgoCDTrigger{} + require.NoError(t, c.Get(context.Background(), triggerKey, got)) + assert.True(t, conditionIsTrue(got.Status.Conditions, ApplyTriggerConditionConfigured), + "created in the same second: the namespace decides") + require.NoError(t, c.Get(context.Background(), inLineKey, got)) + configured := findCondition(got.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, configured.Reason) + assert.Contains(t, configured.Message, "Application argocd/checkout") + assert.Contains(t, configured.Message, "held by ArgoCDTrigger team-a/checkout") +} diff --git a/internal/controller/applytrigger_rule_test.go b/internal/controller/applytrigger_rule_test.go new file mode 100644 index 000000000..a2778f229 --- /dev/null +++ b/internal/controller/applytrigger_rule_test.go @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/authz" +) + +// The rule (review 7): a trigger acts only while an eligible GitTarget references it. These tests +// pin its three cases (never referenced or released, referenced, retiring), the binding verdict, and +// the line of triggers for one applier. + +// ruleFixture is the hold fixture over a control cluster holding objects, the ClusterProviders and +// the GitProvider platform, and no GitTarget unless objects carries one. +func ruleFixture(t *testing.T, objects ...ctrlclient.Object) *holdFixture { + t.Helper() + sim := applierSim(t, applierFluxObjects()...) + c := triggerTestClient(t, append(objects, unreferenced{})...) + r := triggerReconciler(c, configbutleraiv1alpha3.ApplyTriggerKindFlux, sim.Client, nil, true) + recorder := record.NewFakeRecorder(64) + r.Recorder = recorder + return &holdFixture{r: r, sim: sim, recorder: recorder} +} + +// fluxTarget is the GitTarget team-a/checkout-flux naming the FluxTrigger checkout, writing main. +func fluxTarget() *configbutleraiv1alpha3.GitTarget { + return referencingTarget("checkout-flux", configbutleraiv1alpha3.ApplyTriggerKindFlux) +} + +// settle reconciles the trigger at key until its Held condition is neither Suspending nor +// Releasing, and returns it. +func (f *holdFixture) settle(t *testing.T, key k8stypes.NamespacedName) *configbutleraiv1alpha3.FluxTrigger { + t.Helper() + trigger := &configbutleraiv1alpha3.FluxTrigger{} + require.Eventually(t, func() bool { + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, f.r.Get(context.Background(), key, trigger)) + held := findCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld) + return held != nil && held.Reason != ReasonSuspending && held.Reason != ReasonReleasing + }, 5*time.Second, 5*time.Millisecond) + return trigger +} + +// heldReason is the trigger's Held reason. +func heldReason(t *testing.T, trigger *configbutleraiv1alpha3.FluxTrigger) string { + t.Helper() + held := findCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld) + require.NotNil(t, held) + return held.Reason +} + +// TestApplyTriggerRule_AnUnreferencedTriggerDoesNothingAndIsReady: no call reaches the applier's +// cluster, nothing is held, no finalizer is added, and the trigger is Ready, so a Flux health check +// that waits on it does not wait for the GitTarget that will name it. +func TestApplyTriggerRule_AnUnreferencedTriggerDoesNothingAndIsReady(t *testing.T) { + for _, kind := range []string{configbutleraiv1alpha3.ApplyTriggerKindFlux, configbutleraiv1alpha3.ApplyTriggerKindArgoCD} { + t.Run(kind, func(t *testing.T) { + var trigger ctrlclient.Object = testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + if kind == configbutleraiv1alpha3.ApplyTriggerKindArgoCD { + trigger = testArgoCDTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + } + r := triggerReconciler(triggerTestClient(t, trigger, unreferenced{}), kind, nil, nil, true) + calls := 0 + r.ApplierClients = func(context.Context, string) (dynamic.Interface, error) { + calls++ + return nil, errors.New("the applier's cluster must not be asked") + } + + got := reconcileTrigger(t, r) + + assert.Zero(t, calls, "nothing is read in the applier's cluster") + configured := findCondition(*got.StatusConditions(), ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, metav1.ConditionUnknown, configured.Status) + assert.Equal(t, applier.ReasonNotReferenced, configured.Reason) + ready := findCondition(*got.StatusConditions(), ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionTrue, ready.Status) + assert.Equal(t, applier.ReasonNotReferenced, ready.Reason) + assert.False(t, conditionIsTrue(*got.StatusConditions(), ConditionTypeReconciling)) + assert.Empty(t, got.GetFinalizers()) + url, _, known := got.SourceBinding() + assert.False(t, known, "no source is published: %s", url) + if flux, ok := got.(*configbutleraiv1alpha3.FluxTrigger); ok { + assert.Equal(t, applier.ReasonNotReferenced, heldReason(t, flux)) + } + }) + } +} + +// TestApplyTriggerRule_TheFirstEligibleReferenceAdoptsAndTheLastReleases: naming the trigger is the +// adoption moment; the GitTarget going hands the Kustomization back, and the trigger is idle again. +func TestApplyTriggerRule_TheFirstEligibleReferenceAdoptsAndTheLastReleases(t *testing.T) { + f := ruleFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + idle := f.settle(t, triggerKey) + assert.Equal(t, applier.ReasonNotReferenced, heldReason(t, idle)) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "Flux applies it on its own") + + target := fluxTarget() + require.NoError(t, f.r.Create(context.Background(), target)) + held := f.settle(t, triggerKey) + assert.Equal(t, ReasonSuspended, heldReason(t, held)) + assert.True(t, conditionIsTrue(held.Status.Conditions, ConditionTypeReady)) + assert.True(t, controllerutil.ContainsFinalizer(held, configbutleraiv1alpha3.FluxTriggerFinalizer)) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Normal Adopted"), events[0]) + + require.NoError(t, f.r.Delete(context.Background(), target)) + released := f.settle(t, triggerKey) + assert.Equal(t, applier.ReasonNotReferenced, heldReason(t, released)) + ready := findCondition(released.Status.Conditions, ConditionTypeReady) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionTrue, ready.Status) + assert.Equal(t, applier.ReasonNotReferenced, ready.Reason) + assert.Empty(t, released.Finalizers, "a trigger that holds nothing deletes at once") + assert.Nil(t, released.Status.Source) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "handed back to Flux") + events = f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Normal Released handed Kustomization flux-system/checkout back"), + events[0]) +} + +// TestApplyTriggerRule_ARecreatedGitTargetReleasesAndReadopts is the trade-off the rule accepts: +// deleting and re-creating the GitTarget hands the Kustomization back, and adopts it again. +func TestApplyTriggerRule_ARecreatedGitTargetReleasesAndReadopts(t *testing.T) { + f := ruleFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), fluxTarget()) + f.settle(t, triggerKey) + f.events() + + target := &configbutleraiv1alpha3.GitTarget{} + require.NoError(t, f.r.Get(context.Background(), + k8stypes.NamespacedName{Namespace: "team-a", Name: "checkout-flux"}, target)) + require.NoError(t, f.r.Delete(context.Background(), target)) + f.settle(t, triggerKey) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) + require.NoError(t, f.r.Create(context.Background(), fluxTarget())) + again := f.settle(t, triggerKey) + + assert.Equal(t, ReasonSuspended, heldReason(t, again)) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) + var reasons []string + for _, event := range f.events() { + reasons = append(reasons, strings.Fields(event)[1]) + } + assert.Equal(t, []string{EventReasonReleased, EventReasonAdopted}, reasons) +} + +// TestApplyTriggerRule_TheBindingAggregatesItsGitTargets: one eligible GitTarget among mismatched +// ones binds the trigger; none that can be judged yet is progressing; every one refused is stalled, +// with the reason they share, or NotBound for mixed ones, and each GitTarget in the message. +func TestApplyTriggerRule_TheBindingAggregatesItsGitTargets(t *testing.T) { + onBranch := func(name, branch string) *configbutleraiv1alpha3.GitTarget { + target := referencingTarget(name, configbutleraiv1alpha3.ApplyTriggerKindFlux) + target.Spec.Branch = branch + return target + } + elsewhere := onBranch("elsewhere", "main") + elsewhere.Spec.ClusterProviderRef = &meta.LocalObjectReference{Name: "closed"} + cases := []struct { + name string + objects []ctrlclient.Object + wantStatus metav1.ConditionStatus + wantReason string + wantStalled bool + wantNames []string + }{ + {name: "one eligible among mismatches holds", wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, + objects: []ctrlclient.Object{onBranch("release", "release"), onBranch("main", "main")}}, + {name: "none that can be judged yet is progressing", wantStatus: metav1.ConditionUnknown, + wantReason: applier.ReasonNotBound, wantNames: []string{"team-a/unprovided"}, + objects: []ctrlclient.Object{func() ctrlclient.Object { + target := onBranch("unprovided", "main") + target.Spec.GitProviderRef.Name = "absent" + return target + }(), onBranch("release", "release")}}, + {name: "every one mismatched is stalled, with the shared reason", wantStatus: metav1.ConditionFalse, + wantReason: applier.ReasonSourceMismatch, wantStalled: true, + wantNames: []string{"team-a/release", "team-a/staging"}, + objects: []ctrlclient.Object{onBranch("release", "release"), onBranch("staging", "staging")}}, + {name: "refused for different reasons is stalled, NotBound", wantStatus: metav1.ConditionFalse, + wantReason: applier.ReasonNotBound, wantStalled: true, + wantNames: []string{"team-a/elsewhere", "team-a/release"}, + objects: []ctrlclient.Object{onBranch("release", "release"), elsewhere, + &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: "closed"}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{Names: []string{"team-b"}}, + }, + }}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + objects := append([]ctrlclient.Object{testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)}, + tc.objects...) + f := ruleFixture(t, objects...) + + got := f.settle(t, triggerKey) + + configured := findCondition(got.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, tc.wantStatus, configured.Status, configured.Message) + assert.Equal(t, tc.wantReason, configured.Reason) + assert.Equal(t, tc.wantStalled, conditionIsTrue(got.Status.Conditions, ConditionTypeStalled)) + for _, name := range tc.wantNames { + assert.Contains(t, configured.Message, "GitTarget "+name+":") + } + assert.Equal(t, tc.wantStatus == metav1.ConditionTrue, + suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "held only once bound") + assert.Equal(t, tc.wantStatus == metav1.ConditionTrue, got.Status.Source != nil, + "the source is published only once bound") + }) + } +} + +// TestApplyTriggerRule_AnOlderTriggerNotInLineKeepsNoneOut: an older trigger whose GitTarget writes +// another branch, or whose namespace is not admitted, no longer keeps a younger eligible one from +// holding the Kustomization. +func TestApplyTriggerRule_AnOlderTriggerNotInLineKeepsNoneOut(t *testing.T) { + for name, older := range map[string]func(*configbutleraiv1alpha3.GitTarget, *[]ctrlclient.Object){ + "bound to another branch": func(target *configbutleraiv1alpha3.GitTarget, _ *[]ctrlclient.Object) { + target.Spec.Branch = "release" + }, + "not admitted": func(target *configbutleraiv1alpha3.GitTarget, objects *[]ctrlclient.Object) { + target.Spec.ClusterProviderRef = &meta.LocalObjectReference{Name: "closed"} + *objects = append(*objects, admitting("closed", "team-a")) + }, + } { + t.Run(name, func(t *testing.T) { + stray := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + stray.Namespace = inLineKey.Namespace + stray.CreationTimestamp = metav1.NewTime(time.Unix(1_000_000, 0)) + mine := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + mine.CreationTimestamp = metav1.NewTime(stray.CreationTimestamp.Add(time.Hour)) + objects := []ctrlclient.Object{stray, mine} + strayRefs := referencedIn(inLineKey.Namespace, configbutleraiv1alpha3.ApplyTriggerKindFlux) + older(strayRefs[1].(*configbutleraiv1alpha3.GitTarget), &objects) + objects = append(objects, strayRefs...) + objects = append(objects, referencedIn("team-a", configbutleraiv1alpha3.ApplyTriggerKindFlux)...) + objects = append(objects, admitting(configbutleraiv1alpha3.DefaultClusterProviderName, "team-a", "team-b")) + f := ruleFixture(t, objects...) + + got := f.settle(t, triggerKey) + + assert.True(t, conditionIsTrue(got.Status.Conditions, ApplyTriggerConditionConfigured), + findCondition(got.Status.Conditions, ApplyTriggerConditionConfigured).Message) + assert.Equal(t, ReasonSuspended, heldReason(t, got)) + strayConfigured := findCondition(f.settle(t, inLineKey).Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, strayConfigured) + assert.NotEqual(t, metav1.ConditionTrue, strayConfigured.Status) + assert.NotEqual(t, applier.ReasonHeldByAnotherTrigger, strayConfigured.Reason, + "it is refused for its own binding, before the line is asked") + }) + } +} + +// TestApplyTriggerRule_ARetiringHolderStaysInLineUntilItsReleaseIsDone: the holder's GitTarget goes +// while a successor waits. Until the holder has suspended the Kustomization and seen that handled, +// it stays first in line, Held=False (Releasing), and the successor does nothing; then the holder +// leaves the Kustomization suspended for it, and the successor takes it over. +func TestApplyTriggerRule_ARetiringHolderStaysInLineUntilItsReleaseIsDone(t *testing.T) { + holder := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + holder.CreationTimestamp = metav1.NewTime(time.Unix(1_000_000, 0)) + successor := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + successor.Namespace = inLineKey.Namespace + successor.CreationTimestamp = metav1.NewTime(holder.CreationTimestamp.Add(time.Hour)) + objects := []ctrlclient.Object{holder, successor, + admitting(configbutleraiv1alpha3.DefaultClusterProviderName, "team-a", "team-b")} + objects = append(objects, referencedIn("team-a", configbutleraiv1alpha3.ApplyTriggerKindFlux)...) + objects = append(objects, referencedIn("team-b", configbutleraiv1alpha3.ApplyTriggerKindFlux)...) + f := ruleFixture(t, objects...) + f.settle(t, triggerKey) + f.settle(t, inLineKey) + f.events() + + // A worker's apply has the Kustomization resumed, and kustomize-controller is busy. + resume(t, f) + unpause := f.sim.PauseKustomizations() + target := &configbutleraiv1alpha3.GitTarget{} + require.NoError(t, f.r.Get(context.Background(), + k8stypes.NamespacedName{Namespace: "team-a", Name: "checkout-fluxtrigger"}, target)) + require.NoError(t, f.r.Delete(context.Background(), target)) + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + retiring := f.trigger(t) + assert.Equal(t, ReasonReleasing, heldReason(t, retiring)) + assert.True(t, controllerutil.ContainsFinalizer(retiring, configbutleraiv1alpha3.FluxTriggerFinalizer)) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "suspended first") + _, err = f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: inLineKey}) + require.NoError(t, err) + waiting := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), inLineKey, waiting)) + configured := findCondition(waiting.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, applier.ReasonHeldByAnotherTrigger, configured.Reason, "the retiring holder is still first") + assert.Contains(t, configured.Message, "which holds it now") + + unpause() + done := f.settle(t, triggerKey) + assert.Empty(t, done.Finalizers, "handed over") + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "left suspended for the successor") + taken := f.settle(t, inLineKey) + assert.True(t, conditionIsTrue(taken.Status.Conditions, ApplyTriggerConditionConfigured)) + assert.Equal(t, ReasonSuspended, heldReason(t, taken)) + events := f.events() + require.NotEmpty(t, events) + assert.True(t, strings.HasPrefix(events[0], "Normal Released left Kustomization flux-system/checkout suspended "+ + "for FluxTrigger team-b/checkout"), events[0]) +} + +// resume resumes the Kustomization as a branch worker's apply does, under GitOps Reverser's field +// manager. +func resume(t *testing.T, f *holdFixture) { + t.Helper() + f.sim.Edit(checkoutKustomization, func(ks *unstructured.Unstructured) { + _ = unstructured.SetNestedField(ks.Object, false, "spec", "suspend") + }) +} + +// TestApplyTriggerRule_AReferenceReturningDuringTheReleaseHoldsAgain: the GitTarget is gone, and +// back, before the release finished: the trigger holds the Kustomization again, as it is, and +// hands nothing back. +func TestApplyTriggerRule_AReferenceReturningDuringTheReleaseHoldsAgain(t *testing.T) { + f := ruleFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), fluxTarget()) + f.settle(t, triggerKey) + f.events() + resume(t, f) + unpause := f.sim.PauseKustomizations() + target := &configbutleraiv1alpha3.GitTarget{} + require.NoError(t, f.r.Get(context.Background(), + k8stypes.NamespacedName{Namespace: "team-a", Name: "checkout-flux"}, target)) + require.NoError(t, f.r.Delete(context.Background(), target)) + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + assert.Equal(t, ReasonReleasing, heldReason(t, f.trigger(t))) + + require.NoError(t, f.r.Create(context.Background(), fluxTarget())) + unpause() + back := f.settle(t, triggerKey) + + assert.Equal(t, ReasonSuspended, heldReason(t, back)) + assert.True(t, controllerutil.ContainsFinalizer(back, configbutleraiv1alpha3.FluxTriggerFinalizer)) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "never handed back") + for _, event := range f.events() { + assert.NotContains(t, event, EventReasonReleased) + } +} + +// TestApplyTriggerRule_ReleasingWithAnApplyInFlightSuspendsFirst: the last GitTarget goes while an +// apply has the Kustomization resumed, followed by nobody here. The release suspends it, waits until +// that is handled, then hands it back, and completes the operation nobody would complete. +func TestApplyTriggerRule_ReleasingWithAnApplyInFlightSuspendsFirst(t *testing.T) { + f := ruleFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), fluxTarget()) + f.settle(t, triggerKey) + f.events() + trigger := f.trigger(t) + trigger.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationRunning, Reason: configbutleraiv1alpha3.ApplyReasonCatchUp, + Token: "T1", RequestedAt: metav1.Now(), + } + require.NoError(t, f.r.Status().Update(context.Background(), trigger)) + resume(t, f) + before := f.sim.Reconciles(checkoutKustomization) + target := &configbutleraiv1alpha3.GitTarget{} + require.NoError(t, f.r.Get(context.Background(), + k8stypes.NamespacedName{Namespace: "team-a", Name: "checkout-flux"}, target)) + require.NoError(t, f.r.Delete(context.Background(), target)) + + released := f.settle(t, triggerKey) + + assert.Greater(t, f.sim.Reconciles(checkoutKustomization), before, "the suspend was handled before the release") + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "then handed back") + require.NotNil(t, released.Status.Operation) + assert.Equal(t, configbutleraiv1alpha3.ApplyOperationFailed, released.Status.Operation.Phase) + assert.Contains(t, released.Status.Operation.Message, "handed its Kustomization back") +} + +// TestApplyTriggerRule_AHolderLeftUnconfiguredSaysTheKustomizationIsSuspended is C3: a holder whose +// configuration turns invalid keeps its finalizer and the Kustomization suspended, and its Held +// message says so. +func TestApplyTriggerRule_AHolderLeftUnconfiguredSaysTheKustomizationIsSuspended(t *testing.T) { + f := ruleFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), fluxTarget()) + f.settle(t, triggerKey) + f.r.newApplier = triggerReconciler(nil, configbutleraiv1alpha3.ApplyTriggerKindFlux, nil, nil, false).newApplier + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + got := f.trigger(t) + + configured := findCondition(got.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, applier.ReasonForbidden, configured.Reason) + held := findCondition(got.Status.Conditions, ApplyTriggerConditionHeld) + require.NotNil(t, held) + assert.Equal(t, metav1.ConditionUnknown, held.Status) + assert.Equal(t, ReasonNotConfigured, held.Reason) + assert.Contains(t, held.Message, "is left suspended under GitOps Reverser's field manager") + assert.True(t, controllerutil.ContainsFinalizer(got, configbutleraiv1alpha3.FluxTriggerFinalizer)) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) +} + +// TestApplyTriggerRule_AGitTargetEventRerunsTheTriggersItNames: a GitTarget naming a trigger, or +// that named one before an update, re-runs its check; a GitProvider re-runs the triggers of the +// GitTargets writing through it. +func TestApplyTriggerRule_AGitTargetEventRerunsTheTriggersItNames(t *testing.T) { + c := triggerTestClient(t, fluxTarget(), platformProvider()) + r := &ApplyTriggerReconciler{Client: c, Kind: configbutleraiv1alpha3.ApplyTriggerKindFlux} + + requests := r.gitProviderToTriggers(context.Background(), platformProvider()) + require.Len(t, requests, 1) + assert.Equal(t, triggerKey, requests[0].NamespacedName) + + argo := &ApplyTriggerReconciler{Client: c, Kind: configbutleraiv1alpha3.ApplyTriggerKindArgoCD} + assert.Empty(t, argo.gitProviderToTriggers(context.Background(), platformProvider()), "another kind's") +} + +// TestApplyTriggerRule_AnApplierInAnotherNamespaceNeedsTheOptIn is B1: a namespace a ClusterProvider +// admits may suspend and apply an applier in another namespace only when the provider sets +// spec.allowCrossNamespace.appliers; one in the trigger's own namespace needs nothing. Refused, the +// trigger never asks the applier's cluster, and holds nothing. +func TestApplyTriggerRule_AnApplierInAnotherNamespaceNeedsTheOptIn(t *testing.T) { + for _, kind := range []string{configbutleraiv1alpha3.ApplyTriggerKindFlux, configbutleraiv1alpha3.ApplyTriggerKindArgoCD} { + for name, tc := range map[string]struct { + sameNamespace, optIn bool + wantStatus metav1.ConditionStatus + wantReason string + }{ + "another namespace, without the opt-in": { + wantStatus: metav1.ConditionFalse, wantReason: authz.ReasonApplierNamespaceNotAllowed, + }, + "another namespace, with the opt-in": {optIn: true, wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded}, + "its own namespace, which needs nothing": { + sameNamespace: true, wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, + }, + } { + t.Run(kind+"/"+name, func(t *testing.T) { + r, calls := applierNamespaceReconciler(t, kind, tc.sameNamespace, tc.optIn) + + got := reconcileTrigger(t, r) + + configured := findCondition(*got.StatusConditions(), ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, tc.wantStatus, configured.Status, configured.Message) + assert.Equal(t, tc.wantReason, configured.Reason) + if tc.wantStatus == metav1.ConditionFalse { + assert.Contains(t, configured.Message, "spec.allowCrossNamespace.appliers") + assert.True(t, conditionIsTrue(*got.StatusConditions(), ConditionTypeStalled)) + assert.Zero(t, *calls, "the applier's cluster is never asked") + assert.Empty(t, got.GetFinalizers(), "and nothing is held") + } + }) + } + } +} + +// applierNamespaceReconciler is a reconciler of kind over the trigger team-a/checkout, whose +// applier is in team-a when sameNamespace is set and in flux-system or argocd otherwise, with the +// default ClusterProvider setting allowCrossNamespace.appliers when optIn is set. It counts the +// lookups of the applier's cluster. +func applierNamespaceReconciler( + t *testing.T, kind string, sameNamespace, optIn bool, +) (*ApplyTriggerReconciler, *int) { + t.Helper() + trigger := ctrlclient.Object(testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + appliers := applierFluxObjects() + if kind == configbutleraiv1alpha3.ApplyTriggerKindArgoCD { + trigger = testArgoCDTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + appliers = []runtime.Object{applierArgoApplication(nil)} + } + if sameNamespace { + switch typed := trigger.(type) { + case *configbutleraiv1alpha3.FluxTrigger: + typed.Spec.KustomizationRef.Namespace = "" + case *configbutleraiv1alpha3.ArgoCDTrigger: + typed.Spec.ApplicationRef.Namespace = "" + } + for _, obj := range appliers { + obj.(*unstructured.Unstructured).SetNamespace("team-a") + } + } + provider := admitting(configbutleraiv1alpha3.DefaultClusterProviderName, "team-a") + if !optIn { + provider.Spec.AllowCrossNamespace = nil + } + r := triggerReconciler(triggerTestClient(t, trigger, provider), kind, applierFakeSource(t, appliers...), nil, true) + calls := new(int) + clients := r.ApplierClients + r.ApplierClients = func(ctx context.Context, cluster string) (dynamic.Interface, error) { + *calls++ + return clients(ctx, cluster) + } + return r, calls +} diff --git a/internal/controller/applytrigger_schema_test.go b/internal/controller/applytrigger_schema_test.go new file mode 100644 index 000000000..976b9214d --- /dev/null +++ b/internal/controller/applytrigger_schema_test.go @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "time" + + meta "github.com/fluxcd/pkg/apis/meta" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" +) + +// The API server holds the trigger kinds to their contract: each names one applier and its +// cluster, neither can be pointed elsewhere once created, quietPeriod is bounded and defaulted, and +// a GitTarget names a trigger by a kind this release serves. +var _ = Describe("Apply trigger schema", func() { + ctx := context.Background() + create := func(obj client.Object) error { + err := k8sClient.Create(ctx, obj) + if err == nil { + DeferCleanup(func() { _ = k8sClient.Delete(ctx, obj) }) + } + return err + } + flux := func(name string) *configbutleraiv1alpha3.FluxTrigger { + return &configbutleraiv1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: configbutleraiv1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: "hub"}, + KustomizationRef: meta.NamespacedObjectReference{Namespace: "flux-system", Name: "apps"}, + }, + } + } + argo := func(name string) *configbutleraiv1alpha3.ArgoCDTrigger { + return &configbutleraiv1alpha3.ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: configbutleraiv1alpha3.ArgoCDTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: "hub"}, + ApplicationRef: meta.NamespacedObjectReference{Namespace: "argocd", Name: "apps"}, + }, + } + } + + It("defaults a FluxTrigger's quiet period, and bounds it", func() { + trigger := flux("flux-default") + Expect(create(trigger)).To(Succeed()) + Expect(trigger.Spec.QuietPeriod).NotTo(BeNil()) + Expect(trigger.Spec.QuietPeriod.Duration).To(Equal(configbutleraiv1alpha3.DefaultApplierQuietPeriod)) + + long := flux("flux-long") + long.Spec.QuietPeriod = &metav1.Duration{Duration: 2 * time.Hour} + Expect(create(long)).To(MatchError(ContainSubstring("spec.quietPeriod must be a Go duration of at most 1h"))) + }) + + It("defaults the cluster to the default ClusterProvider, as a GitTarget does, and requires names", func() { + omitted := argo("argo-default-cluster") + omitted.Spec.ClusterProviderRef = nil + Expect(create(omitted)).To(Succeed()) + Expect(omitted.Spec.ClusterProviderRef).To(Equal( + &meta.LocalObjectReference{Name: configbutleraiv1alpha3.DefaultClusterProviderName})) + + noCluster := flux("flux-no-cluster") + noCluster.Spec.ClusterProviderRef.Name = "" + Expect(create(noCluster)).To(MatchError(ContainSubstring("spec.clusterProviderRef.name must not be empty"))) + + noApplication := argo("argo-no-app") + noApplication.Spec.ApplicationRef.Name = "" + Expect(create(noApplication)).To(MatchError(ContainSubstring("spec.applicationRef.name must not be empty"))) + + noKustomization := flux("flux-no-ks") + noKustomization.Spec.KustomizationRef.Name = "" + Expect(create(noKustomization)).To(MatchError(ContainSubstring("spec.kustomizationRef.name must not be empty"))) + }) + + // A trigger no GitTarget names adopts nothing, so it carries no finalizer and deletes at once; + // it still publishes its conditions, Ready because it does all it is asked to. + It("publishes a new FluxTrigger's conditions, and holds nothing while no GitTarget names it", func() { + trigger := flux("flux-first-pass") + Expect(create(trigger)).To(Succeed()) + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trigger), trigger)).To(Succeed()) + ready := findCondition(trigger.Status.Conditions, ConditionTypeReady) + g.Expect(ready).NotTo(BeNil()) + g.Expect(ready.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(ready.Reason).To(Equal(applier.ReasonNotReferenced)) + }, 20*time.Second, 100*time.Millisecond).Should(Succeed()) + Expect(trigger.Finalizers).NotTo(ContainElement(configbutleraiv1alpha3.FluxTriggerFinalizer)) + held := findCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld) + Expect(held).NotTo(BeNil()) + Expect(held.Reason).To(Equal(applier.ReasonNotReferenced)) + }) + + It("never points a trigger at another applier or cluster", func() { + // The suite's FluxTrigger reconciler writes the trigger too (its finalizer), so each update + // is made from a fresh read, and one that loses that race is made again. + update := func(obj client.Object, mutate func()) error { + var err error + Eventually(func() bool { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + mutate() + err = k8sClient.Update(ctx, obj) + return !apierrors.IsConflict(err) + }).Should(BeTrue()) + return err + } + fluxTrigger := flux("flux-immutable") + Expect(create(fluxTrigger)).To(Succeed()) + Expect(update(fluxTrigger, func() { fluxTrigger.Spec.KustomizationRef.Name = "other" })). + To(MatchError(ContainSubstring("spec.kustomizationRef is immutable"))) + Expect(update(fluxTrigger, func() { fluxTrigger.Spec.ClusterProviderRef.Name = "other" })). + To(MatchError(ContainSubstring("spec.clusterProviderRef is immutable"))) + Expect(update(fluxTrigger, func() { + fluxTrigger.Spec.QuietPeriod = &metav1.Duration{Duration: time.Minute} + })).To(Succeed(), "the quiet period is not what it holds") + + argoTrigger := argo("argo-immutable") + Expect(create(argoTrigger)).To(Succeed()) + Expect(update(argoTrigger, func() { argoTrigger.Spec.ApplicationRef.Namespace = "elsewhere" })). + To(MatchError(ContainSubstring("spec.applicationRef is immutable"))) + }) + + It("lets a GitTarget name a trigger of a kind it serves, and requires one for Reapply", func() { + target := func(name string, ref *configbutleraiv1alpha3.ApplyTriggerReference) *configbutleraiv1alpha3.GitTarget { + return &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "prov"}, + Branch: "main", + Path: "apps/" + name, + ApplyTriggerRef: ref, + }, + } + } + Expect(create(target("names-flux", &configbutleraiv1alpha3.ApplyTriggerReference{ + Kind: configbutleraiv1alpha3.ApplyTriggerKindFlux, Name: "apps", + }))).To(Succeed()) + Expect(create(target("names-argo", &configbutleraiv1alpha3.ApplyTriggerReference{ + Kind: configbutleraiv1alpha3.ApplyTriggerKindArgoCD, Name: "apps", + }))).To(Succeed()) + Expect(create(target("names-webhook", &configbutleraiv1alpha3.ApplyTriggerReference{ + Kind: "WebhookTrigger", Name: "apps", + }))).To(MatchError(ContainSubstring("Unsupported value"))) + + reapply := target("reapply-alone", nil) + reapply.Spec.OnRefusal = configbutleraiv1alpha3.RefusalActionReapply + Expect(create(reapply)).To(MatchError(ContainSubstring("spec.onRefusal: Reapply needs spec.applyTriggerRef"))) + }) +}) + +// The trigger reconciler runs under the manager: a trigger created gets its trio, Ready while no +// GitTarget names it, and a GitTarget naming it re-runs the check, though nothing else changed on +// the trigger. The suite reaches no applier cluster, so that check goes no further. +var _ = Describe("Apply trigger reconcile", func() { + ctx := context.Background() + + It("publishes the trigger's verdict", func() { + trigger := &configbutleraiv1alpha3.ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: "reconciled", Namespace: "default"}, + Spec: configbutleraiv1alpha3.ArgoCDTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: configbutleraiv1alpha3.DefaultClusterProviderName}, + ApplicationRef: meta.NamespacedObjectReference{Namespace: "argocd", Name: "apps"}, + }, + } + Expect(k8sClient.Create(ctx, trigger)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, trigger) }) + + configuredReason := func(g Gomega) string { + got := &configbutleraiv1alpha3.ArgoCDTrigger{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trigger), got)).To(Succeed()) + g.Expect(got.Status.ObservedGeneration).To(Equal(got.Generation)) + configured := findCondition(got.Status.Conditions, ApplyTriggerConditionConfigured) + g.Expect(configured).NotTo(BeNil()) + return configured.Reason + } + Eventually(func(g Gomega) { + g.Expect(configuredReason(g)).To(Equal(applier.ReasonNotReferenced)) + }, 30*time.Second, 250*time.Millisecond).Should(Succeed()) + + target := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "names-reconciled", Namespace: "default"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "prov"}, Branch: "main", Path: "apps/reconciled", + ApplyTriggerRef: &configbutleraiv1alpha3.ApplyTriggerReference{ + Kind: configbutleraiv1alpha3.ApplyTriggerKindArgoCD, Name: "reconciled", + }, + }, + } + Expect(k8sClient.Create(ctx, target)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, target) }) + Eventually(func(g Gomega) { + g.Expect(configuredReason(g)).NotTo(Equal(applier.ReasonNotReferenced)) + }, 30*time.Second, 250*time.Millisecond).Should(Succeed()) + }) +}) diff --git a/internal/controller/condition_helper.go b/internal/controller/condition_helper.go index e523eb31a..3d06bb438 100644 --- a/internal/controller/condition_helper.go +++ b/internal/controller/condition_helper.go @@ -37,6 +37,15 @@ func findCondition(conditions []metav1.Condition, conditionType string) *metav1. return apimeta.FindStatusCondition(conditions, conditionType) } +// keptCondition is the value of conditionType that a check that could not be made publishes: the +// last one published, or Unknown with message when there is none. +func keptCondition(conditions []metav1.Condition, conditionType, message string) conditionValue { + if cond := findCondition(conditions, conditionType); cond != nil { + return conditionValue{Status: cond.Status, Reason: cond.Reason, Message: cond.Message} + } + return conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonProgressing, Message: message} +} + func conditionIsTrue(conditions []metav1.Condition, conditionType string) bool { return apimeta.IsStatusConditionTrue(conditions, conditionType) } diff --git a/internal/controller/fluxtrigger_hold.go b/internal/controller/fluxtrigger_hold.go new file mode 100644 index 000000000..4e4b6514a --- /dev/null +++ b/internal/controller/fluxtrigger_hold.go @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// A FluxTrigger holds its Kustomization suspended while idle; the branch workers resume it for each +// apply and suspend it again (internal/applier/flux.go). The reconciler does the rest of the +// lifecycle, from docs/design/apply-trigger.md ("Who owns spec.suspend"): +// +// - Adoption: a Configured trigger suspends its Kustomization, under GitOps Reverser's field +// manager, with a token; Held (and so Ready) waits until the Kustomization handled it. +// - External resume: a held Kustomization found resumed with no operation of a branch worker in +// flight was resumed by somebody else (`flux resume`). It is suspended again with a token, every +// time, with a Normal Event, and recorded as an Unresolved operation that completes once that +// token is handled. There is no step-back. +// - Hand-back: the finalizer suspends once more, waits until that token is handled, and releases +// spec.suspend, so Flux owns the Kustomization's applies again. +// +// All of it is the holder's alone (applier.HeldByAnother): another trigger naming the same Kustomization +// is Configured=False (HeldByAnotherTrigger), so it never suspends or records an external +// resume, and its deletion hands nothing back. It takes the Kustomization over, with an adoption, +// once the holder is gone. + +// holdKustomization holds a Configured FluxTrigger's Kustomization suspended, and returns the +// trigger's Held condition. It writes nothing while a branch worker's operation is in flight +// and followed: the worker resumes the Kustomization and suspends it again itself. One nobody +// follows any more is suspended here (suspendUnfollowed), so the Kustomization is never left resumed +// with nothing guarding it. +func (r *ApplyTriggerReconciler) holdKustomization( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, +) conditionValue { + ctx, cancel := context.WithTimeout(ctx, applierCheckTimeout) + defer cancel() + held, err := r.readHold(ctx, trigger) + if err != nil { + return keptCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld, err.Error()) + } + if !held.inFlight() || held.operation.Token == "" { + r.unfollowed.forget(trigger, "") + } + if held.inFlight() && held.operation.Token != "" { + if r.followed(trigger, held.operation) { + return applyingCondition(held.operation) + } + return r.suspendUnfollowed(ctx, trigger, held) + } + return r.hold(ctx, trigger, held) +} + +// applyingCondition is Held=True (Applying) for a branch worker's operation in flight that is +// followed: the Kustomization is resumed for it, or about to be, and suspended again when it ends. +func applyingCondition(operation *configbutleraiv1alpha3.ApplyOperation) conditionValue { + return conditionValue{Status: metav1.ConditionTrue, Reason: ReasonApplying, Message: fmt.Sprintf( + "applying %s for a %s; suspended again when it ends", applier.DescribeCommit(operation.Commit), + operation.Reason)} +} + +// OperationFollower answers whether this process carries out or follows a branch worker's +// operation: (*applier.Dispatcher).Following, which runs beside this reconciler on the leader. +type OperationFollower interface { + Following(trigger git.TriggerRef, token string) bool +} + +// DefaultUnfollowedGrace is how long a branch worker's operation in flight may go unfollowed +// before the reconciler suspends it. It covers the moments nobody follows an operation that is fine: +// a process that just started, whose worker has not recovered the operation yet, and an operation +// whose completion is not written yet. +const DefaultUnfollowedGrace = time.Minute + +// EventReasonOperationUnfollowed is a branch worker's operation nobody followed any more, whose +// Kustomization the reconciler suspended and whose operation it completed. +const EventReasonOperationUnfollowed = "OperationUnfollowed" + +// followed reports whether the reconciler leaves the trigger's in-flight operation to its worker: +// this process follows it, or has seen it unfollowed for less than the grace. +func (r *ApplyTriggerReconciler) followed( + trigger *configbutleraiv1alpha3.FluxTrigger, operation *configbutleraiv1alpha3.ApplyOperation, +) bool { + ref := git.TriggerRef{Kind: trigger.TriggerKind(), Namespace: trigger.Namespace, Name: trigger.Name} + if r.Follower != nil && r.Follower.Following(ref, operation.Token) { + r.unfollowed.forget(trigger, operation.Token) + return true + } + grace := r.UnfollowedGrace + if grace <= 0 { + grace = DefaultUnfollowedGrace + } + return time.Since(r.unfollowed.since(trigger, operation.Token)) < grace +} + +// suspendUnfollowed sees a branch worker's operation that nobody follows through, as the worker's own +// follow would have: it suspends the Kustomization with a token, waits until that is handled, and +// completes the operation Failed, since whether its apply ran cannot be told from here. The worker, +// should it still name the trigger, asks again. The suspend token is recorded on the operation, so a +// worker that recovers it meanwhile follows the same suspend. +func (r *ApplyTriggerReconciler) suspendUnfollowed( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, held heldState, +) conditionValue { + ref := trigger.Applier() + operation := held.operation.DeepCopy() + switch { + case !held.suspension.Suspended || !held.suspension.Held: + token, err := held.flux.Suspend(ctx, ref) + if err != nil { + return keptCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld, err.Error()) + } + r.event(trigger, corev1.EventTypeWarning, EventReasonOperationUnfollowed, + "the %s apply of %s is followed by nobody; suspended Kustomization %s again", operation.Reason, + applier.DescribeCommit(operation.Commit), ref) + operation.SuspendToken = token + operation.Message = "followed by nobody; suspended again, and complete once nothing it started is running" + trigger.Status.Operation = operation + return conditionValue{Status: metav1.ConditionFalse, Reason: ReasonSuspending, + Message: fmt.Sprintf("Kustomization %s is being suspended: its apply is followed by nobody", ref)} + case !held.suspension.Handled: + return conditionValue{Status: metav1.ConditionFalse, Reason: ReasonSuspending, + Message: fmt.Sprintf("Kustomization %s is suspended; waiting for it to handle the suspend", ref)} + } + finished := metav1.NewTime(time.Now()) + operation.FinishedAt, operation.AppliedRevision = &finished, held.suspension.Applied.AppliedRevision + operation.Phase = configbutleraiv1alpha3.ApplyOperationFailed + operation.Message = "followed by nobody to its end; the Kustomization is suspended again, and whether the apply " + + "ran cannot be told" + trigger.Status.Operation = operation + r.unfollowed.forget(trigger, operation.Token) + return conditionValue{Status: metav1.ConditionTrue, Reason: ReasonSuspended, + Message: fmt.Sprintf("Kustomization %s is suspended; GitOps Reverser starts every apply it makes", ref)} +} + +// unfollowedOperations remembers when the reconciler first saw each trigger's in-flight operation +// followed by nobody, by trigger and token. It is in memory: a process that starts gives every +// operation the whole grace, which is when restart recovery picks it up. +type unfollowedOperations struct { + mu sync.Mutex + first map[unfollowedKey]time.Time +} + +type unfollowedKey struct { + trigger k8stypes.NamespacedName + token string +} + +// since is when the operation was first seen unfollowed, now if this is the first time. +func (u *unfollowedOperations) since(trigger *configbutleraiv1alpha3.FluxTrigger, token string) time.Time { + u.mu.Lock() + defer u.mu.Unlock() + key := unfollowedKey{ + trigger: k8stypes.NamespacedName{Namespace: trigger.Namespace, Name: trigger.Name}, token: token, + } + if u.first == nil { + u.first = map[unfollowedKey]time.Time{} + } + first, seen := u.first[key] + if !seen { + first = time.Now() + u.first[key] = first + } + return first +} + +// forget drops what is remembered of the trigger's operation with token, or of all its operations +// when token is empty. +func (u *unfollowedOperations) forget(trigger *configbutleraiv1alpha3.FluxTrigger, token string) { + u.mu.Lock() + defer u.mu.Unlock() + name := k8stypes.NamespacedName{Namespace: trigger.Namespace, Name: trigger.Name} + for key := range u.first { + if key.trigger == name && (token == "" || key.token == token) { + delete(u.first, key) + } + } +} + +// heldState is what holdKustomization reads before it decides. +type heldState struct { + flux *applier.Flux + suspension applier.Suspension + operation *configbutleraiv1alpha3.ApplyOperation +} + +// inFlight reports whether the trigger records an operation that has not completed. +func (h heldState) inFlight() bool { return h.operation != nil && !h.operation.Phase.Completed() } + +// readHold reads the Kustomization's suspension and the trigger's operation, uncached. The +// operation is read before and after the Kustomization, and the two must be the same operation in +// the same phase, or the read is an error and the next reconcile reads again. A worker writes its +// operation before it resumes and completes it after it suspended again, so with both reads equal, +// a Kustomization read resumed with no operation in flight was resumed by somebody else. With one +// read, an operation that completed between the two GETs would be taken for an external resume. +func (r *ApplyTriggerReconciler) readHold( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, +) (heldState, error) { + ref := trigger.Applier() + flux, err := r.fluxClient(ctx, trigger) + if err != nil { + return heldState{}, fmt.Errorf("cannot reach the cluster of Kustomization %s: %w", ref, err) + } + before, err := r.liveOperation(ctx, trigger) + if err != nil { + return heldState{}, fmt.Errorf("cannot read the operation of %s: %w", trigger.Name, err) + } + suspension, err := flux.Suspension(ctx, ref) + if err != nil { + return heldState{}, fmt.Errorf("cannot read Kustomization %s: %w", ref, err) + } + operation, err := r.liveOperation(ctx, trigger) + if err != nil { + return heldState{}, fmt.Errorf("cannot read the operation of %s: %w", trigger.Name, err) + } + if !sameOperationPhase(before, operation) { + return heldState{}, fmt.Errorf("the operation of %s moved while Kustomization %s was read; reading again", + trigger.Name, ref) + } + return heldState{flux: flux, suspension: suspension, operation: operation}, nil +} + +// sameOperationPhase reports whether two reads of a trigger's operation show the same operation, by +// its tokens, in the same phase. +func sameOperationPhase(a, b *configbutleraiv1alpha3.ApplyOperation) bool { + if a == nil || b == nil { + return a == b + } + return a.Token == b.Token && a.SuspendToken == b.SuspendToken && a.Phase == b.Phase && + a.RequestedAt.Equal(&b.RequestedAt) +} + +// hold suspends the Kustomization when it is not held with its suspend handled, and completes the operation an +// external resume was recorded as once it is. +func (r *ApplyTriggerReconciler) hold( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, held heldState, +) conditionValue { + ref := trigger.Applier() + suspending := func(message string) conditionValue { + return conditionValue{Status: metav1.ConditionFalse, Reason: ReasonSuspending, Message: message} + } + switch { + case !held.suspension.Suspended: + // Read resumed: the resume can be another trigger's, one that came first in line since the + // cached read this reconcile decided on. That trigger recorded its operation before it + // resumed, so a read made now sees it. + if other, err := applier.HeldByAnother(ctx, r.uncachedReader(), trigger); err != nil || other != nil { + message := fmt.Sprintf("cannot check which trigger holds Kustomization %s: %v", ref, err) + if other != nil { + message = applier.HeldByAnotherMessage(trigger, other) + } + return conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonNotConfigured, Message: message} + } + token, err := held.flux.Suspend(ctx, ref) + if err != nil { + return keptCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld, err.Error()) + } + r.suspended(trigger, token, heldBefore(trigger) || held.inFlight()) + return suspending(fmt.Sprintf("Kustomization %s is being suspended", ref)) + case !held.suspension.Held: + // Suspended by somebody else's write (`flux suspend`): take the field over, with a token, so + // nothing that write let run is taken for idle. + if _, err := held.flux.Suspend(ctx, ref); err != nil { + return keptCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld, err.Error()) + } + return suspending(fmt.Sprintf("Kustomization %s is being suspended under GitOps Reverser", ref)) + case !held.suspension.Handled: + return suspending(fmt.Sprintf("Kustomization %s is suspended; waiting for it to handle the suspend", ref)) + } + if held.inFlight() { + completeExternalOperation(trigger, held.operation, held.suspension) + } + return conditionValue{Status: metav1.ConditionTrue, Reason: ReasonSuspended, + Message: fmt.Sprintf("Kustomization %s is suspended; GitOps Reverser starts every apply it makes", ref)} +} + +// suspended records a Kustomization just suspended: an adoption, or, for one held before, an +// external resume, recorded as an Unresolved operation that completes once token is handled. +func (r *ApplyTriggerReconciler) suspended(trigger *configbutleraiv1alpha3.FluxTrigger, token string, resumed bool) { + ref := trigger.Applier() + if !resumed { + r.event(trigger, corev1.EventTypeNormal, EventReasonAdopted, + "suspended Kustomization %s: from now on GitOps Reverser starts every apply it makes", ref) + return + } + r.event(trigger, corev1.EventTypeNormal, EventReasonResumedElsewhere, + "Kustomization %s was resumed outside GitOps Reverser, which suspended it again; the apply "+ + "that resume started was not checked against the branch", ref) + trigger.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationUnresolved, SuspendToken: token, + RequestedAt: metav1.NewTime(time.Now()), + Message: "the Kustomization was resumed outside GitOps Reverser; suspended again, and " + + "complete once nothing that resume started is running", + } +} + +// completeExternalOperation completes recorded, the operation an external resume was recorded as, +// once the suspend after it was handled, with what the Kustomization's last apply reported. +// recorded is the operation read uncached, which the hold decided on: the cached trigger can still +// lack it. +func completeExternalOperation( + trigger *configbutleraiv1alpha3.FluxTrigger, recorded *configbutleraiv1alpha3.ApplyOperation, + suspension applier.Suspension, +) { + operation := recorded.DeepCopy() + finished := metav1.NewTime(time.Now()) + operation.FinishedAt, operation.AppliedRevision = &finished, suspension.Applied.AppliedRevision + operation.Phase = configbutleraiv1alpha3.ApplyOperationSucceeded + operation.Message = "the Kustomization was resumed outside GitOps Reverser, and is suspended again" + if suspension.Applied.Outcome != git.ApplySucceeded { + operation.Phase = configbutleraiv1alpha3.ApplyOperationFailed + operation.Message += ": " + suspension.Applied.Message + } + trigger.Status.Operation = operation +} + +// heldBefore reports whether the trigger last said it held its Kustomization. +func heldBefore(trigger *configbutleraiv1alpha3.FluxTrigger) bool { + return apimeta.IsStatusConditionTrue(trigger.Status.Conditions, ApplyTriggerConditionHeld) +} + +// liveOperation reads the trigger's status.operation uncached: a branch worker writes it just before +// it resumes the Kustomization, and a cached read could still show the operation before it. +func (r *ApplyTriggerReconciler) liveOperation( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, +) (*configbutleraiv1alpha3.ApplyOperation, error) { + live := &configbutleraiv1alpha3.FluxTrigger{} + if err := r.uncachedReader().Get(ctx, client.ObjectKeyFromObject(trigger), live); err != nil { + return nil, err + } + return live.Status.Operation, nil +} + +// uncachedReader is APIReader, or the client when none is set. +func (r *ApplyTriggerReconciler) uncachedReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +// fluxClient is the Flux client of the trigger's cluster. +func (r *ApplyTriggerReconciler) fluxClient( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, +) (*applier.Flux, error) { + dynamicClient, err := r.ApplierClients(ctx, trigger.ClusterProvider()) + if err != nil { + return nil, err + } + return applier.NewFlux(dynamicClient), nil +} + +// releaseDeadline is how long a retiring FluxTrigger tries to hand its Kustomization back, counted +// from when its Held condition turned Releasing, before it lets go anyway with a Warning Event that +// says what the Kustomization is left as. The suspend it waits for is handled once the apply running then ends, so +// the deadline is longer than an apply usually takes; a kustomize-controller that is down never +// handles it, and the deletion must not wait on that forever. +const releaseDeadline = 10 * time.Minute + +// retire hands a FluxTrigger's Kustomization back to Flux, and then lets go of it: a trigger being +// deleted (check nil), or one no eligible GitTarget references any more (check is what the +// reconcile found). It suspends the Kustomization once more if it is resumed (a branch worker's +// apply may still be running), waits until that suspend's token is handled, and releases +// spec.suspend, unless a trigger in line takes the Kustomization over (releaseStep). Until that is +// done the trigger stays the holder, Held=False (Releasing), so no successor acts on the +// Kustomization meanwhile. A Kustomization that is gone has nothing to hand back; a cluster no +// client reaches any more since its ClusterProvider went (releaseClient), and a Kustomization the +// identity may not write, are let go with a Warning Event: none should hold the trigger forever. Nor does anything else past releaseDeadline. A trigger behind another in line for the +// Kustomization hands nothing back: the Kustomization is that one's. Letting go removes the +// finalizer; a live trigger then holds nothing until an eligible GitTarget references it again, +// which adopts the Kustomization anew. +func (r *ApplyTriggerReconciler) retire( + ctx context.Context, st *reconcileStatus, trigger *configbutleraiv1alpha3.FluxTrigger, check *triggerCheck, +) (ctrl.Result, error) { + holder, err := applier.HeldByAnother(ctx, r.Client, trigger) + if err != nil { + return ctrl.Result{}, err + } + var pending *pendingRelease + if holder != nil { + // A trigger in line never held the Kustomization: its spec.suspend is the holder's, under the + // same field manager, so there is nothing of this trigger's to hand back. + logf.FromContext(ctx).Info("the Kustomization is held by another trigger; handing nothing back", + "kustomization", trigger.Applier().String(), "holder", holder.GetNamespace()+"/"+holder.GetName()) + } else { + successor, err := r.nextInLine(ctx, trigger) + if err != nil { + return ctrl.Result{}, err + } + step, cancel := context.WithTimeout(ctx, applierCheckTimeout) + pending = r.releaseStep(step, trigger, successor) + cancel() + } + if pending != nil { + since := releasingSince(trigger) + if time.Since(since) < releaseDeadline { + r.publishReleasing(st, trigger, check, since) + if err := st.commit(ctx); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: st.requeueAfter(pending.result.RequeueAfter)}, pending.err + } + r.event(trigger, corev1.EventTypeWarning, EventReasonReleaseFailed, + "gave up handing Kustomization %s back to Flux %s after it began: %s", + trigger.Applier(), releaseDeadline, pending.left) + } + if check == nil { + controllerutil.RemoveFinalizer(trigger, configbutleraiv1alpha3.FluxTriggerFinalizer) + return ctrl.Result{}, client.IgnoreNotFound(r.Update(ctx, trigger)) + } + if err := r.setFinalizer(ctx, st, trigger, false); err != nil { + return ctrl.Result{}, err + } + r.letGoOfOperation(trigger) + held := conditionValue{Status: metav1.ConditionFalse, Reason: applier.ReasonNotReferenced, + Message: "nothing to hold: no GitTarget names this trigger, and its Kustomization was handed back"} + if check.refs.Named() { + held = conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonNotConfigured, + Message: "the Kustomization was handed back, and is held again once an eligible GitTarget references " + + "the trigger"} + } + st.setValue(ApplyTriggerConditionHeld, held) + st.applyReadiness(r.triggerReadiness(trigger, check.configured, &held)) + if err := st.commit(ctx); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: st.requeueAfter(fluxTriggerSteadyInterval)}, nil +} + +// publishReleasing sets Held=False (Releasing) and the trio on a trigger whose hand-back is not +// done, its deadline counted from since. The condition is written anew when it was not Releasing +// before, so its lastTransitionTime says when the release began. +func (r *ApplyTriggerReconciler) publishReleasing( + st *reconcileStatus, trigger *configbutleraiv1alpha3.FluxTrigger, check *triggerCheck, since time.Time, +) { + why := "the trigger is being deleted" + if check != nil && check.refs.Named() { + why = "no GitTarget naming the trigger is eligible any more" + } else if check != nil { + why = "no GitTarget names the trigger any more" + } + held := conditionValue{Status: metav1.ConditionFalse, Reason: ReasonReleasing, Message: fmt.Sprintf( + "%s: handing Kustomization %s back to Flux, by %s at the latest", why, trigger.Applier(), + since.Add(releaseDeadline).UTC().Format(time.RFC3339))} + if previous := findCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld); previous == nil || + previous.Reason != ReasonReleasing { + st.remove(ApplyTriggerConditionHeld) + } + st.setValue(ApplyTriggerConditionHeld, held) + if check == nil { + rd := newReadiness("", trigger.TriggerKind()+" is not stalled") + rd.progressing(held.Status, held.Reason, held.Message) + st.applyReadiness(rd) + return + } + st.applyReadiness(r.triggerReadiness(trigger, check.configured, &held)) +} + +// releasingSince is when the trigger's Held condition turned Releasing, or now when it has not. +func releasingSince(trigger *configbutleraiv1alpha3.FluxTrigger) time.Time { + held := findCondition(trigger.Status.Conditions, ApplyTriggerConditionHeld) + if held == nil || held.Reason != ReasonReleasing || held.LastTransitionTime.IsZero() { + return time.Now() + } + return held.LastTransitionTime.Time +} + +// letGoOfOperation completes an operation a retired trigger still records in flight, when this +// process does not follow it: nobody would complete it, and its Kustomization is handed back. +func (r *ApplyTriggerReconciler) letGoOfOperation(trigger *configbutleraiv1alpha3.FluxTrigger) { + operation := trigger.Status.Operation + if operation == nil || operation.Phase.Completed() { + return + } + ref := git.TriggerRef{Kind: trigger.TriggerKind(), Namespace: trigger.Namespace, Name: trigger.Name} + if operation.Token != "" && r.Follower != nil && r.Follower.Following(ref, operation.Token) { + return + } + completed := operation.DeepCopy() + finished := metav1.NewTime(time.Now()) + completed.Phase, completed.FinishedAt = configbutleraiv1alpha3.ApplyOperationFailed, &finished + completed.Message = "the trigger handed its Kustomization back before the apply was seen to end" + trigger.Status.Operation = completed + r.unfollowed.forget(trigger, "") +} + +// setFinalizer puts the finalizer on the trigger, or takes it off, and carries the metadata that +// write left into the status session, so the status patch that follows is made against it. The +// finalizer is what hands the Kustomization back, so it is in place before the first adoption +// write, and off once the trigger has handed the Kustomization back or over. +func (r *ApplyTriggerReconciler) setFinalizer( + ctx context.Context, st *reconcileStatus, trigger *configbutleraiv1alpha3.FluxTrigger, present bool, +) error { + update := trigger.DeepCopy() + changed := controllerutil.RemoveFinalizer(update, configbutleraiv1alpha3.FluxTriggerFinalizer) + if present { + changed = controllerutil.AddFinalizer(update, configbutleraiv1alpha3.FluxTriggerFinalizer) + } + if !changed { + return nil + } + if err := r.Update(ctx, update); err != nil { + return err + } + trigger.SetFinalizers(update.GetFinalizers()) + trigger.SetResourceVersion(update.GetResourceVersion()) + st.metadataWritten() + return nil +} + +// nextInLine is the trigger that takes the Kustomization over once trigger is gone, or nil when no +// trigger will: one in line that is not being deleted, and whose Configured is True or False only +// because trigger holds the Kustomization (HeldByAnotherTrigger is checked last, so such a trigger +// passed every other check). +func (r *ApplyTriggerReconciler) nextInLine( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, +) (configbutleraiv1alpha3.ApplyTrigger, error) { + others, err := applier.TriggersOfApplier(ctx, r.Client, trigger) + if err != nil { + return nil, err + } + var next configbutleraiv1alpha3.ApplyTrigger + var nextCreated metav1.Time + for _, other := range others { + configured := findCondition(*other.StatusConditions(), ApplyTriggerConditionConfigured) + takesOver := configured != nil && (configured.Status == metav1.ConditionTrue || + configured.Reason == applier.ReasonHeldByAnotherTrigger) + created := other.GetCreationTimestamp() + if takesOver && other.GetDeletionTimestamp().IsZero() && (next == nil || created.Before(&nextCreated)) { + next, nextCreated = other, created + } + } + return next, nil +} + +// pendingRelease is a hand-back that is not done: what to retry it on, and what the Kustomization is +// left as should the trigger go now. +type pendingRelease struct { + result ctrl.Result + err error + left string +} + +// releaseStep takes the hand-back one step, and returns nil once the trigger may go. With a +// successor (nextInLine), the Kustomization is not released but left suspended, its suspend handled, for it: +// released, Flux would apply it at once, unchecked against the branch, before the successor's +// adoption could suspend it again. +func (r *ApplyTriggerReconciler) releaseStep( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, successor configbutleraiv1alpha3.ApplyTrigger, +) *pendingRelease { + flux, err := r.releaseClient(ctx, trigger) + if err != nil { + return &pendingRelease{ + err: err, left: fmt.Sprintf("its cluster cannot be reached (%v), so it is left as it was", err), + } + } + if flux == nil { + return nil + } + ref := trigger.Applier() + suspending := &pendingRelease{ + result: ctrl.Result{RequeueAfter: applyTriggerSuspendingInterval}, + left: "it is suspended, and has not handled the suspend yet (is kustomize-controller running?); " + + "`flux resume` resumes it", + } + suspension, err := flux.Suspension(ctx, ref) + switch { + case apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err): + return nil + case err != nil: + return r.releaseFailed(trigger, err, fmt.Sprintf("it cannot be read (%v), so it is left as it was", err)) + case !suspension.Held: + return nil + case !suspension.Suspended: + if _, err := flux.Suspend(ctx, ref); err != nil { + return r.releaseFailed(trigger, err, fmt.Sprintf("it is resumed, and suspending it failed (%v), so "+ + "GitOps Reverser's field manager still holds spec.suspend", err)) + } + return suspending + case !suspension.Handled: + return suspending + } + if successor != nil { + r.event(trigger, corev1.EventTypeNormal, EventReasonReleased, + "left Kustomization %s suspended for %s %s/%s, which holds it next", ref, successor.TriggerKind(), + successor.GetNamespace(), successor.GetName()) + return nil + } + keeper, err := flux.Release(ctx, ref) + if err != nil { + return r.releaseFailed(trigger, err, fmt.Sprintf("it is suspended, and releasing spec.suspend failed (%v); "+ + "`flux resume` resumes it", err)) + } + if keeper != "" { + r.event(trigger, corev1.EventTypeNormal, EventReasonReleased, + "handed Kustomization %s back to Flux: it stays suspended, because field manager %q also sets "+ + "spec.suspend", ref, keeper) + return nil + } + r.event(trigger, corev1.EventTypeNormal, EventReasonReleased, + "handed Kustomization %s back to Flux: it is resumed, and applies on its own again", ref) + return nil +} + +// releaseFailed is a hand-back step that failed with err: one the identity may not make lets the +// trigger go with a Warning Event, since a retry cannot change it, and any other is retried. +func (r *ApplyTriggerReconciler) releaseFailed( + trigger *configbutleraiv1alpha3.FluxTrigger, err error, left string, +) *pendingRelease { + if apierrors.IsForbidden(err) { + r.event(trigger, corev1.EventTypeWarning, EventReasonReleaseFailed, + "could not hand Kustomization %s back to Flux: %v", trigger.Applier(), err) + return nil + } + return &pendingRelease{err: err, left: left} +} + +// releaseClient is the Flux client a retiring trigger hands its Kustomization back through, or nil +// when there is nothing to reach: no cluster clients are wired, or the ClusterProvider is gone and +// no client reaches its cluster any more, which lets the trigger go with a Warning Event. A +// ClusterProvider that is gone does not end the hand-back by itself: one without kubeConfig names +// the operator's own cluster, which still answers, and a remote one may still be reached through the +// client it left. +func (r *ApplyTriggerReconciler) releaseClient( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, +) (*applier.Flux, error) { + if r.ApplierClients == nil { + return nil, nil //nolint:nilnil // nothing to reach is no error + } + provider := &configbutleraiv1alpha3.ClusterProvider{} + err := r.Get(ctx, client.ObjectKey{Name: trigger.ClusterProvider()}, provider) + gone := apierrors.IsNotFound(err) + if err != nil && !gone { + return nil, err + } + flux, err := r.fluxClient(ctx, trigger) + switch { + case err != nil && gone: + r.event(trigger, corev1.EventTypeWarning, EventReasonReleaseFailed, + "could not hand Kustomization %s back to Flux: its ClusterProvider %q is gone; it is left suspended, "+ + "and `flux resume` resumes it", trigger.Applier(), trigger.ClusterProvider()) + return nil, nil //nolint:nilnil // nothing to reach, and the Event says so + case err != nil: + return nil, fmt.Errorf("reach the cluster of Kustomization %s to hand it back: %w", trigger.Applier(), err) + } + return flux, nil +} + +// event records an Event on the trigger, when there is a recorder. +func (r *ApplyTriggerReconciler) event(object client.Object, eventType, reason, messageFmt string, args ...any) { + if r.Recorder != nil { + r.Recorder.Eventf(object, eventType, reason, messageFmt, args...) + } +} + +// publishConsumers sets the trigger's status.source.otherConsumers and status.source.prefetch: the +// consumers of its GitRepository in the applier's cluster, other than its own Kustomization, that no +// FluxTrigger on the same ClusterProvider holds. Prefetch also needs its own Kustomization held, +// which held, this pass's Held, says. A listing that fails publishes none and no prefetch, +// because the precondition cannot be checked. Information only (decision 2): it never affects Ready. +func (r *ApplyTriggerReconciler) publishConsumers( + ctx context.Context, trigger *configbutleraiv1alpha3.FluxTrigger, held conditionValue, +) { + source := trigger.Status.Source + if source == nil || r.ApplierClients == nil { + return + } + source.OtherConsumers, source.Prefetch = nil, false + ctx, cancel := context.WithTimeout(ctx, applierCheckTimeout) + defer cancel() + flux, err := r.fluxClient(ctx, trigger) + if err != nil { + return + } + consumers, err := flux.Consumers(ctx, k8stypes.NamespacedName{ + Namespace: source.GitRepository.Namespace, Name: source.GitRepository.Name, + }) + if err != nil { + logf.FromContext(ctx).V(1).Info("cannot list the GitRepository's consumers; not prefetching it", + "gitRepository", source.GitRepository.Namespace+"/"+source.GitRepository.Name, "error", err.Error()) + return + } + others, err := r.heldKustomizations(ctx, trigger.ClusterProvider()) + if err != nil { + return + } + for _, consumer := range consumers { + key := k8stypes.NamespacedName{Namespace: consumer.Namespace, Name: consumer.Name} + itself := key == trigger.Applier() + if consumer.Kind == configbutleraiv1alpha3.ApplierKindKustomization && (itself || others[key]) { + continue + } + source.OtherConsumers = append(source.OtherConsumers, consumer) + } + source.Prefetch = len(source.OtherConsumers) == 0 && held.Status == metav1.ConditionTrue +} + +// heldKustomizations is every Kustomization a FluxTrigger on the ClusterProvider holds: one whose +// Held condition is True. A trigger that is not Configured (SuspendOwnedElsewhere), or whose +// Kustomization has not handled its suspend yet, does not hold it: Flux may still apply a new artifact on its own. +func (r *ApplyTriggerReconciler) heldKustomizations( + ctx context.Context, + cluster string, +) (map[k8stypes.NamespacedName]bool, error) { + var triggers configbutleraiv1alpha3.FluxTriggerList + if err := r.List(ctx, &triggers); err != nil { + return nil, err + } + held := map[k8stypes.NamespacedName]bool{} + for i := range triggers.Items { + trigger := &triggers.Items[i] + if trigger.DeletionTimestamp.IsZero() && trigger.ClusterProvider() == cluster && heldBefore(trigger) { + held[trigger.Applier()] = true + } + } + return held, nil +} diff --git a/internal/controller/fluxtrigger_hold_test.go b/internal/controller/fluxtrigger_hold_test.go new file mode 100644 index 000000000..196f32eb9 --- /dev/null +++ b/internal/controller/fluxtrigger_hold_test.go @@ -0,0 +1,639 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + k8sschema "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +var checkoutKustomization = k8stypes.NamespacedName{Namespace: "flux-system", Name: "checkout"} + +// holdFixture is a FluxTrigger team-a/checkout over a simulated Flux cluster holding objs, with +// Events recorded. +type holdFixture struct { + r *ApplyTriggerReconciler + sim *fluxsim.Sim + recorder *record.FakeRecorder +} + +func newHoldFixture(t *testing.T, trigger *configbutleraiv1alpha3.FluxTrigger, objs ...runtime.Object) *holdFixture { + t.Helper() + if objs == nil { + objs = applierFluxObjects() + } + sim := applierSim(t, objs...) + r := triggerReconciler(triggerTestClient(t, trigger), configbutleraiv1alpha3.ApplyTriggerKindFlux, sim.Client, nil, + true) + recorder := record.NewFakeRecorder(32) + r.Recorder = recorder + return &holdFixture{r: r, sim: sim, recorder: recorder} +} + +// events drains the Events of the suspend lifecycle, leaving out the Ready transitions the status +// session records. +func (f *holdFixture) events() []string { + var out []string + for { + select { + case event := <-f.recorder.Events: + for _, reason := range []string{ + EventReasonAdopted, EventReasonResumedElsewhere, EventReasonReleased, EventReasonReleaseFailed, + EventReasonOperationUnfollowed, + } { + if strings.Contains(event, " "+reason+" ") { + out = append(out, event) + } + } + default: + return out + } + } +} + +func (f *holdFixture) trigger(t *testing.T) *configbutleraiv1alpha3.FluxTrigger { + t.Helper() + trigger := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), triggerKey, trigger)) + return trigger +} + +func suspendedSpec(ks *unstructured.Unstructured) bool { + value, _, _ := unstructured.NestedBool(ks.Object, "spec", "suspend") + return value +} + +// TestFluxTrigger_AdoptsItsKustomization: a Configured trigger suspends its Kustomization under GitOps +// Reverser's field manager, and is Ready only once that suspend's token is handled, so nothing that +// ran before the adoption is still running. +func TestFluxTrigger_AdoptsItsKustomization(t *testing.T) { + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + + got := reconcileTrigger(t, f.r).(*configbutleraiv1alpha3.FluxTrigger) + + held := findCondition(got.Status.Conditions, ApplyTriggerConditionHeld) + require.NotNil(t, held) + assert.Equal(t, metav1.ConditionTrue, held.Status, held.Message) + assert.Equal(t, ReasonSuspended, held.Reason) + assert.True(t, conditionIsTrue(got.Status.Conditions, ConditionTypeReady)) + ks := f.sim.Kustomization(checkoutKustomization) + assert.True(t, suspendedSpec(ks)) + assert.Contains(t, kustomizationManagers(ks), applier.SuspendFieldManager, + "held under GitOps Reverser's field manager") + assert.Nil(t, got.Status.Operation, "an adoption is no operation") + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Normal Adopted suspended Kustomization flux-system/checkout"), + events[0]) +} + +// TestFluxTrigger_AnExternalResumeIsSuspendedAgain is decision 3: `flux resume` on a held +// Kustomization is suspended again with a token, every time, with a Normal Event each time, and +// recorded as an Unresolved operation that completes once nothing that resume started is running. +func TestFluxTrigger_AnExternalResumeIsSuspendedAgain(t *testing.T) { + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + reconcileTrigger(t, f.r) + f.events() + + for round := range 2 { + patch := []byte(`{"spec":{"suspend":false}}`) + _, err := f.sim.Client.Resource(fluxsim.KustomizationGVR()).Namespace("flux-system"). + Patch(context.Background(), "checkout", k8stypes.MergePatchType, patch, metav1.PatchOptions{FieldManager: "flux"}) + require.NoError(t, err) + require.Eventually(t, func() bool { return len(f.sim.Applied()) == round+1 }, 5*time.Second, + time.Millisecond, "Flux applies the moment it is resumed") + + _, err = f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + recorded := f.trigger(t).Status.Operation + require.NotNil(t, recorded, "round %d", round) + assert.Equal(t, configbutleraiv1alpha3.ApplyOperationUnresolved, recorded.Phase) + assert.NotEmpty(t, recorded.SuspendToken) + assert.Empty(t, recorded.Token, "no request of GitOps Reverser's") + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) + events := f.events() + require.Len(t, events, 1, "an Event every time") + assert.True(t, strings.HasPrefix(events[0], "Normal ResumedElsewhere Kustomization flux-system/checkout "+ + "was resumed outside GitOps Reverser"), events[0]) + + got := reconcileTrigger(t, f.r).(*configbutleraiv1alpha3.FluxTrigger) + require.NotNil(t, got.Status.Operation) + assert.Equal(t, configbutleraiv1alpha3.ApplyOperationSucceeded, got.Status.Operation.Phase, + "complete once the suspend's token is handled") + assert.NotNil(t, got.Status.Operation.FinishedAt) + assert.Equal(t, recorded.SuspendToken, got.Status.Operation.SuspendToken) + assert.True(t, conditionIsTrue(got.Status.Conditions, ApplyTriggerConditionHeld)) + } + assert.Len(t, f.sim.Applied(), 2, "each resume applied once, unchecked; that is what the record says") +} + +// TestFluxTrigger_LeavesABranchWorkersOperationAlone: a worker records its operation before it +// resumes the Kustomization, so a resumed Kustomization with an operation of the worker's in flight +// is that operation, not an external resume; the worker suspends it again itself. +func TestFluxTrigger_LeavesABranchWorkersOperationAlone(t *testing.T) { + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + trigger.Finalizers = []string{configbutleraiv1alpha3.FluxTriggerFinalizer} + trigger.Status.Conditions = []metav1.Condition{{ + Type: ApplyTriggerConditionHeld, Status: metav1.ConditionTrue, Reason: ReasonSuspended, + LastTransitionTime: metav1.Now(), + }} + trigger.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationRunning, Reason: configbutleraiv1alpha3.ApplyReasonRefusal, + Token: "T1", RequestedAt: metav1.Now(), + } + f := newHoldFixture(t, trigger) + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "left resumed for the worker") + assert.Empty(t, f.events()) + assert.Equal(t, "T1", f.trigger(t).Status.Operation.Token) +} + +// followerStub answers Following for the operations it lists, by token. +type followerStub map[string]bool + +func (f followerStub) Following(_ git.TriggerRef, token string) bool { return f[token] } + +// workerOperationTrigger is a held trigger whose worker recorded operation T1 in flight, the +// Kustomization left resumed for it. +func workerOperationTrigger() *configbutleraiv1alpha3.FluxTrigger { + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + trigger.Finalizers = []string{configbutleraiv1alpha3.FluxTriggerFinalizer} + trigger.Status.Conditions = []metav1.Condition{{ + Type: ApplyTriggerConditionHeld, Status: metav1.ConditionTrue, Reason: ReasonSuspended, + LastTransitionTime: metav1.Now(), + }} + trigger.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationRunning, Reason: configbutleraiv1alpha3.ApplyReasonRefusal, + Token: "T1", RequestedAt: metav1.Now(), + } + return trigger +} + +// TestFluxTrigger_AnOperationNobodyFollowsIsSuspended is review 4's B backstop: the reconciler left a +// Kustomization resumed for any worker operation in flight, for ever, though nothing followed it +// (abandoned, undelivered, lost at startup). Now it leaves it only while this process follows the +// operation, or for a grace after it first saw nobody following it; past that it suspends the +// Kustomization, and completes the operation Failed once the suspend is handled. +func TestFluxTrigger_AnOperationNobodyFollowsIsSuspended(t *testing.T) { + f := newHoldFixture(t, workerOperationTrigger()) + f.r.UnfollowedGrace = time.Nanosecond + f.r.Follower = followerStub{"T1": true} + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "followed: left to the worker") + applying := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, f.r.Get(context.Background(), triggerKey, applying)) + held := findCondition(applying.Status.Conditions, ApplyTriggerConditionHeld) + require.NotNil(t, held) + assert.Equal(t, metav1.ConditionTrue, held.Status, "an operation of ours in flight is held") + assert.Equal(t, ReasonApplying, held.Reason) + + f.r.Follower = followerStub{} + got := reconcileTrigger(t, f.r).(*configbutleraiv1alpha3.FluxTrigger) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "followed by nobody: suspended") + require.NotNil(t, got.Status.Operation) + assert.Equal(t, configbutleraiv1alpha3.ApplyOperationFailed, got.Status.Operation.Phase) + assert.Equal(t, "T1", got.Status.Operation.Token, "the worker's operation, completed") + assert.NotEmpty(t, got.Status.Operation.SuspendToken) + assert.NotNil(t, got.Status.Operation.FinishedAt) + assert.Contains(t, got.Status.Operation.Message, "followed by nobody") + assert.True(t, conditionIsTrue(got.Status.Conditions, ApplyTriggerConditionHeld)) + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Warning OperationUnfollowed the Refusal apply"), events[0]) +} + +// operationFlipper is an APIReader whose reads of the trigger show its operation in flight until the +// Kustomization was read to be held (the second GET of it, after the Configured check), and completed +// after, as a worker that completes it in between leaves it. +type operationFlipper struct { + ctrlclient.Reader + + kustomizationReads atomic.Int32 +} + +func (o *operationFlipper) Get( + ctx context.Context, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption, +) error { + if err := o.Reader.Get(ctx, key, obj, opts...); err != nil { + return err + } + if trigger, ok := obj.(*configbutleraiv1alpha3.FluxTrigger); ok && trigger.Status.Operation != nil && + o.kustomizationReads.Load() >= 2 { + trigger.Status.Operation.Phase = configbutleraiv1alpha3.ApplyOperationSucceeded + } + return nil +} + +// TestFluxTrigger_AnOperationThatCompletesWhileReadIsNoExternalResume is review 4's F, the +// reconciler's half: the Kustomization read resumed for a worker's operation, which completed before +// the operation was read, looked like an external resume. Now the operation is read before and +// after the Kustomization, and a move between the two reads again. +func TestFluxTrigger_AnOperationThatCompletesWhileReadIsNoExternalResume(t *testing.T) { + f := newHoldFixture(t, workerOperationTrigger()) + flipper := &operationFlipper{Reader: f.r.Client} + f.r.APIReader = flipper + f.sim.Client.PrependReactor("get", "kustomizations", func(k8stesting.Action) (bool, runtime.Object, error) { + flipper.kustomizationReads.Add(1) + return false, nil, nil + }) + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "nothing suspended on a moving read") + assert.Empty(t, f.events(), "and no external resume recorded") + assert.Equal(t, "T1", f.trigger(t).Status.Operation.Token) +} + +// TestFluxTrigger_ASuspendAppliedElsewhereIsNeverHeld: a Kustomization whose manifest sets +// spec.suspend, applied by another manager, is Configured=False and is never suspended: the next +// apply of that manifest would resume it. +func TestFluxTrigger_ASuspendAppliedElsewhereIsNeverHeld(t *testing.T) { + objs := applierFluxObjects() + ks := fluxsim.OwnSuspend(t, objs[0].(*unstructured.Unstructured), false, git.FluxFieldManager, + metav1.ManagedFieldsOperationApply) + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), ks, objs[1]) + + got := reconcileTrigger(t, f.r).(*configbutleraiv1alpha3.FluxTrigger) + + configured := findCondition(got.Status.Conditions, ApplyTriggerConditionConfigured) + require.NotNil(t, configured) + assert.Equal(t, applier.ReasonSuspendOwnedElsewhere, configured.Reason) + assert.True(t, conditionIsTrue(got.Status.Conditions, ConditionTypeStalled), "a human fixes the manifest") + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) +} + +// TestFluxTrigger_DeletionHandsTheKustomizationBack: deleting the trigger suspends its Kustomization +// once more when it is resumed, waits until that suspend is handled, and then releases spec.suspend, +// so Flux applies on its own again; then the finalizer goes. +func TestFluxTrigger_DeletionHandsTheKustomizationBack(t *testing.T) { + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + reconcileTrigger(t, f.r) + f.events() + // A worker's apply is running as the trigger is deleted. + patch := []byte(`{"spec":{"suspend":false}}`) + _, err := f.sim.Client.Resource(fluxsim.KustomizationGVR()).Namespace("flux-system").Patch(context.Background(), + "checkout", k8stypes.MergePatchType, patch, metav1.PatchOptions{FieldManager: applier.SuspendFieldManager}) + require.NoError(t, err) + require.NoError(t, f.r.Delete(context.Background(), f.trigger(t))) + + require.Eventually(t, func() bool { + result, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + if err := f.r.Get( + context.Background(), + triggerKey, + &configbutleraiv1alpha3.FluxTrigger{}, + ); apierrors.IsNotFound( + err, + ) { + return true + } + assert.Equal(t, applyTriggerSuspendingInterval, result.RequeueAfter) + return false + }, 5*time.Second, 5*time.Millisecond, "the finalizer goes once the Kustomization is handed back") + + ks := f.sim.Kustomization(checkoutKustomization) + assert.False(t, suspendedSpec(ks), "resumed: Flux owns its applies again") + for _, entry := range ks.GetManagedFields() { + assert.NotEqual(t, applier.SuspendFieldManager, entry.Manager, "nothing of GitOps Reverser's is left") + } + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Normal Released handed Kustomization flux-system/checkout back"), + events[0]) +} + +// TestFluxTrigger_DeletionLetsGoOfWhatIsGone: a Kustomization or a ClusterProvider that is gone has +// nothing to hand back, and must not hold the trigger. +func TestFluxTrigger_DeletionLetsGoOfWhatIsGone(t *testing.T) { + for name, cluster := range map[string]string{ + "the Kustomization is gone": configbutleraiv1alpha3.DefaultClusterProviderName, + "the ClusterProvider is gone": "absent", + } { + t.Run(name, func(t *testing.T) { + trigger := testFluxTrigger(cluster) + trigger.Finalizers = []string{configbutleraiv1alpha3.FluxTriggerFinalizer} + sim := applierSim(t) + r := triggerReconciler(triggerTestClient(t, trigger), configbutleraiv1alpha3.ApplyTriggerKindFlux, + sim.Client, nil, true) + r.ApplierClients = func(context.Context, string) (dynamic.Interface, error) { return sim.Client, nil } + require.NoError(t, r.Delete(context.Background(), trigger)) + + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + err = r.Get(context.Background(), triggerKey, &configbutleraiv1alpha3.FluxTrigger{}) + assert.True(t, apierrors.IsNotFound(err), "the finalizer is gone: %v", err) + }) + } +} + +// TestFluxTrigger_AGoneClusterProviderStillHandsBack is review 7's C1: a folder holding both a +// ClusterProvider and a FluxTrigger is removed, and the provider goes first. The hand-back was +// skipped with an Info log line, and the Kustomization stayed suspended under gitops-reverser for +// good, though for a provider without kubeConfig its cluster is still reachable. Now the release +// goes on through whatever client still resolves, and only when none does is the trigger let go, +// with a Warning ReleaseFailed saying the Kustomization is left suspended. +func TestFluxTrigger_AGoneClusterProviderStillHandsBack(t *testing.T) { + held := func(t *testing.T) *holdFixture { + t.Helper() + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + reconcileTrigger(t, f.r) + f.events() + require.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization))) + provider := &configbutleraiv1alpha3.ClusterProvider{} + require.NoError(t, f.r.Get(context.Background(), + ctrlclient.ObjectKey{Name: configbutleraiv1alpha3.DefaultClusterProviderName}, provider)) + require.NoError(t, f.r.Delete(context.Background(), provider)) + require.NoError(t, f.r.Delete(context.Background(), f.trigger(t))) + return f + } + gone := func(t *testing.T, f *holdFixture) { + t.Helper() + require.Eventually(t, func() bool { + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + err = f.r.Get(context.Background(), triggerKey, &configbutleraiv1alpha3.FluxTrigger{}) + return apierrors.IsNotFound(err) + }, 5*time.Second, 5*time.Millisecond, "the finalizer goes") + } + + t.Run("its cluster still answers, so the Kustomization is handed back", func(t *testing.T) { + f := held(t) + gone(t, f) + assert.False(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "handed back to Flux") + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Normal Released handed Kustomization flux-system/checkout back"), + events[0]) + }) + t.Run("its cluster no longer answers, so the trigger lets go loudly", func(t *testing.T) { + f := held(t) + f.r.ApplierClients = func(context.Context, string) (dynamic.Interface, error) { + return nil, errors.New(`ClusterProvider "default" not found`) + } + gone(t, f) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "left suspended, as the Event says") + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Warning ReleaseFailed could not hand Kustomization "+ + "flux-system/checkout back to Flux: its ClusterProvider \"default\" is gone; it is left suspended, "+ + "and `flux resume` resumes it"), events[0]) + }) +} + +// TestFluxTrigger_TakesOverASuspendBySomebodyElse: a Kustomization `flux suspend` suspended is not +// held: that write may have let a reconcile run, so the trigger takes the field over with a token of +// its own and is Ready once it is handled. +func TestFluxTrigger_TakesOverASuspendBySomebodyElse(t *testing.T) { + objs := applierFluxObjects() + ks := fluxsim.OwnSuspend(t, objs[0].(*unstructured.Unstructured), true, "flux", + metav1.ManagedFieldsOperationUpdate) + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), ks, objs[1]) + + got := reconcileTrigger(t, f.r).(*configbutleraiv1alpha3.FluxTrigger) + + assert.True(t, conditionIsTrue(got.Status.Conditions, ApplyTriggerConditionHeld)) + assert.Contains(t, kustomizationManagers(f.sim.Kustomization(checkoutKustomization)), applier.SuspendFieldManager) + assert.Empty(t, f.events(), "neither an adoption of a running Kustomization nor an external resume") +} + +// TestFluxTrigger_AHandBackItMayNotMakeLetsGo: an identity that may not write the Kustomization +// cannot hand it back; the trigger goes anyway, with a Warning that says what is left behind. +func TestFluxTrigger_AHandBackItMayNotMakeLetsGo(t *testing.T) { + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + reconcileTrigger(t, f.r) + f.events() + f.sim.Client.PrependReactor("patch", "kustomizations", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(fluxsim.KustomizationGVR().GroupResource(), "checkout", + errors.New("no patch for you")) + }) + require.NoError(t, f.r.Delete(context.Background(), f.trigger(t))) + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + err = f.r.Get(context.Background(), triggerKey, &configbutleraiv1alpha3.FluxTrigger{}) + assert.True(t, apierrors.IsNotFound(err), "the trigger is let go: %v", err) + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Warning ReleaseFailed could not hand Kustomization"), events[0]) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), "left suspended, as the Event says") +} + +// kustomizationManagers lists a Kustomization's field managers. +func kustomizationManagers(ks *unstructured.Unstructured) []string { + var managers []string + for _, entry := range ks.GetManagedFields() { + managers = append(managers, entry.Manager) + } + return managers +} + +// TestFluxTrigger_PublishesTheSourcesOtherConsumers is decision 2: the consumers of the +// GitRepository that no FluxTrigger holds are listed, information only, and the source is prefetched +// only when there are none. +func TestFluxTrigger_PublishesTheSourcesOtherConsumers(t *testing.T) { + objs := applierFluxObjects() + infra := objs[0].(*unstructured.Unstructured).DeepCopy() + infra.SetName("infra") + release := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "helm.toolkit.fluxcd.io/v2", "kind": "HelmRelease", + "metadata": map[string]any{"namespace": "apps", "name": "podinfo"}, + "spec": map[string]any{"chart": map[string]any{"spec": map[string]any{ + "chart": "./charts/podinfo", + "sourceRef": map[string]any{"kind": "GitRepository", "name": "platform", "namespace": "flux-system"}, + }}}, + }} + f := newHoldFixture(t, testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), + append(objs, infra, release)...) + + source := reconcileTrigger(t, f.r).(*configbutleraiv1alpha3.FluxTrigger).Status.Source + require.NotNil(t, source) + assert.Equal(t, []meta.NamespacedObjectKindReference{ + {Kind: "HelmRelease", Namespace: "apps", Name: "podinfo"}, + {Kind: "Kustomization", Namespace: "flux-system", Name: "infra"}, + }, source.OtherConsumers, "itself is not another consumer") + assert.False(t, source.Prefetch) + assert.True(t, conditionIsTrue(f.trigger(t).Status.Conditions, ConditionTypeReady), "information only") + + // The fourth review's finding C: a trigger holds its Kustomization only once it says so + // (Held=True). One that is not Configured (SuspendOwnedElsewhere), or still suspending, leaves + // a Kustomization Flux applies on its own. + infraTrigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + infraTrigger.Name, infraTrigger.Spec.KustomizationRef.Name = "infra", "infra" + require.NoError(t, f.r.Create(context.Background(), infraTrigger)) + infraOther := []meta.NamespacedObjectKindReference{ + {Kind: "HelmRelease", Namespace: "apps", Name: "podinfo"}, + {Kind: "Kustomization", Namespace: "flux-system", Name: "infra"}, + } + for _, suspended := range []metav1.Condition{ + {Type: ApplyTriggerConditionHeld, Status: metav1.ConditionUnknown, Reason: ReasonNotConfigured}, + {Type: ApplyTriggerConditionHeld, Status: metav1.ConditionFalse, Reason: ReasonSuspending}, + } { + setTriggerCondition(t, f.r, "infra", suspended) + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + assert.Equal(t, infraOther, f.trigger(t).Status.Source.OtherConsumers, + "a trigger that is %s does not hold its Kustomization", suspended.Reason) + } + + setTriggerCondition(t, f.r, "infra", metav1.Condition{ + Type: ApplyTriggerConditionHeld, Status: metav1.ConditionTrue, Reason: ReasonSuspended, + }) + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + assert.Equal(t, []meta.NamespacedObjectKindReference{{Kind: "HelmRelease", Namespace: "apps", Name: "podinfo"}}, + f.trigger(t).Status.Source.OtherConsumers, "a Kustomization another trigger holds is not listed") + + require.NoError(t, f.sim.Client.Resource(k8sschema.GroupVersionResource{ + Group: "helm.toolkit.fluxcd.io", Version: "v2", Resource: "helmreleases", + }).Namespace("apps").Delete(context.Background(), "podinfo", metav1.DeleteOptions{})) + _, err = f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + got := f.trigger(t).Status.Source + assert.Empty(t, got.OtherConsumers) + assert.True(t, got.Prefetch, "every consumer is held: the source is prefetched") + + trigger := f.trigger(t) + f.r.publishConsumers(context.Background(), trigger, + conditionValue{Status: metav1.ConditionFalse, Reason: ReasonSuspending}) + assert.Empty(t, trigger.Status.Source.OtherConsumers) + assert.False(t, trigger.Status.Source.Prefetch, "nor while its own Kustomization is not held yet") +} + +// setTriggerCondition publishes a condition on the FluxTrigger team-a/name. +func setTriggerCondition(t *testing.T, c ctrlclient.Client, name string, condition metav1.Condition) { + t.Helper() + trigger := &configbutleraiv1alpha3.FluxTrigger{} + require.NoError(t, c.Get(context.Background(), k8stypes.NamespacedName{Namespace: "team-a", Name: name}, trigger)) + condition.LastTransitionTime = metav1.Now() + trigger.Status.Conditions = []metav1.Condition{condition} + require.NoError(t, c.Status().Update(context.Background(), trigger)) +} + +// TestFluxTrigger_CompletesTheLiveExternalOperation is the fourth review's finding K: the operation +// an external resume was recorded as is completed from the uncached read, which is what the hold +// decided on. The cached trigger may not show it yet, and completing that one dereferenced nil. +func TestFluxTrigger_CompletesTheLiveExternalOperation(t *testing.T) { + objs := applierFluxObjects() + ks := fluxsim.OwnSuspend(t, objs[0].(*unstructured.Unstructured), true, applier.SuspendFieldManager, + metav1.ManagedFieldsOperationApply) + ks.SetAnnotations(map[string]string{meta.ReconcileRequestAnnotation: "T2"}) + require.NoError(t, unstructured.SetNestedField(ks.Object, "T2", "status", "lastHandledReconcileAt")) + cached := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + cached.Finalizers = []string{configbutleraiv1alpha3.FluxTriggerFinalizer} + f := newHoldFixture(t, cached, ks, objs[1]) + live := cached.DeepCopy() + live.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationUnresolved, SuspendToken: "T2", RequestedAt: metav1.Now(), + } + f.r.APIReader = triggerTestClient(t, live) + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + + got := f.trigger(t) + require.NotNil(t, got.Status.Operation, "the live operation is completed") + assert.True(t, got.Status.Operation.Phase.Completed(), got.Status.Operation.Phase) + assert.Equal(t, "T2", got.Status.Operation.SuspendToken) + assert.NotNil(t, got.Status.Operation.FinishedAt) +} + +// TestFluxTrigger_ASuspendItMayNotMakeLetsGo is the fourth review's finding D: a resumed +// Kustomization the identity may not suspend again cannot be handed back either, so the trigger goes, +// with a Warning, as it does for a release it may not make. It used to retry forever. +func TestFluxTrigger_ASuspendItMayNotMakeLetsGo(t *testing.T) { + objs := applierFluxObjects() + ks := fluxsim.OwnSuspend(t, objs[0].(*unstructured.Unstructured), false, applier.SuspendFieldManager, + metav1.ManagedFieldsOperationApply) + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + trigger.Finalizers = []string{configbutleraiv1alpha3.FluxTriggerFinalizer} + f := newHoldFixture(t, trigger, ks, objs[1]) + f.sim.Client.PrependReactor("patch", "kustomizations", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(fluxsim.KustomizationGVR().GroupResource(), "checkout", + errors.New("no patch for you")) + }) + require.NoError(t, f.r.Delete(context.Background(), f.trigger(t))) + + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + err = f.r.Get(context.Background(), triggerKey, &configbutleraiv1alpha3.FluxTrigger{}) + assert.True(t, apierrors.IsNotFound(err), "the trigger is let go: %v", err) + events := f.events() + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Warning ReleaseFailed could not hand Kustomization"), events[0]) +} + +// TestFluxTrigger_AHandBackThatNeverEndsLetsGoAtTheDeadline is the fourth review's finding D: with +// kustomize-controller down, the suspend of a hand-back is never handled. The trigger waits for it +// until releaseDeadline after its deletion, and then goes, with a Warning that says what the +// Kustomization is left as. +func TestFluxTrigger_AHandBackThatNeverEndsLetsGoAtTheDeadline(t *testing.T) { + for name, deleted := range map[string]time.Duration{ + "before the deadline": 0, + "past the deadline": releaseDeadline + time.Minute, + } { + t.Run(name, func(t *testing.T) { + objs := applierFluxObjects() + ks := fluxsim.OwnSuspend(t, objs[0].(*unstructured.Unstructured), true, applier.SuspendFieldManager, + metav1.ManagedFieldsOperationApply) + // No source: the simulated kustomize-controller never reconciles it, as one that is down. + unstructured.RemoveNestedField(ks.Object, "spec", "sourceRef") + ks.SetAnnotations(map[string]string{meta.ReconcileRequestAnnotation: "T2"}) + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + trigger.Finalizers = []string{configbutleraiv1alpha3.FluxTriggerFinalizer} + trigger.DeletionTimestamp = &metav1.Time{Time: time.Now()} + // The deadline counts from when Held turned Releasing, not from the deletion. + trigger.Status.Conditions = []metav1.Condition{{ + Type: ApplyTriggerConditionHeld, Status: metav1.ConditionFalse, Reason: ReasonReleasing, + LastTransitionTime: metav1.NewTime(time.Now().Add(-deleted)), + }} + f := newHoldFixture(t, trigger, ks, objs[1]) + + result, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: triggerKey}) + require.NoError(t, err) + err = f.r.Get(context.Background(), triggerKey, &configbutleraiv1alpha3.FluxTrigger{}) + events := f.events() + if deleted == 0 { + require.NoError(t, err, "still waiting for the suspend") + assert.Equal(t, applyTriggerSuspendingInterval, result.RequeueAfter) + assert.Empty(t, events) + return + } + assert.True(t, apierrors.IsNotFound(err), "the trigger is let go: %v", err) + require.Len(t, events, 1) + assert.True(t, strings.HasPrefix(events[0], "Warning ReleaseFailed gave up handing Kustomization "+ + "flux-system/checkout back to Flux 10m0s after it began: it is suspended, and has "+ + "not handled the suspend yet"), events[0]) + assert.True(t, suspendedSpec(f.sim.Kustomization(checkoutKustomization)), + "left suspended, as the Event says") + }) + } +} diff --git a/internal/controller/fluxtrigger_suspend_apiserver_test.go b/internal/controller/fluxtrigger_suspend_apiserver_test.go new file mode 100644 index 000000000..1885dec0e --- /dev/null +++ b/internal/controller/fluxtrigger_suspend_apiserver_test.go @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "encoding/json" + "fmt" + + fluxmeta "github.com/fluxcd/pkg/apis/meta" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/applier/fluxsim" +) + +// The ownership of spec.suspend on a real API server, which internal/applier/fluxsim models. A +// forced apply of the value a field holds does not take the field: it shares it with the manager +// that set it. So a suspend that takes over a `flux suspend` (a merge patch, an Update entry of +// manager "flux") co-owns spec.suspend, and an empty apply alone leaves it set. Release has to take +// the update's share first, or a hand-back leaves the Kustomization suspended while the Released +// Event says it resumed. +var _ = Describe("Kustomization spec.suspend on a real API server", func() { + It("is shared with a flux suspend, and Release still resumes it", func() { + ctx := context.Background() + _, err := envtest.InstallCRDs(cfg, envtest.CRDInstallOptions{ + CRDs: []*apiextensionsv1.CustomResourceDefinition{kustomizationCRD()}, + }) + Expect(err).NotTo(HaveOccurred()) + dyn, err := dynamic.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + const namespace = "suspend-ownership" + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})).To(Succeed()) + kustomizations := dyn.Resource(fluxsim.KustomizationGVR()).Namespace(namespace) + _, err = kustomizations.Create(ctx, &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kustomize.toolkit.fluxcd.io/v1", "kind": "Kustomization", + "metadata": map[string]any{"namespace": namespace, "name": "apps"}, + "spec": map[string]any{"path": "./apps"}, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + ref := k8stypes.NamespacedName{Namespace: namespace, Name: "apps"} + read := func() *unstructured.Unstructured { + ks, err := kustomizations.Get(ctx, ref.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return ks + } + + By("flux suspend: a merge patch, an Update entry") + _, err = kustomizations.Patch(ctx, ref.Name, k8stypes.MergePatchType, []byte(`{"spec":{"suspend":true}}`), + metav1.PatchOptions{FieldManager: "flux"}) + Expect(err).NotTo(HaveOccurred()) + Expect(suspendEntries(read())).To(ConsistOf("flux/Update")) + + By("our forced apply of the same value shares the field") + flux := applier.NewFlux(dyn) + _, err = flux.Suspend(ctx, ref) + Expect(err).NotTo(HaveOccurred()) + Expect(suspendEntries(read())).To(ConsistOf("flux/Update", applier.SuspendFieldManager+"/Apply")) + + By("Release takes the update's share, and resumes it") + keeper, err := flux.Release(ctx, ref) + Expect(err).NotTo(HaveOccurred()) + Expect(keeper).To(BeEmpty()) + released := read() + _, found, _ := unstructured.NestedFieldNoCopy(released.Object, "spec", "suspend") + Expect(found).To(BeFalse(), "spec.suspend is gone, so the Kustomization is resumed") + Expect(suspendEntries(released)).To(BeEmpty()) + Expect(released.GetAnnotations()).NotTo(HaveKey(fluxmeta.ReconcileRequestAnnotation)) + for _, entry := range released.GetManagedFields() { + Expect(entry.Manager).NotTo(Equal(applier.SuspendFieldManager), "nothing of GitOps Reverser's is left") + } + + By("an update of the value the field holds takes nothing") + _, err = flux.Suspend(ctx, ref) + Expect(err).NotTo(HaveOccurred()) + _, err = kustomizations.Patch(ctx, ref.Name, k8stypes.MergePatchType, []byte(`{"spec":{"suspend":true}}`), + metav1.PatchOptions{FieldManager: "flux"}) + Expect(err).NotTo(HaveOccurred()) + Expect(suspendEntries(read())).To(ConsistOf(applier.SuspendFieldManager + "/Apply")) + }) +}) + +// The preconditions a resume carries, on a real API server. Flux.resume reads the Kustomization +// again just before it resumes it, and applies spec.suspend: false with that read's uid and +// resourceVersion, so the API server refuses the resume of a Kustomization that changed since it +// was checked. internal/applier/fluxsim models the answers; these pin them: a resourceVersion that +// moved is a Conflict, another object's uid is Invalid, a deleted object's uid is a Conflict rather +// than a create, and neither precondition lands in managedFields. +var _ = Describe("A Kustomization resume guarded by uid and resourceVersion, on a real API server", func() { + const ( + repositoryURL = "https://example.com/apps.git" + branch = "main" + commit = "0123456789abcdef0123456789abcdef01234567" + ) + var ( + ctx context.Context + dyn dynamic.Interface + kustomizations dynamic.ResourceInterface + ref k8stypes.NamespacedName + ) + // setUp makes a namespace holding GitRepository apps, whose artifact holds commit, and + // Kustomization apps, which applies it and is suspended by GitOps Reverser. + setUp := func(namespace string) { + ctx = context.Background() + _, err := envtest.InstallCRDs(cfg, envtest.CRDInstallOptions{ + CRDs: []*apiextensionsv1.CustomResourceDefinition{kustomizationCRD(), gitRepositoryCRD()}, + }) + Expect(err).NotTo(HaveOccurred()) + dyn, err = dynamic.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})).To(Succeed()) + sources := dyn.Resource(fluxsim.GitRepositoryGVR()).Namespace(namespace) + source, err := sources.Create(ctx, &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "source.toolkit.fluxcd.io/v1", "kind": "GitRepository", + "metadata": map[string]any{"namespace": namespace, "name": "apps"}, + "spec": map[string]any{"url": repositoryURL, "ref": map[string]any{"branch": branch}}, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(unstructured.SetNestedField(source.Object, branch+"@sha1:"+commit, + "status", "artifact", "revision")).To(Succeed()) + _, err = sources.UpdateStatus(ctx, source, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + kustomizations = dyn.Resource(fluxsim.KustomizationGVR()).Namespace(namespace) + _, err = kustomizations.Create(ctx, &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "kustomize.toolkit.fluxcd.io/v1", "kind": "Kustomization", + "metadata": map[string]any{"namespace": namespace, "name": "apps"}, + "spec": map[string]any{ + "path": "./apps", + "sourceRef": map[string]any{"kind": "GitRepository", "name": "apps"}, + }, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + ref = k8stypes.NamespacedName{Namespace: namespace, Name: "apps"} + _, err = applier.NewFlux(dyn).Suspend(ctx, ref) + Expect(err).NotTo(HaveOccurred()) + } + // request asks Flux.Request for an apply of commit, with held run where the trigger's hold is + // checked: after the resume's read of the Kustomization, before its write. + request := func(held func(context.Context) error) (applier.Started, error) { + return applier.NewFlux(dyn).Request(ctx, ref, applier.Ask{ + Want: applier.Binding{RepositoryURL: repositoryURL, Branch: branch}, + Commit: commit, + Reason: configbutleraiv1alpha3.ApplyReasonRepair, + Token: "guarded-resume", + Held: held, + }) + } + read := func() *unstructured.Unstructured { + ks, err := kustomizations.Get(ctx, ref.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return ks + } + suspendedNow := func() bool { + value, _, _ := unstructured.NestedBool(read().Object, "spec", "suspend") + return value + } + + It("resumes the Kustomization it read, and keeps uid and resourceVersion out of managedFields", func() { + setUp("guarded-resume") + started, err := request(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(started.Outcome).To(Equal(applier.OutcomeResumed)) + Expect(started.Unconfirmed).NotTo(HaveOccurred()) + Expect(suspendedNow()).To(BeFalse()) + + var ours []metav1.ManagedFieldsEntry + for _, entry := range read().GetManagedFields() { + if entry.Manager == applier.SuspendFieldManager { + ours = append(ours, entry) + } + } + Expect(ours).To(HaveLen(1)) + var fields map[string]any + Expect(json.Unmarshal(ours[0].FieldsV1.GetRawBytes(), &fields)).To(Succeed()) + Expect(fields).To(HaveKey("f:spec"), "the resume owns spec.suspend") + metadata, _ := fields["f:metadata"].(map[string]any) + Expect(metadata).NotTo(HaveKey("f:uid"), "a precondition is no field the resume owns") + Expect(metadata).NotTo(HaveKey("f:resourceVersion"), "a precondition is no field the resume owns") + }) + + It("does not resume a Kustomization written to after the resume's read", func() { + setUp("guarded-resume-written") + writes := 0 + _, err := request(func(ctx context.Context) error { + writes++ + _, err := kustomizations.Patch(ctx, ref.Name, k8stypes.MergePatchType, + fmt.Appendf(nil, `{"metadata":{"labels":{"write":"%d"}}}`, writes), metav1.PatchOptions{}) + return err + }) + Expect(apierrors.IsConflict(err)).To(BeTrue(), "the API server refuses the stale resourceVersion: %v", err) + Expect(writes).To(BeNumerically(">", 1), "a Conflict reads the Kustomization again") + Expect(suspendedNow()).To(BeTrue(), "nothing was resumed") + }) + + It("does not re-create a Kustomization deleted after the resume's read", func() { + setUp("guarded-resume-deleted") + _, err := request(func(ctx context.Context) error { + return kustomizations.Delete(ctx, ref.Name, metav1.DeleteOptions{}) + }) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "the read after the Conflict finds it gone: %v", err) + _, err = kustomizations.Get(ctx, ref.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "the resume did not re-create it") + }) + + // The answers themselves, to the apply the resume sends. applier's hold builds that apply and is + // not exported, so guardedResume sends the same body: apiVersion, kind, namespace and name, the + // preconditions, the requestedAt token and spec.suspend: false, forced under SuspendFieldManager. + It("answers a moved resourceVersion with Conflict, another uid with Invalid, a deleted object with Conflict", + func() { + setUp("guarded-resume-answers") + guardedResume := func(uid k8stypes.UID, resourceVersion string) error { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(fluxsim.KustomizationGVR().GroupVersion().String()) + obj.SetKind("Kustomization") + obj.SetNamespace(ref.Namespace) + obj.SetName(ref.Name) + obj.SetUID(uid) + obj.SetResourceVersion(resourceVersion) + obj.SetAnnotations(map[string]string{fluxmeta.ReconcileRequestAnnotation: "guarded-resume"}) + Expect(unstructured.SetNestedField(obj.Object, false, "spec", "suspend")).To(Succeed()) + _, err := kustomizations.Apply(ctx, ref.Name, obj, + metav1.ApplyOptions{FieldManager: applier.SuspendFieldManager, Force: true}) + return err + } + + By("a resourceVersion that moved: Conflict") + checked := read() + _, err := kustomizations.Patch(ctx, ref.Name, k8stypes.MergePatchType, + []byte(`{"metadata":{"labels":{"write":"1"}}}`), metav1.PatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + err = guardedResume(checked.GetUID(), checked.GetResourceVersion()) + Expect(apierrors.IsConflict(err)).To(BeTrue(), "got %v", err) + Expect(suspendedNow()).To(BeTrue()) + + By("another object's uid, over the current resourceVersion: Invalid") + err = guardedResume("00000000-0000-0000-0000-000000000000", read().GetResourceVersion()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "got %v", err) + Expect(suspendedNow()).To(BeTrue()) + + By("a deleted object's uid: Conflict, and nothing is created") + checked = read() + Expect(kustomizations.Delete(ctx, ref.Name, metav1.DeleteOptions{})).To(Succeed()) + err = guardedResume(checked.GetUID(), checked.GetResourceVersion()) + Expect(apierrors.IsConflict(err)).To(BeTrue(), "got %v", err) + Expect(err.Error()).To(ContainSubstring("uid mismatch")) + _, err = kustomizations.Get(ctx, ref.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue(), "the resume did not re-create it") + + By("the resourceVersion alone, which is why the resume carries the uid: it re-creates the object") + Expect(guardedResume("", checked.GetResourceVersion())).To(Succeed()) + Expect(read().GetUID()).NotTo(Equal(checked.GetUID())) + }) +}) + +// suspendEntries lists the managedFields entries owning a Kustomization's spec.suspend, as +// "manager/operation". +func suspendEntries(ks *unstructured.Unstructured) []string { + var entries []string + for _, entry := range ks.GetManagedFields() { + if entry.FieldsV1 == nil { + continue + } + var fields map[string]any + Expect(json.Unmarshal(entry.FieldsV1.GetRawBytes(), &fields)).To(Succeed()) + if _, found, _ := unstructured.NestedFieldNoCopy(fields, "f:spec", "f:suspend"); found { + entries = append(entries, entry.Manager+"/"+string(entry.Operation)) + } + } + return entries +} + +// kustomizationCRD is as much of Flux's Kustomization CRD as spec.suspend's ownership and a +// request need: a structural schema whose spec.suspend is a boolean, so the API server tracks it as +// a field, and which keeps spec.sourceRef. +func kustomizationCRD() *apiextensionsv1.CustomResourceDefinition { + preserve := true + gvr := fluxsim.KustomizationGVR() + return &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: gvr.Resource + "." + gvr.Group}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: gvr.Group, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: gvr.Resource, Singular: "kustomization", Kind: "Kustomization", ListKind: "KustomizationList", + }, + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: gvr.Version, Served: true, Storage: true, + Subresources: &apiextensionsv1.CustomResourceSubresources{ + Status: &apiextensionsv1.CustomResourceSubresourceStatus{}, + }, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": {Type: "object", Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "suspend": {Type: "boolean"}, + "path": {Type: "string"}, + "sourceRef": {Type: "object", XPreserveUnknownFields: &preserve}, + }}, + "status": {Type: "object", XPreserveUnknownFields: &preserve}, + }, + }}, + }}, + }, + } +} + +// gitRepositoryCRD is as much of Flux's GitRepository CRD as a request reads: spec and status kept +// as written, and a status subresource for the artifact. +func gitRepositoryCRD() *apiextensionsv1.CustomResourceDefinition { + preserve := true + gvr := fluxsim.GitRepositoryGVR() + return &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: gvr.Resource + "." + gvr.Group}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: gvr.Group, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: gvr.Resource, Singular: "gitrepository", Kind: "GitRepository", ListKind: "GitRepositoryList", + }, + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: gvr.Version, Served: true, Storage: true, + Subresources: &apiextensionsv1.CustomResourceSubresources{ + Status: &apiextensionsv1.CustomResourceSubresourceStatus{}, + }, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": {Type: "object", XPreserveUnknownFields: &preserve}, + "status": {Type: "object", XPreserveUnknownFields: &preserve}, + }, + }}, + }}, + }, + } +} diff --git a/internal/controller/gittarget_applier.go b/internal/controller/gittarget_applier.go deleted file mode 100644 index de6a477f6..000000000 --- a/internal/controller/gittarget_applier.go +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "fmt" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" - - configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/applier" - "github.com/ConfigButler/gitops-reverser/internal/authz" - "github.com/ConfigButler/gitops-reverser/internal/git" - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -// GitTargetConditionApplierConfigured reports whether spec.applier names an object GitOps Reverser -// can ask to apply: it exists in the applier's cluster, applies this target's repository and branch, -// may be written, and (an Argo CD Application) has automated sync off. Present only while -// spec.applier is set. -const GitTargetConditionApplierConfigured = "ApplierConfigured" - -// applierCheckTimeout bounds the ApplierConfigured check: at most two GETs and two access reviews -// against the applier's cluster. Its client has no request timeout, because it also carries the watches, and -// this controller runs one reconcile worker, so a source API server that accepts the connection and -// never answers would otherwise stop every GitTarget's reconcile, not only this one's. -const applierCheckTimeout = 10 * time.Second - -// ApplyRequestLedger is what the GitTarget controller reads to publish status.applier: the last -// apply request delivered for a GitTarget, for the incarnation with this uid. Forget drops a -// GitTarget that is gone. The applier dispatcher implements it. -type ApplyRequestLedger interface { - LastRequest(ref types.ResourceReference, uid k8stypes.UID) (configbutleraiv1alpha3.GitTargetApplierStatus, bool) - Forget(ref types.ResourceReference) -} - -// observeApplier publishes status.applier and the ApplierConfigured condition, and returns the -// condition for the readiness trio. present is false when the target names no applier, in which -// case both are removed. -// -// The check reads the applier on this reconcile rather than watching it. Flux or Argo CD may not be -// installed in the applier's cluster at all, and an informer on a kind that is not served would be a failure to -// report rather than a check that answers "not found"; the cadence is the target's own. -func (r *GitTargetReconciler) observeApplier( - ctx context.Context, - st *reconcileStatus, - target *configbutleraiv1alpha3.GitTarget, - gitProvider *configbutleraiv1alpha3.GitProvider, -) (conditionValue, bool) { - if target.Spec.Applier == nil { - st.remove(GitTargetConditionApplierConfigured) - target.Status.Applier = nil - return conditionValue{}, false - } - - if r.ApplyRequests != nil { - ref := types.NewResourceReference(target.Name, target.Namespace) - if last, ok := r.ApplyRequests.LastRequest(ref, target.UID); ok { - target.Status.Applier = &last - } - } - - value := r.applierVerdict(ctx, target, gitProvider) - st.setValue(GitTargetConditionApplierConfigured, value) - return value, true -} - -// applierVerdict checks the applier in its cluster (spec.applier.clusterProviderRef, by default the -// source cluster). A check that could not be made keeps the verdict already published, since a blip -// reading that cluster says nothing about the applier; with nothing published it is Unknown. -func (r *GitTargetReconciler) applierVerdict( - ctx context.Context, - target *configbutleraiv1alpha3.GitTarget, - gitProvider *configbutleraiv1alpha3.GitProvider, -) conditionValue { - applierRef := git.ApplierOf(target) - ref := k8stypes.NamespacedName{Namespace: applierRef.Namespace, Name: applierRef.Name} - kind := applierRef.Kind - ctx, cancel := context.WithTimeout(ctx, applierCheckTimeout) - defer cancel() - if r.ApplierClients == nil || gitProvider == nil { - return conditionValue{ - Status: metav1.ConditionUnknown, - Reason: GitTargetReasonNotChecked, - Message: fmt.Sprintf("%s %s has not been checked: no cluster client is wired", kind, ref), - } - } - cluster := target.ApplierCluster() - if cluster != target.SourceCluster() { - decision, err := authz.ApplierAdmitted(ctx, r.Client, target) - if err != nil { - return keptOrUnknown(target, fmt.Sprintf("Cannot check ClusterProvider %q: %v", cluster, err)) - } - if !decision.Allowed { - return conditionValue{Status: metav1.ConditionFalse, Reason: decision.Reason, Message: decision.Message} - } - } - client, err := r.ApplierClients(ctx, cluster) - if err != nil { - return keptOrUnknown(target, - fmt.Sprintf("Cannot reach cluster %q to check %s %s: %v", cluster, kind, ref, err)) - } - newApplier := r.newApplier - if newApplier == nil { - newApplier = applier.NewClient - } - verdict := newApplier(kind, client).Validate(ctx, ref, - applier.Binding{RepositoryURL: gitProvider.Spec.URL, Branch: target.Spec.Branch}) - switch { - case verdict.OK: - return conditionValue{ - Status: metav1.ConditionTrue, - Reason: ReasonSucceeded, - Message: fmt.Sprintf("%s %s applies this repository and branch, and can be asked to apply", kind, ref), - } - case verdict.Err != nil: - return keptOrUnknown(target, verdict.Message) - default: - return conditionValue{Status: metav1.ConditionFalse, Reason: verdict.Reason, Message: verdict.Message} - } -} - -// keptOrUnknown is the ApplierConfigured value a check that could not be made publishes. -func keptOrUnknown(target *configbutleraiv1alpha3.GitTarget, message string) conditionValue { - for _, cond := range target.Status.Conditions { - if cond.Type == GitTargetConditionApplierConfigured { - return conditionValue{Status: cond.Status, Reason: cond.Reason, Message: cond.Message} - } - } - return conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonProgressing, Message: message} -} - -// applierReadiness contributes ApplierConfigured to the trio. An applier that applies another -// repository, that the operator may not write, or (an Argo CD Application) that has automated sync -// on, needs a human; one that is missing may be on its way (Flux and Argo CD objects are often -// created after the GitTarget), and an unknown verdict is still settling. -func applierReadiness(rd *readiness, value conditionValue, present bool) { - if !present { - return - } - rd.stalledIf(value.Status == metav1.ConditionFalse && value.Reason != applier.ReasonApplierNotFound, - value.Reason, value.Message) - rd.progressingIf(value.Status == metav1.ConditionFalse, metav1.ConditionFalse, value.Reason, value.Message) - rd.progressingIf(value.Status == metav1.ConditionUnknown, metav1.ConditionUnknown, value.Reason, value.Message) -} diff --git a/internal/controller/gittarget_applier_schema_test.go b/internal/controller/gittarget_applier_schema_test.go deleted file mode 100644 index 995bfbd53..000000000 --- a/internal/controller/gittarget_applier_schema_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "time" - - meta "github.com/fluxcd/pkg/apis/meta" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" -) - -// spec.applier names exactly one applier, of one kind, and quietPeriod (which times a CatchUp) -// belongs to a Kustomization alone: Argo CD is never asked to catch up. The API server enforces -// both, so a knob that would do nothing is never accepted. -var _ = Describe("GitTarget applier schema", func() { - kustomization := &meta.NamespacedObjectReference{Namespace: "flux-system", Name: "apps"} - application := &meta.NamespacedObjectReference{Namespace: "argocd", Name: "apps"} - quiet := &metav1.Duration{Duration: 5 * time.Second} - - create := func(name string, applier *configbutleraiv1alpha3.GitTargetApplier) error { - ctx := context.Background() - target := &configbutleraiv1alpha3.GitTarget{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, - Spec: configbutleraiv1alpha3.GitTargetSpec{ - GitProviderRef: meta.LocalObjectReference{Name: "prov"}, - Branch: "main", - Path: "apps/" + name, - Applier: applier, - }, - } - err := k8sClient.Create(ctx, target) - if err == nil { - DeferCleanup(func() { _ = k8sClient.Delete(ctx, target) }) - } - return err - } - - It("accepts a Kustomization, with or without a quiet period, and an Application without one", func() { - Expect(create("applier-kustomization", &configbutleraiv1alpha3.GitTargetApplier{ - KustomizationRef: kustomization, QuietPeriod: quiet, - })).To(Succeed()) - Expect(create("applier-kustomization-default", &configbutleraiv1alpha3.GitTargetApplier{ - KustomizationRef: kustomization, - })).To(Succeed()) - Expect(create("applier-application", &configbutleraiv1alpha3.GitTargetApplier{ - ApplicationRef: application, - })).To(Succeed()) - }) - - It("rejects both kinds at once, and neither", func() { - Expect(create("applier-both", &configbutleraiv1alpha3.GitTargetApplier{ - KustomizationRef: kustomization, ApplicationRef: application, - })).To(MatchError(ContainSubstring("exactly one of kustomizationRef and applicationRef"))) - Expect(create("applier-neither", &configbutleraiv1alpha3.GitTargetApplier{})). - To(MatchError(ContainSubstring("exactly one of kustomizationRef and applicationRef"))) - }) - - It("rejects a quiet period on an Application", func() { - Expect(create("applier-application-quiet", &configbutleraiv1alpha3.GitTargetApplier{ - ApplicationRef: application, QuietPeriod: quiet, - })).To(MatchError(ContainSubstring("quietPeriod applies only to a kustomizationRef"))) - }) -}) diff --git a/internal/controller/gittarget_applier_test.go b/internal/controller/gittarget_applier_test.go deleted file mode 100644 index ef6dca79f..000000000 --- a/internal/controller/gittarget_applier_test.go +++ /dev/null @@ -1,373 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "errors" - "testing" - "time" - - meta "github.com/fluxcd/pkg/apis/meta" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - authorizationv1 "k8s.io/api/authorization/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" - dynamicfake "k8s.io/client-go/dynamic/fake" - - configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/applier" - "github.com/ConfigButler/gitops-reverser/internal/authz" - "github.com/ConfigButler/gitops-reverser/internal/types" -) - -const applierTestRepo = "https://github.com/acme/platform.git" - -// applierFluxObjects is a Kustomization flux-system/checkout applying a GitRepository on branch. -func applierFluxObjects(branch string) []runtime.Object { - ks := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "kustomize.toolkit.fluxcd.io/v1", - "kind": "Kustomization", - "metadata": map[string]any{"namespace": "flux-system", "name": "checkout"}, - "spec": map[string]any{ - "path": "./apps", - "sourceRef": map[string]any{"kind": "GitRepository", "name": "platform"}, - }, - }} - repo := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "source.toolkit.fluxcd.io/v1", - "kind": "GitRepository", - "metadata": map[string]any{"namespace": "flux-system", "name": "platform"}, - "spec": map[string]any{ - "url": "https://github.com/acme/platform", - "ref": map[string]any{"branch": branch}, - }, - }} - return []runtime.Object{ks, repo} -} - -func applierFakeSource(objs ...runtime.Object) *dynamicfake.FakeDynamicClient { - return dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), - map[schema.GroupVersionResource]string{ - {Group: "kustomize.toolkit.fluxcd.io", Version: "v1", Resource: "kustomizations"}: "KustomizationList", - {Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "gitrepositories"}: "GitRepositoryList", - {Group: "argoproj.io", Version: "v1alpha1", Resource: "applications"}: "ApplicationList", - }, objs...) -} - -type fixedLedger map[types.ResourceReference]configbutleraiv1alpha3.GitTargetApplierStatus - -func (l fixedLedger) LastRequest( - ref types.ResourceReference, _ k8stypes.UID, -) (configbutleraiv1alpha3.GitTargetApplierStatus, bool) { - status, ok := l[ref] - return status, ok -} - -func (l fixedLedger) Forget(ref types.ResourceReference) { delete(l, ref) } - -func applierTarget(applierSet bool) *configbutleraiv1alpha3.GitTarget { - target := &configbutleraiv1alpha3.GitTarget{ - ObjectMeta: metav1.ObjectMeta{Name: "checkout", Namespace: "team-a"}, - Spec: configbutleraiv1alpha3.GitTargetSpec{Branch: "main", Path: "apps"}, - } - if applierSet { - target.Spec.Applier = &configbutleraiv1alpha3.GitTargetApplier{ - KustomizationRef: &meta.NamespacedObjectReference{Namespace: "flux-system", Name: "checkout"}, - } - } - return target -} - -// applierReconciler checks appliers against a fake source cluster whose access review answers allowed. -func applierReconciler(source dynamic.Interface, sourceErr error, allowed bool) *GitTargetReconciler { - return &GitTargetReconciler{ - ApplierClients: func(context.Context, string) (dynamic.Interface, error) { return source, sourceErr }, - newApplier: func(kind string, client dynamic.Interface) applier.Client { - canI := func(context.Context, authorizationv1.ResourceAttributes) (bool, error) { return allowed, nil } - if kind == configbutleraiv1alpha3.ApplierKindApplication { - argo := applier.NewArgoCD(client) - argo.CanI = canI - return argo - } - flux := applier.NewFlux(client) - flux.CanI = canI - return flux - }, - } -} - -func applierProvider() *configbutleraiv1alpha3.GitProvider { - return &configbutleraiv1alpha3.GitProvider{Spec: configbutleraiv1alpha3.GitProviderSpec{URL: applierTestRepo}} -} - -func TestObserveApplier_NoApplierRemovesTheCondition(t *testing.T) { - target := applierTarget(false) - target.Status.Conditions = []metav1.Condition{{Type: GitTargetConditionApplierConfigured, Status: "True"}} - target.Status.Applier = &configbutleraiv1alpha3.GitTargetApplierStatus{RequestedCommit: "old"} - st := beginStatus(nil, nil, target) - - _, present := applierReconciler(nil, nil, true).observeApplier(context.Background(), st, target, applierProvider()) - - assert.False(t, present) - assert.Nil(t, findCondition(target.Status.Conditions, GitTargetConditionApplierConfigured)) - assert.Nil(t, target.Status.Applier) -} - -func TestObserveApplier_Verdicts(t *testing.T) { - cases := []struct { - name string - objects []runtime.Object - allowed bool - wantStatus metav1.ConditionStatus - wantReason string - }{ - {name: "a Kustomization applying this repository and branch", objects: applierFluxObjects("main"), - allowed: true, wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded}, - {name: "a Kustomization applying another branch", objects: applierFluxObjects("release"), - allowed: true, wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonSourceMismatch}, - {name: "no such Kustomization", allowed: true, - wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonApplierNotFound}, - {name: "an identity that may not patch it", objects: applierFluxObjects("main"), - wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonForbidden}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - target := applierTarget(true) - st := beginStatus(nil, nil, target) - - value, present := applierReconciler(applierFakeSource(tc.objects...), nil, tc.allowed). - observeApplier(context.Background(), st, target, applierProvider()) - - require.True(t, present) - assert.Equal(t, tc.wantStatus, value.Status, value.Message) - assert.Equal(t, tc.wantReason, value.Reason) - published := findCondition(target.Status.Conditions, GitTargetConditionApplierConfigured) - require.NotNil(t, published) - assert.Equal(t, tc.wantReason, published.Reason) - }) - } -} - -// applierArgoApplication is an Application argocd/checkout syncing branch of the test repository, -// with a syncPolicy when one is given. -func applierArgoApplication(branch string, syncPolicy map[string]any) runtime.Object { - app := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "argoproj.io/v1alpha1", - "kind": "Application", - "metadata": map[string]any{"namespace": "argocd", "name": "checkout"}, - "spec": map[string]any{ - "source": map[string]any{ - "repoURL": "https://github.com/acme/platform", "targetRevision": branch, "path": "apps", - }, - }, - }} - if syncPolicy != nil { - _ = unstructured.SetNestedMap(app.Object, syncPolicy, "spec", "syncPolicy") - } - return app -} - -// TestObserveApplier_ArgoCDVerdicts: an applicationRef is checked as an Argo CD Application, with -// automated sync on as a verdict of its own. -func TestObserveApplier_ArgoCDVerdicts(t *testing.T) { - cases := []struct { - name string - objects []runtime.Object - allowed bool - wantStatus metav1.ConditionStatus - wantReason string - wantMessage string - }{ - {name: "automated sync off", objects: []runtime.Object{applierArgoApplication("main", nil)}, allowed: true, - wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, - wantMessage: "Application argocd/checkout applies this repository and branch"}, - {name: "automated sync switched off, keeping prune", allowed: true, - objects: []runtime.Object{applierArgoApplication("main", map[string]any{ - "automated": map[string]any{"enabled": false, "prune": true}, - })}, - wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded}, - {name: "automated sync on", allowed: true, - objects: []runtime.Object{applierArgoApplication("main", map[string]any{ - "automated": map[string]any{"prune": true}, - })}, - wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonAutomatedSyncEnabled}, - {name: "another branch", objects: []runtime.Object{applierArgoApplication("release", nil)}, allowed: true, - wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonSourceMismatch}, - {name: "no such Application", allowed: true, - wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonApplierNotFound}, - {name: "an identity that may not update it", objects: []runtime.Object{applierArgoApplication("main", nil)}, - wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonForbidden}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - target := applierTarget(true) - target.Spec.Applier = &configbutleraiv1alpha3.GitTargetApplier{ - ApplicationRef: &meta.NamespacedObjectReference{Namespace: "argocd", Name: "checkout"}, - } - - value, present := applierReconciler(applierFakeSource(tc.objects...), nil, tc.allowed). - observeApplier(context.Background(), beginStatus(nil, nil, target), target, applierProvider()) - - require.True(t, present) - assert.Equal(t, tc.wantStatus, value.Status, value.Message) - assert.Equal(t, tc.wantReason, value.Reason) - assert.Contains(t, value.Message, tc.wantMessage) - }) - } -} - -// TestObserveApplier_AnUnreachableSourceKeepsThePublishedVerdict: a blip reading the source cluster -// says nothing about the applier, so it must not flip a True to Unknown and back on every hiccup. -func TestObserveApplier_AnUnreachableSourceKeepsThePublishedVerdict(t *testing.T) { - r := applierReconciler(nil, errors.New("connection refused"), true) - - target := applierTarget(true) - value, _ := r.observeApplier(context.Background(), beginStatus(nil, nil, target), target, applierProvider()) - assert.Equal(t, metav1.ConditionUnknown, value.Status, "nothing published yet") - - target = applierTarget(true) - target.Status.Conditions = []metav1.Condition{{ - Type: GitTargetConditionApplierConfigured, Status: metav1.ConditionTrue, Reason: ReasonSucceeded, - }} - value, _ = r.observeApplier(context.Background(), beginStatus(nil, nil, target), target, applierProvider()) - assert.Equal(t, metav1.ConditionTrue, value.Status, "the published verdict stands") -} - -// TestObserveApplier_AHangingSourceIsBounded is the review's finding 6: the source client has no -// request timeout, so a source API server that accepts the connection and never answers held the -// reconcile, and with it every GitTarget, for as long as it liked. The check now gives up, and a -// check that could not be made keeps the published verdict. -func TestObserveApplier_AHangingSourceIsBounded(t *testing.T) { - var sawDeadline bool - r := &GitTargetReconciler{ - ApplierClients: func(ctx context.Context, _ string) (dynamic.Interface, error) { - _, sawDeadline = ctx.Deadline() - return applierFakeSource(applierFluxObjects("main")...), nil - }, - newApplier: func(_ string, client dynamic.Interface) applier.Client { - flux := applier.NewFlux(client) - flux.CanI = func(ctx context.Context, _ authorizationv1.ResourceAttributes) (bool, error) { - deadline, ok := ctx.Deadline() - require.True(t, ok, "the access review must run under a deadline") - assert.LessOrEqual(t, time.Until(deadline), applierCheckTimeout) - return false, context.DeadlineExceeded // what a server that never answers ends in - } - return flux - }, - } - - target := applierTarget(true) - target.Status.Conditions = []metav1.Condition{{ - Type: GitTargetConditionApplierConfigured, Status: metav1.ConditionTrue, Reason: ReasonSucceeded, - }} - value, _ := r.observeApplier(context.Background(), beginStatus(nil, nil, target), target, applierProvider()) - - assert.True(t, sawDeadline, "the client lookup must run under a deadline") - assert.Equal(t, metav1.ConditionTrue, value.Status, "a timed-out check keeps the published verdict") -} - -func TestObserveApplier_PublishesTheLastRequest(t *testing.T) { - target := applierTarget(true) - at := metav1.NewTime(time.Date(2026, 10, 20, 10, 14, 3, 0, time.UTC)) - r := applierReconciler(applierFakeSource(applierFluxObjects("main")...), nil, true) - r.ApplyRequests = fixedLedger{types.NewResourceReference("checkout", "team-a"): { - RequestedCommit: "4f2c1ab9", RequestedAt: &at, Reason: configbutleraiv1alpha3.ApplyReasonForeignCommit, - }} - - r.observeApplier(context.Background(), beginStatus(nil, nil, target), target, applierProvider()) - - require.NotNil(t, target.Status.Applier) - assert.Equal(t, "4f2c1ab9", target.Status.Applier.RequestedCommit) - assert.Equal(t, configbutleraiv1alpha3.ApplyReasonForeignCommit, target.Status.Applier.Reason) -} - -func TestApplierReadiness_StallsOnlyWhatNeedsAHuman(t *testing.T) { - cases := []struct { - name string - value conditionValue - present bool - wantConverged bool - wantStalled bool - }{ - {name: "no applier", present: false, wantConverged: true}, - {name: "configured", present: true, wantConverged: true, - value: conditionValue{Status: metav1.ConditionTrue, Reason: ReasonSucceeded}}, - {name: "not created yet", present: true, - value: conditionValue{Status: metav1.ConditionFalse, Reason: applier.ReasonApplierNotFound}}, - {name: "another branch", present: true, wantStalled: true, - value: conditionValue{Status: metav1.ConditionFalse, Reason: applier.ReasonSourceMismatch}}, - {name: "forbidden", present: true, wantStalled: true, - value: conditionValue{Status: metav1.ConditionFalse, Reason: applier.ReasonForbidden}}, - {name: "not checked yet", present: true, - value: conditionValue{Status: metav1.ConditionUnknown, Reason: ReasonProgressing}}, - {name: "an Application with automated sync on", present: true, wantStalled: true, - value: conditionValue{Status: metav1.ConditionFalse, Reason: applier.ReasonAutomatedSyncEnabled}}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - rd := newGitTargetReadiness() - applierReadiness(rd, tc.value, tc.present) - assert.Equal(t, tc.wantConverged, rd.converged()) - assert.Equal(t, tc.wantStalled, !rd.converged() && !rd.converging()) - }) - } -} - -// TestObserveApplier_AnApplierInAnotherCluster: spec.applier.clusterProviderRef moves the check to -// that cluster, behind that provider's accessFrom, exactly as the dispatcher asks. -func TestObserveApplier_AnApplierInAnotherCluster(t *testing.T) { - admitting := func(name string, namespaces ...string) *configbutleraiv1alpha3.ClusterProvider { - return &configbutleraiv1alpha3.ClusterProvider{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: configbutleraiv1alpha3.ClusterProviderSpec{ - AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{Names: namespaces}, - }, - } - } - cases := []struct { - name string - hub *configbutleraiv1alpha3.ClusterProvider - wantStatus metav1.ConditionStatus - wantReason string - wantClients []string - }{ - {name: "a provider that admits the namespace is asked", hub: admitting("hub", "team-a"), - wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, wantClients: []string{"hub"}}, - {name: "a provider that does not admit it is never asked", hub: admitting("hub", "team-b"), - wantStatus: metav1.ConditionFalse, wantReason: authz.ReasonNamespaceNotAuthorized}, - {name: "a provider that does not exist is never asked", - wantStatus: metav1.ConditionFalse, wantReason: authz.ReasonClusterProviderNotFound}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - objects := []runtime.Object{admitting(configbutleraiv1alpha3.DefaultClusterProviderName, "team-a")} - if tc.hub != nil { - objects = append(objects, tc.hub) - } - target := applierTarget(true) - target.Spec.Applier.ClusterProviderRef = &meta.LocalObjectReference{Name: "hub"} - r := applierReconciler(applierFakeSource(applierFluxObjects("main")...), nil, true) - r.Client = newGitProviderTestClient(t, objects...) - var asked []string - source := r.ApplierClients - r.ApplierClients = func(ctx context.Context, cluster string) (dynamic.Interface, error) { - asked = append(asked, cluster) - return source(ctx, cluster) - } - - value, present := r.observeApplier(context.Background(), beginStatus(nil, nil, target), target, - applierProvider()) - - require.True(t, present) - assert.Equal(t, tc.wantStatus, value.Status, value.Message) - assert.Equal(t, tc.wantReason, value.Reason) - assert.Equal(t, tc.wantClients, asked) - }) - } -} diff --git a/internal/controller/gittarget_apply_trigger.go b/internal/controller/gittarget_apply_trigger.go new file mode 100644 index 000000000..01ef83629 --- /dev/null +++ b/internal/controller/gittarget_apply_trigger.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" + "github.com/ConfigButler/gitops-reverser/internal/git" +) + +// GitTargetConditionApplyTriggerReady reports whether spec.applyTriggerRef names a trigger that is +// Ready and whose applier applies this target's repository and branch. Present only while +// spec.applyTriggerRef is set. +const GitTargetConditionApplyTriggerReady = "ApplyTriggerReady" + +// ReasonApplyTriggerNotFound is ApplyTriggerReady=False for a GitTarget naming a trigger that does +// not exist in its namespace. +const ReasonApplyTriggerNotFound = "ApplyTriggerNotFound" + +// applyTriggerVerdict is the ApplyTriggerReady value, and whether waiting will not clear it. +type applyTriggerVerdict struct { + value conditionValue + stalled bool +} + +// observeApplyTrigger publishes the ApplyTriggerReady condition and returns it for the readiness +// trio. present is false when the target names no trigger, in which case the condition is removed. +// +// It reads the trigger from the cache; the trigger's own reconciler checks its applier. What is +// left to check here is what only the GitTarget knows: the repository and branch it writes. +func (r *GitTargetReconciler) observeApplyTrigger( + ctx context.Context, + st *reconcileStatus, + target *configbutleraiv1alpha3.GitTarget, + gitProvider *configbutleraiv1alpha3.GitProvider, +) (applyTriggerVerdict, bool) { + ref := target.Spec.ApplyTriggerRef + if ref == nil { + st.remove(GitTargetConditionApplyTriggerReady) + return applyTriggerVerdict{}, false + } + verdict := r.applyTriggerVerdict(ctx, target, gitProvider) + st.setValue(GitTargetConditionApplyTriggerReady, verdict.value) + return verdict, true +} + +func (r *GitTargetReconciler) applyTriggerVerdict( + ctx context.Context, + target *configbutleraiv1alpha3.GitTarget, + gitProvider *configbutleraiv1alpha3.GitProvider, +) applyTriggerVerdict { + ref := target.Spec.ApplyTriggerRef + name := fmt.Sprintf("%s %s/%s", ref.Kind, target.Namespace, ref.Name) + trigger, err := git.ResolveApplyTrigger(ctx, r.Client, target) + switch { + case err != nil: + return applyTriggerVerdict{value: keptCondition(target.Status.Conditions, GitTargetConditionApplyTriggerReady, + fmt.Sprintf("Cannot read %s: %v", name, err))} + case trigger == nil: + return applyTriggerVerdict{value: conditionValue{ + Status: metav1.ConditionFalse, + Reason: ReasonApplyTriggerNotFound, + Message: name + " not found", + }} + } + + ready := findCondition(*trigger.StatusConditions(), ConditionTypeReady) + switch { + case ready == nil || ready.ObservedGeneration != trigger.GetGeneration() || ready.Status == metav1.ConditionUnknown: + reason, message := conditionReasonMessage(ready, ReasonProgressing, "has not been checked yet") + return applyTriggerVerdict{value: conditionValue{ + Status: metav1.ConditionUnknown, Reason: reason, Message: name + ": " + message, + }} + case ready.Reason == applier.ReasonNotReferenced: + // The trigger's reconcile has not seen this reference yet: naming a trigger does not change its + // generation, so the generation check above cannot tell. + return applyTriggerVerdict{value: conditionValue{ + Status: metav1.ConditionUnknown, Reason: applier.ReasonNotReferenced, + Message: name + " has not observed that this GitTarget names it yet", + }} + case ready.Status == metav1.ConditionFalse: + return applyTriggerVerdict{ + value: conditionValue{ + Status: metav1.ConditionFalse, + Reason: ready.Reason, + Message: name + ": " + ready.Message, + }, + stalled: conditionIsTrue(*trigger.StatusConditions(), ConditionTypeStalled), + } + } + + if gitProvider != nil { + want := applier.Binding{RepositoryURL: gitProvider.Spec.URL, Branch: target.Spec.Branch} + if mismatch := applier.BindingMismatch(trigger, want); mismatch != "" { + return applyTriggerVerdict{ + value: conditionValue{ + Status: metav1.ConditionFalse, + Reason: applier.ReasonSourceMismatch, + Message: mismatch, + }, + stalled: true, + } + } + } + return applyTriggerVerdict{value: conditionValue{ + Status: metav1.ConditionTrue, + Reason: ReasonSucceeded, + Message: fmt.Sprintf("%s holds %s %s, which applies this repository and branch", + name, trigger.ApplierKind(), trigger.Applier()), + }} +} + +// applyTriggerReadiness contributes ApplyTriggerReady to the trio. A trigger that applies another +// repository, or one stalled itself, needs a human; one that is missing may be on its way, and one +// that is not Ready yet is still settling. +func applyTriggerReadiness(rd *readiness, verdict applyTriggerVerdict, present bool) { + if !present { + return + } + value := verdict.value + rd.stalledIf(value.Status == metav1.ConditionFalse && verdict.stalled, value.Reason, value.Message) + rd.progressingIf(value.Status == metav1.ConditionFalse, metav1.ConditionFalse, value.Reason, value.Message) + rd.progressingIf(value.Status == metav1.ConditionUnknown, metav1.ConditionUnknown, value.Reason, value.Message) +} diff --git a/internal/controller/gittarget_apply_trigger_test.go b/internal/controller/gittarget_apply_trigger_test.go new file mode 100644 index 000000000..73652c8c5 --- /dev/null +++ b/internal/controller/gittarget_apply_trigger_test.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "errors" + "testing" + + meta "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/applier" +) + +// triggerTarget is the GitTarget team-a/checkout on branch main, naming the FluxTrigger checkout +// when named is set. +func triggerTarget(named bool) *configbutleraiv1alpha3.GitTarget { + target := &configbutleraiv1alpha3.GitTarget{ + ObjectMeta: metav1.ObjectMeta{Name: "checkout", Namespace: "team-a"}, + Spec: configbutleraiv1alpha3.GitTargetSpec{ + GitProviderRef: meta.LocalObjectReference{Name: "platform"}, Branch: "main", Path: "apps", + }, + } + if named { + target.Spec.ApplyTriggerRef = &configbutleraiv1alpha3.ApplyTriggerReference{ + Kind: configbutleraiv1alpha3.ApplyTriggerKindFlux, Name: "checkout", + } + } + return target +} + +func triggerProvider() *configbutleraiv1alpha3.GitProvider { + return &configbutleraiv1alpha3.GitProvider{Spec: configbutleraiv1alpha3.GitProviderSpec{URL: applierTestRepo}} +} + +// readyTrigger is the FluxTrigger checkout, as its reconciler published it: Ready with status, +// Stalled when stalled, and applying branch of the test repository. +func readyTrigger(status metav1.ConditionStatus, reason string, stalled bool, branch string) ctrlclient.Object { + trigger := testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName) + stalledStatus := metav1.ConditionFalse + if stalled { + stalledStatus = metav1.ConditionTrue + } + trigger.Status.Conditions = []metav1.Condition{ + {Type: ConditionTypeReady, Status: status, Reason: reason, Message: "as published", ObservedGeneration: 1, + LastTransitionTime: metav1.Now()}, + {Type: ConditionTypeStalled, Status: stalledStatus, Reason: reason, ObservedGeneration: 1, + LastTransitionTime: metav1.Now()}, + } + trigger.Status.Source = &configbutleraiv1alpha3.FluxTriggerSource{ + URL: "https://github.com/acme/platform", + Branch: branch, + } + return trigger +} + +func TestObserveApplyTrigger_NoReferenceRemovesTheCondition(t *testing.T) { + target := triggerTarget(false) + target.Status.Conditions = []metav1.Condition{{Type: GitTargetConditionApplyTriggerReady, Status: "True"}} + st := beginStatus(nil, nil, target) + + r := &GitTargetReconciler{Client: triggerTestClient(t)} + _, present := r.observeApplyTrigger(context.Background(), st, target, triggerProvider()) + + assert.False(t, present) + assert.Nil(t, findCondition(target.Status.Conditions, GitTargetConditionApplyTriggerReady)) +} + +func TestObserveApplyTrigger_Verdicts(t *testing.T) { + cases := []struct { + name string + trigger ctrlclient.Object + wantStatus metav1.ConditionStatus + wantReason string + wantMessage string + wantStalled bool + wantConverged bool + }{ + {name: "a Ready trigger whose applier applies this repository and branch", + trigger: readyTrigger(metav1.ConditionTrue, ReasonSucceeded, false, "main"), + wantStatus: metav1.ConditionTrue, wantReason: ReasonSucceeded, wantConverged: true, + wantMessage: "FluxTrigger team-a/checkout holds Kustomization flux-system/checkout"}, + {name: "no such trigger, which may be on its way", + wantStatus: metav1.ConditionFalse, wantReason: ReasonApplyTriggerNotFound, + wantMessage: "FluxTrigger team-a/checkout not found"}, + {name: "a trigger whose applier applies another branch", + trigger: readyTrigger(metav1.ConditionTrue, ReasonSucceeded, false, "release"), + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonSourceMismatch, wantStalled: true, + wantMessage: `applies branch "release", not main`}, + {name: "a trigger stalled itself", + trigger: readyTrigger(metav1.ConditionFalse, applier.ReasonForbidden, true, "main"), + wantStatus: metav1.ConditionFalse, wantReason: applier.ReasonForbidden, wantStalled: true, + wantMessage: "FluxTrigger team-a/checkout: as published"}, + {name: "a trigger whose operation is unresolved, which clears on its own", + trigger: readyTrigger(metav1.ConditionFalse, ReasonOperationUnresolved, false, "main"), + wantStatus: metav1.ConditionFalse, wantReason: ReasonOperationUnresolved}, + {name: "a trigger that has not observed this reference yet, which clears once it does", + trigger: readyTrigger(metav1.ConditionTrue, applier.ReasonNotReferenced, false, "main"), + wantStatus: metav1.ConditionUnknown, wantReason: applier.ReasonNotReferenced, + wantMessage: "has not observed that this GitTarget names it yet"}, + {name: "a trigger not checked yet", + trigger: testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName), + wantStatus: metav1.ConditionUnknown, wantReason: ReasonProgressing}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var objects []ctrlclient.Object + if tc.trigger != nil { + objects = append(objects, tc.trigger) + } + r := &GitTargetReconciler{Client: triggerTestClient(t, objects...)} + target := triggerTarget(true) + st := beginStatus(nil, nil, target) + + verdict, present := r.observeApplyTrigger(context.Background(), st, target, triggerProvider()) + + require.True(t, present) + assert.Equal(t, tc.wantStatus, verdict.value.Status, verdict.value.Message) + assert.Equal(t, tc.wantReason, verdict.value.Reason) + assert.Contains(t, verdict.value.Message, tc.wantMessage) + published := findCondition(target.Status.Conditions, GitTargetConditionApplyTriggerReady) + require.NotNil(t, published) + assert.Equal(t, tc.wantReason, published.Reason) + + rd := newGitTargetReadiness() + applyTriggerReadiness(rd, verdict, present) + assert.Equal(t, tc.wantConverged, rd.converged()) + assert.Equal(t, tc.wantStalled, !rd.converged() && !rd.converging()) + }) + } +} + +// TestObserveApplyTrigger_AReadyOfAnOlderGenerationIsNotReady: a Ready the trigger's reconciler +// published for an earlier spec says nothing about the one it has now. +func TestObserveApplyTrigger_AReadyOfAnOlderGenerationIsNotReady(t *testing.T) { + trigger := readyTrigger(metav1.ConditionTrue, ReasonSucceeded, false, "main") + trigger.SetGeneration(2) + r := &GitTargetReconciler{Client: triggerTestClient(t, trigger)} + target := triggerTarget(true) + + verdict, _ := r.observeApplyTrigger(context.Background(), beginStatus(nil, nil, target), target, triggerProvider()) + + assert.Equal(t, metav1.ConditionUnknown, verdict.value.Status) +} + +// TestApplyTriggerToGitTargets_FindsTheTargetsNamingIt: a trigger event re-runs the GitTargets in its +// namespace that name it by kind and name, and no other. +func TestApplyTriggerToGitTargets_FindsTheTargetsNamingIt(t *testing.T) { + argo := triggerTarget(true) + argo.Name = "argo" + argo.Spec.ApplyTriggerRef.Kind = configbutleraiv1alpha3.ApplyTriggerKindArgoCD + unnamed := triggerTarget(false) + unnamed.Name = "unnamed" + r := &GitTargetReconciler{Client: triggerTestClient(t, triggerTarget(true), argo, unnamed)} + + requests := r.applyTriggerToGitTargets(context.Background(), + testFluxTrigger(configbutleraiv1alpha3.DefaultClusterProviderName)) + + require.Len(t, requests, 1) + assert.Equal(t, "checkout", requests[0].Name) +} + +// TestApplyTriggerReadinessChanged_IgnoresTheWorkersWrites: the GitTarget re-reads its trigger when +// Ready, Stalled, the source or the spec moved, and not on every operation transition. +func TestApplyTriggerReadinessChanged_IgnoresTheWorkersWrites(t *testing.T) { + p := applyTriggerReadinessChanged() + before := readyTrigger(metav1.ConditionTrue, ReasonSucceeded, false, "main") + + operation := before.DeepCopyObject().(*configbutleraiv1alpha3.FluxTrigger) + operation.Status.Operation = &configbutleraiv1alpha3.ApplyOperation{ + Phase: configbutleraiv1alpha3.ApplyOperationRunning, + } + assert.False(t, p.Update(event.UpdateEvent{ObjectOld: before, ObjectNew: operation})) + + source := before.DeepCopyObject().(*configbutleraiv1alpha3.FluxTrigger) + source.Status.Source.Branch = "release" + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: before, ObjectNew: source})) + + notReady := readyTrigger(metav1.ConditionFalse, applier.ReasonForbidden, true, "main") + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: before, ObjectNew: notReady})) +} + +// TestObserveApplyTrigger_AFailedReadKeepsThePublishedVerdict: a blip reading the trigger says +// nothing about it. +func TestObserveApplyTrigger_AFailedReadKeepsThePublishedVerdict(t *testing.T) { + failing := interceptor.NewClient(triggerTestClient(t).(ctrlclient.WithWatch), interceptor.Funcs{ + Get: func(context.Context, ctrlclient.WithWatch, ctrlclient.ObjectKey, ctrlclient.Object, + ...ctrlclient.GetOption) error { + return errors.New("the API server is away") + }, + }) + r := &GitTargetReconciler{Client: failing} + + target := triggerTarget(true) + verdict, _ := r.observeApplyTrigger(context.Background(), beginStatus(nil, nil, target), target, triggerProvider()) + assert.Equal(t, metav1.ConditionUnknown, verdict.value.Status, "nothing published yet") + + target = triggerTarget(true) + target.Status.Conditions = []metav1.Condition{{ + Type: GitTargetConditionApplyTriggerReady, Status: metav1.ConditionTrue, Reason: ReasonSucceeded, + }} + verdict, _ = r.observeApplyTrigger(context.Background(), beginStatus(nil, nil, target), target, triggerProvider()) + assert.Equal(t, metav1.ConditionTrue, verdict.value.Status, "the published verdict stands") +} diff --git a/internal/controller/gittarget_controller.go b/internal/controller/gittarget_controller.go index 8af66998b..9bdb80600 100644 --- a/internal/controller/gittarget_controller.go +++ b/internal/controller/gittarget_controller.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "path/filepath" + "strconv" "strings" "time" @@ -18,7 +19,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -31,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" - "github.com/ConfigButler/gitops-reverser/internal/applier" "github.com/ConfigButler/gitops-reverser/internal/git" "github.com/ConfigButler/gitops-reverser/internal/reconcile" "github.com/ConfigButler/gitops-reverser/internal/telemetry" @@ -153,15 +152,6 @@ type GitTargetReconciler struct { // clock is the source of "now" for the publication ledger, so a test can hold time still // across a reconcile that is measured in wall clock. Nil means time.Now. clock func() time.Time - - // ApplierClients resolves the dynamic client of a GitTarget's source cluster, where its - // spec.applier lives, and ApplyRequests is the ledger of requests delivered to it. Nil leaves - // ApplierConfigured Unknown and status.applier unpublished (tests, and the CLI). - ApplierClients applier.SourceClients - ApplyRequests ApplyRequestLedger - // newApplier builds the client that checks an applier of a kind; nil is applier.NewClient. A - // seam for tests, whose fake source cluster cannot answer an access review. - newApplier func(kind string, client dynamic.Interface) applier.Client } // clockNow is the reconciler's wall clock. @@ -175,11 +165,12 @@ func (r *GitTargetReconciler) clockNow() time.Time { // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=configbutler.ai,resources=gittargets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=configbutler.ai,resources=gitproviders,verbs=get;list;watch +// +kubebuilder:rbac:groups=configbutler.ai,resources=fluxtriggers;argocdtriggers,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;create;update // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch -// The Flux and Argo CD kinds a spec.applier names are NOT granted here: no GitTarget needs them -// until one names an applier, and an install that does not use Flux or Argo CD should not hold +// The Flux and Argo CD kinds a trigger holds are NOT granted here: nothing needs them until a +// trigger names an applier, and an install that does not use Flux or Argo CD should not hold // write rights on their objects. config/rbac/flux-applier-role.yaml and argocd-applier-role.yaml, // and the chart's applierAccess.flux and applierAccess.argocd, grant them. @@ -297,13 +288,13 @@ func (r *GitTargetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( st.setValue(GitTargetConditionRenderMatchesLive, observed.axes.Render) refs := r.publishReferenceReadiness(st, &target, providerNS, gitProvider, sourceProvider) - applierValue, hasApplier := r.observeApplier(ctx, st, &target, gitProvider) + triggerVerdict, hasTrigger := r.observeApplyTrigger(ctx, st, &target, gitProvider) rd := newGitTargetReadiness() convergeAsSuspended(rd, &target) gitTargetReadinessGates(rd, observed, r.parentBranchReadiness(&target, providerNS, repoIdentityOf(gitProvider)), refs.gitProvider, refs.clusterProvider, refs.sourceCluster, publicationCondition(publication)) - applierReadiness(rd, applierValue, hasApplier) + applyTriggerReadiness(rd, triggerVerdict, hasTrigger) parentRecoveryReadiness(rd, recoveringParent) st.applyReadiness(rd) @@ -1487,10 +1478,6 @@ func (r *GitTargetReconciler) cleanupDeletedGitTarget( // said, and an entry that outlives the object is a rate limit held against a name that may be // recreated tomorrow with nothing published. r.remotePublications.forget(gitDest) - // Same terms: the last apply request delivered for it. - if r.ApplyRequests != nil { - r.ApplyRequests.Forget(gitDest) - } telemetry.ForgetResourceConditions(conditionKindGitTarget, namespacedName.Namespace, namespacedName.Name) // Same terms, same reason: a join series that outlives its GitTarget keeps attributing a live // branch's push failures to an object that no longer exists. @@ -1618,6 +1605,19 @@ func (r *GitTargetReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.clusterWatchRuleToGitTarget), builder.WithPredicates(predicate.GenerationChangedPredicate{}), ). + // React to an apply trigger appearing, going, or changing its Ready or its source, so + // ApplyTriggerReady follows the trigger within one reconcile. The trigger's own reconciler + // checks its applier; a status write that moves neither is filtered. + Watches( + &configbutleraiv1alpha3.FluxTrigger{}, + handler.EnqueueRequestsFromMapFunc(r.applyTriggerToGitTargets), + builder.WithPredicates(applyTriggerReadinessChanged()), + ). + Watches( + &configbutleraiv1alpha3.ArgoCDTrigger{}, + handler.EnqueueRequestsFromMapFunc(r.applyTriggerToGitTargets), + builder.WithPredicates(applyTriggerReadinessChanged()), + ). Named("gittarget") // React to a change of the branch worker's publication report (an outage starting, intake @@ -1707,9 +1707,8 @@ func (r *GitTargetReconciler) gitProviderToGitTargets( return requests } -// clusterProviderToGitTargets maps a ClusterProvider event to every GitTarget that references it, as -// its source or as its applier's cluster, -// across ALL namespaces (the provider is cluster-scoped). It re-enqueues dependents when the +// clusterProviderToGitTargets maps a ClusterProvider event to every GitTarget that references it as +// its source, across ALL namespaces (the provider is cluster-scoped). It re-enqueues dependents when the // provider's Ready flips or its accessFrom policy changes, so the projected // ClusterProviderReady and the namespace-authorization refusal converge without waiting for the // periodic reconcile. @@ -1726,8 +1725,7 @@ func (r *GitTargetReconciler) clusterProviderToGitTargets( var requests []ctrlreconcile.Request for i := range targets.Items { t := &targets.Items[i] - usesApplier := t.Spec.Applier != nil && t.ApplierCluster() == obj.GetName() - if t.SourceCluster() != obj.GetName() && !usesApplier { + if t.SourceCluster() != obj.GetName() { continue } requests = append(requests, ctrlreconcile.Request{ @@ -1737,6 +1735,64 @@ func (r *GitTargetReconciler) clusterProviderToGitTargets( return requests } +// applyTriggerToGitTargets maps an apply trigger event to every GitTarget in its namespace that +// names it. +func (r *GitTargetReconciler) applyTriggerToGitTargets( + ctx context.Context, + obj client.Object, +) []ctrlreconcile.Request { + trigger, ok := obj.(configbutleraiv1alpha3.ApplyTrigger) + if !ok { + return nil + } + var targets configbutleraiv1alpha3.GitTargetList + if err := r.List(ctx, &targets, client.InNamespace(obj.GetNamespace())); err != nil { + logDependencyListError(ctx, err, "GitTargets", obj) + return nil + } + var requests []ctrlreconcile.Request + for i := range targets.Items { + t := &targets.Items[i] + ref := t.Spec.ApplyTriggerRef + if ref == nil || ref.Kind != trigger.TriggerKind() || ref.Name != obj.GetName() { + continue + } + requests = append(requests, ctrlreconcile.Request{ + NamespacedName: k8stypes.NamespacedName{Name: t.Name, Namespace: t.Namespace}, + }) + } + return requests +} + +// applyTriggerReadinessChanged fires on create and delete, and on an update that changes the +// trigger's generation, its Ready or Stalled condition, or its source: what ApplyTriggerReady reads. +func applyTriggerReadinessChanged() predicate.Predicate { + summary := func(obj client.Object) string { + trigger, ok := obj.(configbutleraiv1alpha3.ApplyTrigger) + if !ok { + return "" + } + var out strings.Builder + out.WriteString(strconv.FormatInt(trigger.GetGeneration(), 10)) + for _, conditionType := range []string{ConditionTypeReady, ConditionTypeStalled} { + if c := findCondition(*trigger.StatusConditions(), conditionType); c != nil { + fmt.Fprintf(&out, "|%s/%s/%s/%d", c.Status, c.Reason, c.Message, c.ObservedGeneration) + } + } + url, branch, known := trigger.SourceBinding() + fmt.Fprintf(&out, "|%s|%s|%t", url, branch, known) + return out.String() + } + return predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return true }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + return e.ObjectOld == nil || e.ObjectNew == nil || summary(e.ObjectOld) != summary(e.ObjectNew) + }, + } +} + // namespaceToGitTargets maps a Namespace label change to every GitTarget in that namespace, so the // reconcile-time ClusterProvider authorization (which may match the namespace's labels via a // selector) re-runs and grants/revokes the target promptly. diff --git a/internal/controller/gittarget_parent_branch_test.go b/internal/controller/gittarget_parent_branch_test.go index 96dea49d2..4e60444d2 100644 --- a/internal/controller/gittarget_parent_branch_test.go +++ b/internal/controller/gittarget_parent_branch_test.go @@ -200,7 +200,7 @@ func TestParentBranchReadiness_AnOmittedParentCanStall(t *testing.T) { } // TestParentRecovering_ReadsTheWorkerOnly: the reconciler reports recovery from the branch worker's -// memory and nothing else. No worker manager, no worker, or a worker that owes nothing is not +// memory and nothing else. No worker manager, no worker, or a worker with no pending writes is not // recovering. func TestParentRecovering_ReadsTheWorkerOnly(t *testing.T) { target := parentBranchTarget("apps", "edits", "apps", "release", time.Now()) @@ -212,7 +212,7 @@ func TestParentRecovering_ReadsTheWorkerOnly(t *testing.T) { repo := git.RepoIdentity{ProviderUID: "uid-1", URL: "https://example.invalid/repo.git"} _, err := r.ensureEventStream(context.Background(), target, "shop", repo, logr.Discard()) require.NoError(t, err) - assert.False(t, r.parentRecovering(target, "shop"), "a worker that owes nothing") + assert.False(t, r.parentRecovering(target, "shop"), "a worker with no pending writes") } func TestParentRecoveryReadiness_IsProgressNotAStall(t *testing.T) { diff --git a/internal/controller/gittarget_parent_recovery.go b/internal/controller/gittarget_parent_recovery.go index dddc1b4f6..7931ae7ca 100644 --- a/internal/controller/gittarget_parent_recovery.go +++ b/internal/controller/gittarget_parent_recovery.go @@ -14,7 +14,7 @@ import ( const GitTargetReasonRecoveringParentBranch = "RecoveringParentBranch" // parentRecovering reports whether the branch worker has found the parent branch again and still -// owes the writes it kept while the parent was missing. It reads memory only, and costs no +// has to publish the writes it kept while the parent was missing. It reads memory only, and costs no // connection. func (r *GitTargetReconciler) parentRecovering( target *configbutleraiv1alpha3.GitTarget, diff --git a/internal/controller/gittarget_worker_repoint_test.go b/internal/controller/gittarget_worker_repoint_test.go index a0144f6db..e2a52cc94 100644 --- a/internal/controller/gittarget_worker_repoint_test.go +++ b/internal/controller/gittarget_worker_repoint_test.go @@ -212,6 +212,6 @@ func TestEnsureEventStream_ARecoveryThatCouldNotBeDeliveredIsRetried(t *testing. require.NoError(t, err) assert.Equal(t, []string{"shop/apps"}, manager.ForcedRecheckTargetsForTest(), - "the replay the replacement owes is forced by a later pass, not lost with the first") + "the replay the replacement still needs is forced by a later pass, not lost with the first") assert.False(t, workers.ReplacementPending(branch), "and it is not asked for again") } diff --git a/internal/controller/removed_allow_any_source_namespace_test.go b/internal/controller/removed_allow_any_source_namespace_test.go new file mode 100644 index 000000000..a46c9cd73 --- /dev/null +++ b/internal/controller/removed_allow_any_source_namespace_test.go @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + configbutleraiv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// removedFieldMessage is what the spec-level CEL rule says of allowAnySourceNamespace: true. +const removedFieldMessage = "allowAnySourceNamespace was removed; set allowCrossNamespace.sources" + +// TestRemovedAllowAnySourceNamespace_TrueIsRefusedOnEverySpecChange pins how the removal of +// ClusterProvider.spec.allowAnySourceNamespace stays loud, against the generated CRD on a real API +// server. true is refused on create and on any spec change, also for a provider that stored true +// before the rule existed: the rule sits on spec, so CRD validation ratcheting does not wave the +// stored value through once anything in spec moves. false, which the old default stored on every +// provider, stays accepted, and so does removing the field. +func TestRemovedAllowAnySourceNamespace_TrueIsRefusedOnEverySpecChange(t *testing.T) { + env := &envtest.Environment{ + CRDDirectoryPaths: []string{"../../config/crd/bases"}, + ErrorIfCRDPathMissing: true, + } + cfg, err := env.Start() + if err != nil { + t.Fatalf("start envtest control plane: %v", err) + } + t.Cleanup(func() { _ = env.Stop() }) + sch := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{ + scheme.AddToScheme, configbutleraiv1alpha3.AddToScheme, apiextv1.AddToScheme, + } { + if err := add(sch); err != nil { + t.Fatal(err) + } + } + c, err := client.New(cfg, client.Options{Scheme: sch}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + provider := func(name string, removed *bool) *configbutleraiv1alpha3.ClusterProvider { + return &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{ + AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{Names: []string{"team-a"}}, + RemovedAllowAnySourceNamespace: removed, + }, + } + } + refused := func(what string, err error) { + t.Helper() + if err == nil || !strings.Contains(err.Error(), removedFieldMessage) { + t.Fatalf("%s: want the removal refused by name, got %v", what, err) + } + } + accepted := func(what string, err error) { + t.Helper() + if err != nil { + t.Fatalf("%s: want it accepted, got %v", what, err) + } + } + addNamespace := func(p *configbutleraiv1alpha3.ClusterProvider) { + p.Spec.AccessFrom.Names = append(p.Spec.AccessFrom.Names, fmt.Sprintf("team-%d", len(p.Spec.AccessFrom.Names))) + } + + refused("create with true", c.Create(ctx, provider("created-true", ptr.To(true)))) + + kept := provider("kept-false", ptr.To(false)) + createEventually(ctx, t, c, kept) + addNamespace(kept) + accepted("a spec change keeping false", c.Update(ctx, kept)) + kept.Spec.RemovedAllowAnySourceNamespace = ptr.To(true) + refused("an update setting true", c.Update(ctx, kept)) + if err := c.Get(ctx, client.ObjectKeyFromObject(kept), kept); err != nil { + t.Fatal(err) + } + kept.Spec.RemovedAllowAnySourceNamespace = nil + accepted("removing the field", c.Update(ctx, kept)) + + // A provider that stored true before the rule existed, as one from the release that read it. + setRemovedFieldRule(ctx, t, c, false) + stored := provider("stored-true", ptr.To(true)) + createEventually(ctx, t, c, stored) + setRemovedFieldRule(ctx, t, c, true) + requireRemovedFieldRefused(ctx, t, c) + + if err := c.Get(ctx, client.ObjectKeyFromObject(stored), stored); err != nil { + t.Fatal(err) + } + stored.Labels = map[string]string{"edited": "metadata-only"} + accepted("a metadata-only edit of a stored true, which CEL does not see", c.Update(ctx, stored)) + addNamespace(stored) + refused("a spec change on a stored true", c.Update(ctx, stored)) + stored.Spec.RemovedAllowAnySourceNamespace = nil + accepted("the same spec change, removing the field", c.Update(ctx, stored)) +} + +// setRemovedFieldRule takes the spec-level rule refusing allowAnySourceNamespace: true out of the +// served ClusterProvider schema, or puts it back, standing in for the CRD of the release before it. +func setRemovedFieldRule(ctx context.Context, t *testing.T, c client.Client, present bool) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + var crd apiextv1.CustomResourceDefinition + if err := c.Get(ctx, client.ObjectKey{Name: "clusterproviders.configbutler.ai"}, &crd); err != nil { + t.Fatalf("get CRD: %v", err) + } + spec := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + var rules apiextv1.ValidationRules + for _, rule := range spec.XValidations { + if !strings.Contains(rule.Rule, "allowAnySourceNamespace") { + rules = append(rules, rule) + } + } + if present { + rules = append(rules, apiextv1.ValidationRule{ + Rule: "!has(self.allowAnySourceNamespace) || !self.allowAnySourceNamespace", + Message: removedFieldMessage, + }) + } + spec.XValidations = rules + crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] = spec + err := c.Update(ctx, &crd) + if err == nil { + return + } + if time.Now().After(deadline) { + t.Fatalf("update CRD schema: %v", err) + } + time.Sleep(100 * time.Millisecond) + } +} + +// requireRemovedFieldRefused blocks until the restored rule is the one being served, proven by a +// create that must fail: without it, the assertions after could pass against the schema without +// the rule. +func requireRemovedFieldRefused(ctx context.Context, t *testing.T, c client.Client) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for i := 0; time.Now().Before(deadline); i++ { + probe := &configbutleraiv1alpha3.ClusterProvider{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("removed-field-probe-%d", i)}, + Spec: configbutleraiv1alpha3.ClusterProviderSpec{RemovedAllowAnySourceNamespace: ptr.To(true)}, + } + if err := c.Create(ctx, probe); err != nil { + return + } + _ = c.Delete(ctx, probe) + time.Sleep(100 * time.Millisecond) + } + t.Fatal("the restored rule never took effect, so the assertions after would be vacuous") +} diff --git a/internal/controller/resource_condition_metrics.go b/internal/controller/resource_condition_metrics.go index 7f0918de5..2d71b54b7 100644 --- a/internal/controller/resource_condition_metrics.go +++ b/internal/controller/resource_condition_metrics.go @@ -32,6 +32,8 @@ const ( conditionKindClusterWatchRule = "ClusterWatchRule" conditionKindGitProvider = "GitProvider" conditionKindClusterProvider = "ClusterProvider" + conditionKindFluxTrigger = configbutleraiv1alpha3.ApplyTriggerKindFlux + conditionKindArgoCDTrigger = configbutleraiv1alpha3.ApplyTriggerKindArgoCD ) // conditionMetricKind names the kind an object publishes under, or "" for one that is not on the @@ -55,6 +57,10 @@ func conditionMetricKind(obj client.Object) string { return conditionKindGitProvider case *configbutleraiv1alpha3.ClusterProvider: return conditionKindClusterProvider + case *configbutleraiv1alpha3.FluxTrigger: + return conditionKindFluxTrigger + case *configbutleraiv1alpha3.ArgoCDTrigger: + return conditionKindArgoCDTrigger default: return "" } diff --git a/internal/controller/status.go b/internal/controller/status.go index 9256261e1..974cbb905 100644 --- a/internal/controller/status.go +++ b/internal/controller/status.go @@ -306,3 +306,11 @@ func (s *reconcileStatus) recordReadyTransition() { } s.recorder.Event(s.object, eventType, after.Reason, after.Message) } + +// metadataWritten carries metadata the reconcile wrote to the object mid-session (a finalizer) into +// the snapshot the status patch is computed from: the patch is then made against the version that +// write left, and carries no metadata of its own. +func (s *reconcileStatus) metadataWritten() { + s.before.SetResourceVersion(s.object.GetResourceVersion()) + s.before.SetFinalizers(s.object.GetFinalizers()) +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index cf00d134c..0732ac586 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -127,6 +127,15 @@ var _ = BeforeSuite(func() { }).SetupWithManager(mgr) Expect(err).NotTo(HaveOccurred()) + // No ApplierClients: a trigger's applier is never read here, so Configured stays Unknown + // (NotChecked), which is enough to drive the watches between a trigger and its GitTargets. + for _, kind := range []string{ + configbutleraiv1alpha3.ApplyTriggerKindFlux, configbutleraiv1alpha3.ApplyTriggerKindArgoCD, + } { + err = (&ApplyTriggerReconciler{Client: mgr.GetClient(), Kind: kind}).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + } + err = (&WatchRuleReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/internal/controller/superseded_fields_admission_test.go b/internal/controller/superseded_fields_admission_test.go index 23cedac4a..7e5a82496 100644 --- a/internal/controller/superseded_fields_admission_test.go +++ b/internal/controller/superseded_fields_admission_test.go @@ -69,8 +69,10 @@ var _ = Describe("Superseded source-scope fields", func() { provider := &configbutleraiv1alpha3.ClusterProvider{ ObjectMeta: metav1.ObjectMeta{Name: "renamed-source-scope"}, Spec: configbutleraiv1alpha3.ClusterProviderSpec{ - AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{Names: []string{"team-a"}}, - AllowAnySourceNamespace: true, + AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{Names: []string{"team-a"}}, + AllowCrossNamespace: &configbutleraiv1alpha3.ClusterProviderCrossNamespace{ + Sources: true, Appliers: true, + }, }, } Expect(k8sClient.Create(ctx, provider)).To(Succeed()) @@ -79,7 +81,8 @@ var _ = Describe("Superseded source-scope fields", func() { var storedProvider configbutleraiv1alpha3.ClusterProvider Expect(k8sClient.Get(ctx, types.NamespacedName{Name: provider.Name}, &storedProvider)).To(Succeed()) Expect(storedProvider.Spec.AccessFrom).NotTo(BeNil(), "accessFrom must round-trip, not be pruned") - Expect(storedProvider.Spec.AllowAnySourceNamespace).To(BeTrue()) + Expect(storedProvider.AllowsCrossNamespaceSources()).To(BeTrue(), "allowCrossNamespace must round-trip") + Expect(storedProvider.AllowsCrossNamespaceAppliers()).To(BeTrue()) target := &configbutleraiv1alpha3.GitTarget{ ObjectMeta: metav1.ObjectMeta{Name: "renamed-commit", Namespace: "default"}, diff --git a/internal/controller/watchrule_controller.go b/internal/controller/watchrule_controller.go index c10db196b..415c80da9 100644 --- a/internal/controller/watchrule_controller.go +++ b/internal/controller/watchrule_controller.go @@ -309,7 +309,7 @@ func (r *WatchRuleReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.gitProviderToWatchRules), builder.WithPredicates(predicate.GenerationChangedPredicate{}), ). - // React to a ClusterProvider's allowAnySourceNamespace (or accessFrom) + // React to a ClusterProvider's allowCrossNamespace.sources (or accessFrom) // changing. The GitTarget->WatchRules edge above still cannot be relied on to carry this: a // ClusterProvider change reaches the GitTarget as a status update that need not move the // GitTarget's readiness at all, and only a readiness move re-enqueues the rules. Without diff --git a/internal/controller/watchrule_source_namespace_test.go b/internal/controller/watchrule_source_namespace_test.go index 65a9dc0d6..cb11a4c61 100644 --- a/internal/controller/watchrule_source_namespace_test.go +++ b/internal/controller/watchrule_source_namespace_test.go @@ -14,6 +14,7 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -56,7 +57,7 @@ func wrsnClusterProvider(delegate bool) *configbutleraiv1alpha3.ClusterProvider AccessFrom: &configbutleraiv1alpha3.NamespaceMatcher{ Names: []string{wrsnTenantNS}, }, - AllowAnySourceNamespace: delegate, + AllowCrossNamespace: &configbutleraiv1alpha3.ClusterProviderCrossNamespace{Sources: delegate}, }, } } @@ -189,10 +190,34 @@ func TestReconcile_DeniedSourceNamespaceStartsNoWatch(t *testing.T) { cond := wrsnCondition(t, rule, ConditionTypeSourceNamespaceAuthorized) assert.Equal(t, metav1.ConditionFalse, cond.Status) assert.Equal(t, WatchRuleReasonSourceNamespaceNotAllowed, cond.Reason) - assert.Contains(t, cond.Message, "allowAnySourceNamespace", + assert.Contains(t, cond.Message, "spec.allowCrossNamespace.sources", "the message must name the fix") } +// TestReconcile_TheRemovedFieldDelegatesNothing: a ClusterProvider that still stores +// allowAnySourceNamespace: true, from the release that read it, delegates nothing; the WatchRule +// that relied on it is refused, and the message names the field that replaced it. +func TestReconcile_TheRemovedFieldDelegatesNothing(t *testing.T) { + ctx := context.Background() + objects := wrsnBaseObjects(false, wrsnSourceNS) + for _, obj := range objects { + if provider, ok := obj.(*configbutleraiv1alpha3.ClusterProvider); ok { + provider.Spec.AllowCrossNamespace = nil + provider.Spec.RemovedAllowAnySourceNamespace = ptr.To(true) + } + } + f := newWRSNFixture(t, objects) + + _, err := f.reconcile(ctx) + + require.NoError(t, err) + assert.Empty(t, f.compiledNames(), "the removed field is never read") + cond := wrsnCondition(t, f.reloadRule(ctx, t), ConditionTypeSourceNamespaceAuthorized) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, WatchRuleReasonSourceNamespaceNotAllowed, cond.Reason) + assert.Contains(t, cond.Message, "does not set spec.allowCrossNamespace.sources") +} + // TestReconcile_DeniedSourceNamespacePublishesTheFailedTrio pins the whole kstatus verdict a // refusal produces: Failed, under the one reason an operator greps for. func TestReconcile_DeniedSourceNamespacePublishesTheFailedTrio(t *testing.T) { @@ -260,7 +285,7 @@ func TestReconcile_RevokedSourceNamespaceRemovesTheCompiledRuleAndReplans(t *tes // The platform admin withdraws the delegation. var provider configbutleraiv1alpha3.ClusterProvider require.NoError(t, f.client.Get(ctx, k8stypes.NamespacedName{Name: wrsnProvider}, &provider)) - provider.Spec.AllowAnySourceNamespace = false + provider.Spec.AllowCrossNamespace = nil require.NoError(t, f.client.Update(ctx, &provider)) // Observe the world at the exact moment the data plane is replanned. @@ -398,7 +423,7 @@ func TestReconcile_WildcardWithoutDelegationIsRefused(t *testing.T) { cond := wrsnCondition(t, rule, ConditionTypeSourceNamespaceAuthorized) assert.Equal(t, metav1.ConditionFalse, cond.Status) assert.Equal(t, WatchRuleReasonSourceNamespaceNotAllowed, cond.Reason) - assert.Contains(t, cond.Message, "allowAnySourceNamespace") + assert.Contains(t, cond.Message, "spec.allowCrossNamespace.sources") assert.Equal(t, metav1.ConditionTrue, wrsnCondition(t, rule, ConditionTypeStalled).Status, "a refusal is terminal: nothing this controller does will change the verdict") } diff --git a/internal/git/ado_multiack_test.go b/internal/git/ado_multiack_test.go index 59db008dd..b2ef60f06 100644 --- a/internal/git/ado_multiack_test.go +++ b/internal/git/ado_multiack_test.go @@ -57,7 +57,7 @@ type adoSimulator struct { // ledger tallies every request the backend sees, with bytes, so an operation's cost in // round trips to the Git host can be measured rather than guessed. See - // git_request_ledger_test.go and docs/finished/push-notification-and-reconcile-trigger.md §1.6. + // git_request_ledger_test.go and docs/design/push-notification-and-reconcile-trigger.md §1.6. ledger *gitRequestLedger // around, when set, serves each request in place of the backend and decides whether and how diff --git a/internal/git/apply_attempt_test.go b/internal/git/apply_attempt_test.go index 0ab2ac244..8e07fc65f 100644 --- a/internal/git/apply_attempt_test.go +++ b/internal/git/apply_attempt_test.go @@ -3,8 +3,8 @@ package git // What happens to an apply attempt after the worker decided it, and how one worker serves the -// many GitTargets and appliers on its branch. The worker owns every obligation until an attempt -// delivers it; an attempt that does not deliver gives back only what still stands, and its retry is +// many GitTargets and appliers on its branch. The worker keeps every pending reason until an +// attempt delivers it; an attempt that does not deliver gives back only what still stands, and its retry is // decided again like a first attempt. See docs/design/apply-trigger.md. import ( @@ -57,10 +57,11 @@ func updateTarget(t *testing.T, loop *branchWorkerEventLoop, name string, mutate // reapplying makes a GitTarget consent to a Refusal request. func reapplying(g *configv1alpha3.GitTarget) { g.Spec.OnRefusal = configv1alpha3.RefusalActionReapply } -// oweAlphaRefusal owes alpha's applier a Refusal request for a standing refusal of its deployments. -func oweAlphaRefusal(loop *branchWorkerEventLoop) refusalKey { +// addAlphaUnsentRefusal decides a Refusal request for alpha's applier, for a standing refusal of its +// deployments. +func addAlphaUnsentRefusal(loop *branchWorkerEventLoop) refusalKey { key := refusalKeyFor(itypes.NewResourceReference("alpha", "team-a"), "deployments") - loop.oweReapply(key, owedRefusal{detail: "a base-owned field", uid: "uid-alpha"}, "obs-alpha") + loop.addUnsentRefusal(key, unsentRefusal{detail: "a base-owned field", uid: "uid-alpha"}, "obs-alpha") return key } @@ -69,12 +70,12 @@ func oweAlphaRefusal(loop *branchWorkerEventLoop) refusalKey { // it is then. func TestApplier_ARetryIsDecidedAgainOnAQuietBranch(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() first := requester.taken() require.Len(t, first, 1) - report(first[0], ApplyFailed) + report(first[0], ApplyRequestFailed) loop.serviceAppliers() assert.Empty(t, requester.taken(), "not before its backoff") assert.NotNil(t, loop.applierTimer, "the backoff is armed") @@ -85,11 +86,11 @@ func TestApplier_ARetryIsDecidedAgainOnAQuietBranch(t *testing.T) { assert.Empty(t, requester.taken(), "not while the branch is busy") loop.openWindow = nil - loop.w.debt.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) loop.serviceAppliers() retry := requester.taken() require.Len(t, retry, 1) - assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, retry[0].Reason, "the obligation came back") + assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, retry[0].Reason, "the pending ForeignCommit came back") assert.Equal(t, "ccc", retry[0].Commit, "at the tip as it is now") assert.Equal(t, 1, retry[0].PriorFailures, "so the requester does not report the failure twice") } @@ -100,11 +101,10 @@ func TestApplier_ARetryIsDecidedAgainOnAQuietBranch(t *testing.T) { // the requester only reports that the reason no longer holds, and the worker decides: no Refusal, // and a CatchUp only after the quiet period. func TestApplier_WithdrawnConsentNeverBecomesAnImmediateApply(t *testing.T) { - loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying, func(g *configv1alpha3.GitTarget) { - g.Spec.Applier.QuietPeriod = &metav1.Duration{Duration: time.Hour} - })) - loop.w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // owes a catch-up - oweAlphaRefusal(loop) + loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying), + fluxTrigger("alpha", quietFor(time.Hour))) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // leaves a CatchUp pending + addAlphaUnsentRefusal(loop) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) @@ -122,7 +122,7 @@ func TestApplier_WithdrawnConsentNeverBecomesAnImmediateApply(t *testing.T) { loop.serviceAppliers() assert.Empty(t, requester.taken(), "the catch-up waits out the hour-long quiet period") - assert.Empty(t, loop.reapplyOwed, "and the Refusal is gone with the consent") + assert.Empty(t, loop.unsentRefusals, "and the Refusal is gone with the consent") assert.True(t, loop.appliers[applierRef("alpha")].catchUp) } @@ -130,28 +130,28 @@ func TestApplier_WithdrawnConsentNeverBecomesAnImmediateApply(t *testing.T) { // recovered while its attempt was out came back with the failed attempt and was requested again. func TestApplier_ARecoveredRefusalIsNotRetried(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying)) - key := oweAlphaRefusal(loop) + key := addAlphaUnsentRefusal(loop) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) loop.refusalRecovered(key.target, key.collection) - report(out[0], ApplyFailed) + report(out[0], ApplyRequestFailed) loop.serviceAppliers() dueNow(loop) loop.serviceAppliers() assert.Empty(t, requester.taken(), "a recovered refusal has nothing left to ask for") - assert.Empty(t, loop.reapplyOwed) + assert.Empty(t, loop.unsentRefusals) } -// TestApplier_ARecreatedTargetInheritsNothing is the second review's third finding: an obligation +// TestApplier_ARecreatedTargetInheritsNothing is the second review's third finding: a request // decided for one incarnation of a GitTarget executed for another created under the same name. -// Requests carry the uid, and the worker drops what was owed to an incarnation that is gone. +// Requests carry the uid, and the worker drops what was pending for an incarnation that is gone. func TestApplier_ARecreatedTargetInheritsNothing(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true, reapplying)) loop.openWindow = &openWindow{} - oweAlphaRefusal(loop) + addAlphaUnsentRefusal(loop) old := &configv1alpha3.GitTarget{} require.NoError( @@ -166,9 +166,9 @@ func TestApplier_ARecreatedTargetInheritsNothing(t *testing.T) { loop.openWindow = nil loop.serviceAppliers() assert.Empty(t, requester.taken(), "the refusal was the old incarnation's") - assert.Empty(t, loop.reapplyOwed) + assert.Empty(t, loop.unsentRefusals) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) @@ -181,14 +181,14 @@ func TestApplier_ARecreatedTargetInheritsNothing(t *testing.T) { // recovered. func TestApplier_EachApplierRetriesOnItsOwnClock(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true), applierTestTarget("bravo", true)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 2) loop.appliers[applierRef("bravo")].failures = 6 // a long run of failures behind it for _, req := range out { - report(req, ApplyFailed) + report(req, ApplyRequestFailed) } loop.serviceAppliers() assert.Equal(t, 1, loop.appliers[applierRef("alpha")].failures) @@ -201,21 +201,17 @@ func TestApplier_EachApplierRetriesOnItsOwnClock(t *testing.T) { assert.Equal(t, applierRef("alpha"), retry[0].Applier) } -// TestApplier_OneAttemptForEveryTargetNamingIt: GitTargets that share a Kustomization are asked for -// together, each with its own reason, and the shared applier waits for the longest of their quiet -// periods, because its apply reaches every one of their folders. +// TestApplier_OneAttemptForEveryTargetNamingIt: GitTargets that share a trigger are asked for +// together, each with its own reason, and wait for the trigger's quiet period, because its apply +// reaches every one of their folders. func TestApplier_OneAttemptForEveryTargetNamingIt(t *testing.T) { - shared := func(quiet time.Duration) func(*configv1alpha3.GitTarget) { - return func(g *configv1alpha3.GitTarget) { - g.Spec.Applier = testApplier("shared") - g.Spec.Applier.QuietPeriod = &metav1.Duration{Duration: quiet} - } - } + shared := func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = testApplier("shared") } loop, requester := applyTriggerLoop(t, - applierTestTarget("alpha", true, shared(time.Minute), reapplying), - applierTestTarget("bravo", true, shared(time.Hour)), + applierTestTarget("alpha", true, shared, reapplying), + applierTestTarget("bravo", true, shared), + fluxTrigger("shared", quietFor(time.Hour)), ) - loop.w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) loop.serviceAppliers() assert.Equal(t, time.Hour, loop.appliers[applierRef("shared")].quietPeriod()) @@ -223,7 +219,7 @@ func TestApplier_OneAttemptForEveryTargetNamingIt(t *testing.T) { loop.serviceAppliers() assert.Empty(t, requester.taken(), "past alpha's quiet period, not bravo's") - oweAlphaRefusal(loop) + addAlphaUnsentRefusal(loop) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1, "one attempt for the shared applier") @@ -239,24 +235,24 @@ func TestApplier_OneAttemptForEveryTargetNamingIt(t *testing.T) { UID: "uid-bravo", Reason: configv1alpha3.ApplyReasonCatchUp, }, - }, out[0].Targets, "a Refusal applies the tip, so it answers the catch-up owed to bravo as well") + }, out[0].Targets, "a Refusal applies the tip, so it answers bravo's pending CatchUp as well") assert.False(t, loop.appliers[applierRef("shared")].catchUp) } -// TestApplier_AnAttemptOutHoldsTheNext: one attempt per applier at a time. What is owed while it is -// out waits for its report, and goes in the attempt after. +// TestApplier_AnAttemptOutHoldsTheNext: one attempt per applier at a time. A reason that becomes pending +// while it is out waits for its report, and goes in the attempt after. func TestApplier_AnAttemptOutHoldsTheNext(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) - loop.w.debt.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) loop.serviceAppliers() assert.Empty(t, requester.taken(), "the first attempt is still out") - report(out[0], ApplyDelivered) + report(out[0], ApplySucceeded) loop.serviceAppliers() next := requester.taken() require.Len(t, next, 1) @@ -264,19 +260,19 @@ func TestApplier_AnAttemptOutHoldsTheNext(t *testing.T) { assert.Zero(t, next[0].PriorFailures) } -// TestApplier_ATargetThatNoLongerNamesAnApplierIsOwedNothing: a report for an applier nobody names +// TestApplier_ATargetThatNoLongerNamesAnApplierHasNothingPending: a report for an applier nobody names // any more is dropped with it. -func TestApplier_ATargetThatNoLongerNamesAnApplierIsOwedNothing(t *testing.T) { +func TestApplier_ATargetThatNoLongerNamesAnApplierHasNothingPending(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) - updateTarget(t, loop, "alpha", func(g *configv1alpha3.GitTarget) { g.Spec.Applier = nil }) - loop.w.debt.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) + updateTarget(t, loop, "alpha", func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = nil }) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) loop.serviceAppliers() - report(out[0], ApplyFailed) + report(out[0], ApplyRequestFailed) loop.serviceAppliers() assert.Empty(t, loop.appliers) @@ -303,7 +299,7 @@ func TestSiblings_ABusyWorkerOnTheSameBranchHoldsTheOther(t *testing.T) { other.publishWriteBusy(true) sibling.publishWriteBusy(true) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() assert.Empty(t, requester.taken(), "the sibling's open window is on this branch too") assert.True(t, loop.quietSince.IsZero()) @@ -318,7 +314,40 @@ func TestSiblings_ABusyWorkerOnTheSameBranchHoldsTheOther(t *testing.T) { assert.Len(t, requester.taken(), 1, "a worker on another repository never held it") siblings.remove(sibling) - assert.False(t, loop.w.siblingWriteBusy()) + writing, working := loop.w.siblingBusy() + assert.False(t, writing) + assert.False(t, working) +} + +// TestSiblings_AnItemOnASiblingHoldsTheAttemptButNotTheQuietPeriod: a sibling's queued or +// in-hand item holds the attempt back the way this worker's own queued work does. It keeps +// quietSince, because the branch is busy only once that work opens a window, and the sibling's +// loop wakes this one as it releases its last item. +func TestSiblings_AnItemOnASiblingHoldsTheAttemptButNotTheQuietPeriod(t *testing.T) { + loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) + sibling := newMetricsTestWorker() + siblings := newBranchSiblings() + for _, w := range []*BranchWorker{loop.w, sibling} { + w.repo = RepoIdentity{URL: "https://github.com/acme/platform"} + w.applyWake = make(chan struct{}, 1) + siblings.add(w) + } + + require.True(t, sibling.Enqueue(windowEvent("alpha", "3", "kubectl-edit"))) + observeForeignCommit(loop.w) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "the sibling's item is a live edit not yet decided") + assert.False(t, loop.quietSince.IsZero(), "the quiet period runs on") + + <-sibling.eventQueue // its loop takes the item, and then releases it, as run does + sibling.releaseInflightItem() + select { + case <-loop.w.applyWake: + default: + t.Fatal("a sibling releasing its last item must wake the worker it held back") + } + loop.serviceAppliers() + assert.Len(t, requester.taken(), 1) } func TestApplierRetryDelay_DoublesToACap(t *testing.T) { @@ -337,13 +366,13 @@ func TestApplierRetryDelay_DoublesToACap(t *testing.T) { // carried comes back, and the next quiet moment decides again at the new tip. func TestApplier_AStaleAttemptIsDecidedAgainWithoutBackoff(t *testing.T) { loop, requester := manualLoop(t, applierTestTarget("alpha", true)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) require.True(t, out[0].Current(), "decided at the tip on a quiet branch") - loop.w.debt.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) assert.False(t, out[0].Current(), "the branch moved on") report(out[0], ApplyStale) @@ -358,7 +387,7 @@ func TestApplier_AStaleAttemptIsDecidedAgainWithoutBackoff(t *testing.T) { // TestDecisionHolds_OnlyAtTheTipOnAQuietBranch: the check the requester makes before it patches. func TestDecisionHolds_OnlyAtTheTipOnAQuietBranch(t *testing.T) { w := newMetricsTestWorker() - w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) + w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) assert.True(t, w.decisionHolds("aaa")) assert.False(t, w.decisionHolds("older"), "another commit is not the tip") @@ -366,10 +395,53 @@ func TestDecisionHolds_OnlyAtTheTipOnAQuietBranch(t *testing.T) { assert.False(t, w.decisionHolds("aaa"), "a window open since") w.publishWriteBusy(false) - w.eventQueue <- WorkItem{} + require.True(t, w.Enqueue(windowEvent("alpha", "3", "kubectl-edit"))) assert.False(t, w.decisionHolds("aaa"), "a queued event is a live edit not yet decided") } +// TestDecisionHolds_NotWhileTheLoopHandlesAnItem is the fifth review's second finding. The check +// read the queue's length, and busyWriting, which a wake publishes only as it ends: once the loop +// had taken its last item both said quiet, for as long as that item took to commit and push. The +// item counts until its wake has published what it left behind, for this worker's attempts and for +// a sibling's. The loop runs for real here, held inside the item by the render fidelity gate. +func TestDecisionHolds_NotWhileTheLoopHandlesAnItem(t *testing.T) { + loop, _ := manualLoop(t, applierTestTarget("alpha", true)) + w := loop.w + sibling := newMetricsTestWorker() + siblings := newBranchSiblings() + for _, worker := range []*BranchWorker{w, sibling} { + worker.repo = RepoIdentity{URL: "https://github.com/acme/platform"} + worker.applyWake = make(chan struct{}, 1) + worker.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) + siblings.add(worker) + } + gate := NewRenderFidelityGate() + w.renderFidelityGate = gate + ctx, cancel := context.WithCancel(context.Background()) + w.ctx = ctx + done := make(chan struct{}) + + gate.mu.Lock() // the loop's write check waits here + go func() { + defer close(done) + loop.run() + }() + require.True(t, w.Enqueue(windowEvent("alpha", "3", "kubectl-edit"))) + require.Eventually(t, func() bool { return len(w.eventQueue) == 0 }, 5*time.Second, time.Millisecond) + assert.False(t, w.busyWriting.Load(), "the wake has published nothing yet") + assert.False(t, w.decisionHolds("aaa"), "the item the loop is handling is a live edit") + assert.False(t, sibling.decisionHolds("aaa"), "for a sibling too") + gate.mu.Unlock() + + require.Eventually(t, func() bool { return w.inflightItems.Load() == 0 }, 5*time.Second, time.Millisecond) + assert.True(t, w.busyWriting.Load(), "the window it opened is published before the item is released") + assert.False(t, sibling.decisionHolds("aaa")) + + restartAll(gate, itypes.NewResourceReference("alpha", "team-a"), fidelityScope("apps", "deployments")) + cancel() + <-done +} + // TestPushNotice_TheTipAlreadySeenDoesNotCutAWindow is the customer review's third blocker: the // late webhook for the worker's own push closed the next save's window, splitting that save across // two commits. A notice naming the tip already observed leaves the window alone; one naming diff --git a/internal/git/apply_groups.go b/internal/git/apply_groups.go new file mode 100644 index 000000000..9792fd570 --- /dev/null +++ b/internal/git/apply_groups.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "sort" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// Shared sources (docs/design/apply-trigger.md, "Shared sources and prefetch"). +// +// Flux appliers on one branch that apply one GitRepository form a group, and a group has one +// operation at a time: the artifact is shared, and an operation reads it again just before it +// resumes its Kustomization, so a sibling's fetch landing in between would make it apply another +// commit than the one it checked. The appliers of a group take turns (turnOrder). The group spans +// the workers writing the branch (apply_siblings.go): two GitProviders on one repository are two +// workers, and their triggers can apply one GitRepository, so a worker claims the group before it +// sends, and publishes the groups it has attempts out for. +// +// When every consumer of the GitRepository is held suspended by a FluxTrigger, a new artifact +// applies nothing, so a push somebody else made is fetched as soon as it is observed (prefetch) +// rather than when the branch is next quiet: the artifact is then ready when the apply is due. It +// is asked only while the branch is busy, because on a quiet branch the attempt sent at once fetches +// for itself, and never while an operation of the group is in flight. + +// sourceGroup is the group a trigger's applier belongs to: its cluster and GitRepository, or "" for +// an applier that shares nothing this way. +func sourceGroup(trigger v1alpha3.ApplyTrigger) string { + source, ok := trigger.SharedSource() + if !ok { + return "" + } + return trigger.ClusterProvider() + "/" + source.Namespace + "/" + source.Name +} + +// group is the applier's group, as listed with any GitTarget naming it. +func (s *applierState) group() string { + for _, target := range s.targets { + return target.group + } + return "" +} + +// attemptOut reports whether an attempt for the applier, or for another of its group, is out. +func (l *branchWorkerEventLoop) attemptOut(state *applierState) bool { + return state.inflight != nil || l.groupBusy(state) +} + +// groupBusy reports whether another applier of state's group has an attempt out, on this worker or +// on a sibling. +func (l *branchWorkerEventLoop) groupBusy(state *applierState) bool { + group := state.group() + if group == "" { + return false + } + for _, other := range l.appliers { + if other != state && other.inflight != nil && other.group() == group { + return true + } + } + return l.w.groupOutOnSibling(group) +} + +// publishGroupsOut publishes the groups this worker has an attempt out for, for its siblings +// (branchSiblings.claimGroup). A group no longer out wakes them. +func (l *branchWorkerEventLoop) publishGroupsOut() { + if l.w.siblings == nil { + return + } + out := map[string]struct{}{} + for _, state := range l.appliers { + if group := state.group(); group != "" && state.inflight != nil { + out[group] = struct{}{} + } + } + l.w.siblings.publishGroupsOut(l.w, out) +} + +// turnOrder is the appliers with the one whose last turn is oldest first, those never sent an +// attempt first of all, and by name between equals. Name order alone would starve a group: a push +// somebody else made while an operation is out is pending for every applier of the group, so the first +// by name would go again after each of its own operations, and pushes arriving faster than one +// operation lasts would never let its siblings apply. +func (l *branchWorkerEventLoop) turnOrder() []ApplierRef { + refs := l.sortedAppliers() + sort.SliceStable(refs, func(i, j int) bool { + return l.appliers[refs[i]].lastTurn < l.appliers[refs[j]].lastTurn + }) + return refs +} + +// prefetchSources asks each group whose trigger may prefetch, and that has no attempt out, to fetch +// a push somebody else made. One request per group: its appliers share the GitRepository. +func (l *branchWorkerEventLoop) prefetchSources() { + w := l.w + tip, due := w.inbox.takePrefetch() + if !due || w.applyRequester == nil { + return + } + busy := map[string]bool{} + first := map[string]*applierState{} + for _, ref := range l.sortedAppliers() { + state := l.appliers[ref] + group := state.group() + if group == "" { + continue + } + busy[group] = busy[group] || state.inflight != nil + if _, seen := first[group]; !seen && state.targets[0].prefetch { + first[group] = state + } + } + groups := make([]string, 0, len(first)) + for group := range first { + groups = append(groups, group) + } + sort.Strings(groups) + for _, group := range groups { + if busy[group] || w.groupOutOnSibling(group) { + continue + } + state := first[group] + w.Log.V(1).Info("Fetching a push ahead of its apply", "branch", w.Branch, + "applier", state.ref.String(), "commit", tip) + w.applyRequester.PrefetchSource(PrefetchRequest{ + Applier: state.ref, + Commit: tip, + }) + } +} diff --git a/internal/git/apply_groups_test.go b/internal/git/apply_groups_test.go new file mode 100644 index 000000000..0e404e642 --- /dev/null +++ b/internal/git/apply_groups_test.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// Shared sources (docs/design/apply-trigger.md, "Shared sources and prefetch"): Flux appliers that apply one +// GitRepository have one operation at a time, and a push somebody else made is fetched ahead of the +// apply while the branch is busy, when every consumer of the source is held. + +import ( + "fmt" + "testing" + + "github.com/fluxcd/pkg/apis/meta" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// applying gives a trigger the source its reconciler published. +func applying(gitRepository string, prefetch bool) func(*configv1alpha3.FluxTrigger) { + return func(trigger *configv1alpha3.FluxTrigger) { + trigger.Status.Source = &configv1alpha3.FluxTriggerSource{ + GitRepository: meta.NamespacedObjectReference{Namespace: "flux-system", Name: gitRepository}, + URL: "https://example.com/repo", Branch: "main", Prefetch: prefetch, + } + } +} + +// TestGroups_OneOperationAtATimePerSource: two Kustomizations on one GitRepository are asked one +// after the other, and two on different GitRepositories at once. +func TestGroups_OneOperationAtATimePerSource(t *testing.T) { + loop, requester := manualLoop(t, + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", false)), + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", false)), + applierTestTarget("charlie", true), fluxTrigger("charlie", applying("other", false)), + ) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 2, "one per source") + assert.Equal(t, applierRef("alpha"), first[0].Applier) + assert.Equal(t, applierRef("charlie"), first[1].Applier) + + report(first[0], ApplyRequested) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "bravo waits for alpha's operation on the shared source") + + report(first[0], ApplySucceeded) + loop.serviceAppliers() + second := requester.taken() + require.Len(t, second, 1) + assert.Equal(t, applierRef("bravo"), second[0].Applier) +} + +// TestGroups_TheSourceIsSharedInTurn: a push somebody else made while an operation of the group is +// out is pending for every applier of the group, so if pushes keep coming, the applier that just had its +// turn waits for its siblings instead of going again. +func TestGroups_TheSourceIsSharedInTurn(t *testing.T) { + loop, requester := manualLoop(t, + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", false)), + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", false)), + ) + observeForeignCommit(loop.w) + loop.serviceAppliers() + for i, want := range []string{"alpha", "bravo", "alpha", "bravo"} { + sent := requester.taken() + require.Len(t, sent, 1, "turn %d", i) + assert.Equal(t, applierRef(want), sent[0].Applier, "turn %d", i) + + loop.w.inbox.observe(RemoteObservation{Commit: fmt.Sprintf("push-%d", i), By: ObservedByFetch}) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "one operation at a time on the shared source") + report(sent[0], ApplySucceeded) + loop.serviceAppliers() + } +} + +// TestPrefetch_APushIsFetchedAheadWhileTheBranchIsBusy: a push somebody else made, observed while the +// branch is busy, has the source fetch now, once per source, so its artifact is ready when the +// branch is quiet; a quiet branch asks the applier at once, which fetches for itself. +func TestPrefetch_APushIsFetchedAheadWhileTheBranchIsBusy(t *testing.T) { + loop, requester := manualLoop(t, + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", true)), + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", true)), + applierTestTarget("held", true), fluxTrigger("held", applying("shared", false)), + ) + loop.serviceAppliers() // lists the appliers + loop.openWindow = &openWindow{} + observeForeignCommit(loop.w) + + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "nothing is applied on a busy branch") + require.Len(t, requester.prefetches, 1, "one fetch for the shared source; none for a shared one") + assert.Equal(t, applierRef("alpha"), requester.prefetches[0].Applier) + assert.Equal(t, "bbb", requester.prefetches[0].Commit) + + loop.serviceAppliers() + assert.Len(t, requester.prefetches, 1, "once per push") + + loop.openWindow = nil + loop.serviceAppliers() + assert.Len(t, requester.taken(), 2, "the apply is asked once the branch is quiet") + assert.Len(t, requester.prefetches, 1) +} + +// TestPrefetch_NotWhileAnOperationOfTheSourceIsInFlight: a resumed Kustomization applies whatever the +// artifact becomes, so no fetch is asked for while one of its group is out. +func TestPrefetch_NotWhileAnOperationOfTheSourceIsInFlight(t *testing.T) { + loop, requester := manualLoop(t, + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", true)), + ) + observeForeignCommit(loop.w) + loop.serviceAppliers() + require.Len(t, requester.taken(), 1) + + loop.openWindow = &openWindow{} + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) + loop.serviceAppliers() + assert.Empty(t, requester.prefetches) +} + +// siblingLoops is two workers on one repository and branch, under two GitProviders, whose +// FluxTriggers alpha and bravo apply the GitRepository platform, each with a ForeignCommit pending. +func siblingLoops(t *testing.T) ( + *branchWorkerEventLoop, *recordingRequester, *branchWorkerEventLoop, *recordingRequester, +) { + t.Helper() + first, firstRequests := manualLoop(t, + applierTestTarget("alpha", true), fluxTrigger("alpha", applying("platform", true))) + second, secondRequests := manualLoop(t, + applierTestTarget("bravo", true), fluxTrigger("bravo", applying("platform", true))) + siblings := newBranchSiblings() + for _, loop := range []*branchWorkerEventLoop{first, second} { + loop.w.repo = RepoIdentity{URL: "https://github.com/acme/platform"} + loop.w.applyWake = make(chan struct{}, 1) + siblings.add(loop.w) + observeForeignCommit(loop.w) + } + return first, firstRequests, second, secondRequests +} + +// TestGroups_OneOperationAtATimePerSourceAcrossWorkers: two GitProviders on one repository and +// branch are two workers, and their FluxTriggers can apply one GitRepository. The group spans them: +// one operation at a time, and the worker that held back is woken when the other's ends. +func TestGroups_OneOperationAtATimePerSourceAcrossWorkers(t *testing.T) { + first, firstRequests, second, secondRequests := siblingLoops(t) + + first.serviceAppliers() + sent := firstRequests.taken() + require.Len(t, sent, 1) + second.serviceAppliers() + assert.Empty(t, secondRequests.taken(), "the other worker's operation on the shared source is out") + + report(sent[0], ApplySucceeded) + first.serviceAppliers() + select { + case <-second.w.applyWake: + default: + t.Fatal("the group going idle must wake the worker it held back") + } + second.serviceAppliers() + next := secondRequests.taken() + require.Len(t, next, 1) + assert.Equal(t, applierRef("bravo"), next[0].Applier) + + first.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) + first.serviceAppliers() + assert.Empty(t, firstRequests.taken(), "and now the first waits for the second") +} + +// TestGroups_AClaimIsCheckedAndRecordedAtOnce: two workers that both found the group idle cannot +// both send for it. The claim that loses records nothing, and the winner releasing the group wakes +// the loser. +func TestGroups_AClaimIsCheckedAndRecordedAtOnce(t *testing.T) { + first, _, second, _ := siblingLoops(t) + const group = "default/flux-system/platform" + + require.True(t, first.w.claimGroup(group)) + assert.False(t, second.w.claimGroup(group), "the other worker holds it") + assert.True(t, first.w.claimGroup(group), "a worker's own claim is no obstacle to it") + assert.True(t, second.w.claimGroup("default/flux-system/other"), "another source is its own group") + assert.True(t, first.w.groupOutOnSibling("default/flux-system/other")) + + first.publishGroupsOut() // its attempt is no longer out + select { + case <-second.w.applyWake: + default: + t.Fatal("releasing a group must wake the sibling") + } + assert.True(t, second.w.claimGroup(group)) +} + +// TestPrefetch_NotWhileASiblingsOperationOfTheSourceIsInFlight: a resumed Kustomization applies +// whatever the artifact becomes, whichever worker resumed it. +func TestPrefetch_NotWhileASiblingsOperationOfTheSourceIsInFlight(t *testing.T) { + first, firstRequests, second, secondRequests := siblingLoops(t) + first.serviceAppliers() + require.Len(t, firstRequests.taken(), 1) + + second.openWindow = &openWindow{} + second.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) + second.serviceAppliers() + assert.Empty(t, secondRequests.prefetches) +} diff --git a/internal/git/apply_operation_test.go b/internal/git/apply_operation_test.go new file mode 100644 index 000000000..a359db2be --- /dev/null +++ b/internal/git/apply_operation_test.go @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// An attempt that asked the applier for an apply is an operation, and the worker follows it until +// it completes: Requested, Running, Settling, then Succeeded or Failed. An operation the dispatcher +// stopped expecting to finish soon is Unresolved, never "delivered anyway", because a timer that +// closes the window does not stop the apply. A late apply the window withheld leaves the cluster on +// an older value than Git, and leaves a Repair pending. See docs/design/apply-trigger.md. + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" +) + +// TestApplier_AWithheldRollbackIsRepairedWithASecondSync is the third review's second finding. An +// Argo CD sync of replicas 2 runs, the API edits replicas to 3, and the sync's write of 2 arrives +// after it: the window withholds it, so Git keeps 3, and the cluster runs 2. An Application is never +// sent a CatchUp, so nothing used to sync again. The withheld write leaves a Repair pending: a +// second sync, pinned to the tip that holds the edit, sent on the quiet branch with no quiet +// period. +func TestApplier_AWithheldRollbackIsRepairedWithASecondSync(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true, argoApplier)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + report(first[0], ApplyRequested) + + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit")), "the edit is mirrored") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", ArgoCDFieldManager)), "the rollback is not") + report(first[0], ApplySucceeded) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) // the edit's push + loop.serviceAppliers() + + repair := requester.taken() + require.Len(t, repair, 1, "the rollback is repaired") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) + assert.Equal(t, "ccc", repair[0].Commit, "pinned to the tip that holds the edit") + assert.Equal(t, []ApplyTarget{{ + Ref: repair[0].Targets[0].Ref, UID: "uid-alpha", Reason: configv1alpha3.ApplyReasonRepair, + }}, repair[0].Targets) + + report(repair[0], ApplySucceeded) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "a repair is sent once") +} + +// TestApplier_ARepairWaitsForTheBranchNotForAQuietPeriod: a Repair is decided like a Refusal. It +// waits for a quiet branch, and for the attempt still out, but not for the quiet period; and a +// failed one comes back. +func TestApplier_ARepairWaitsForTheBranchNotForAQuietPeriod(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + + loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit")) + require.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager))) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "the attempt is still out") + + report(first[0], ApplySucceeded) + loop.openWindow = &openWindow{} + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "not while the branch is busy") + + loop.openWindow = nil + loop.quietSince = time.Time{} + loop.serviceAppliers() + repair := requester.taken() + require.Len(t, repair, 1, "the moment the branch is quiet") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) + + report(repair[0], ApplyFailed) + loop.serviceAppliers() + dueNow(loop) + loop.serviceAppliers() + retry := requester.taken() + require.Len(t, retry, 1, "a repair whose apply failed is pending again") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, retry[0].Reason) +} + +// TestApplier_AnUnresolvedOperationKeepsItsWindowOpen is the third review's fourth finding: a +// dispatcher that stopped waiting reported the attempt delivered, and the window closed a grace +// later while the apply could still run. An Unresolved operation holds its window open and its +// applier's next attempt back, for as long as it takes, and is counted while it does. +func TestApplier_AnUnresolvedOperationKeepsItsWindowOpen(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, requester := manualLoop(t, applierTestTarget("alpha", true, argoApplier)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + + for _, phase := range []ApplyOutcome{ApplyRequested, ApplyRunning, ApplyUnresolved} { + report(first[0], phase) + } + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) + loop.serviceAppliers() + state := onlyApplier(t, loop) + require.NotNil(t, state.inflight) + assert.Equal(t, ApplyUnresolved, state.inflight.phase) + assert.Empty(t, requester.taken(), "a reason that becomes pending meanwhile waits for the operation") + assert.True(t, state.window.open(time.Now().Add(24*time.Hour)), "no timer closes the window") + assert.Equal(t, int64(1), loop.w.unresolvedOperations.Load()) + + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit"))) + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", ArgoCDFieldManager)), + "a late apply long after the budget is still withheld") + assert.Equal(t, int64(1), collisions(t, reader, collisionWithheld)) + + report(first[0], ApplySucceeded) + loop.serviceAppliers() + assert.Zero(t, loop.w.unresolvedOperations.Load(), "it resolved on its token") + next := requester.taken() + require.Len(t, next, 1, "the next attempt goes once it completed") + assert.Equal(t, "ccc", next[0].Commit) +} + +// TestApplier_AnApplierNoLongerNamedAbandonsItsOperation: an Unresolved operation also resolves when +// no GitTarget names its applier any more. The worker stops following it, and tells the requester. +func TestApplier_AnApplierNoLongerNamedAbandonsItsOperation(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + report(first[0], ApplyUnresolved) + loop.serviceAppliers() + require.Equal(t, int64(1), loop.w.unresolvedOperations.Load()) + + updateTarget(t, loop, "alpha", func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = nil }) + observeForeignCommit(loop.w) + loop.serviceAppliers() + + assert.Empty(t, loop.appliers) + assert.Zero(t, loop.w.unresolvedOperations.Load()) + select { + case <-first[0].Abandoned: + default: + t.Fatal("the requester must be told to stop following the operation") + } +} + +// TestApplier_AFailedOperationIsRetriedOnItsBackoff: an apply the applier ran and failed gives back +// what it carried, like an attempt that could not ask, and keeps its window for the grace, since it +// may have written part of the commit. +func TestApplier_AFailedOperationIsRetriedOnItsBackoff(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + + report(first[0], ApplyRequested) + report(first[0], ApplyFailed) + loop.serviceAppliers() + state := onlyApplier(t, loop) + assert.Equal(t, 1, state.failures) + assert.True(t, state.window.open(time.Now()), "within the grace") + assert.Empty(t, requester.taken(), "not before its backoff") + + dueNow(loop) + loop.serviceAppliers() + retry := requester.taken() + require.Len(t, retry, 1) + assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, retry[0].Reason) + assert.Equal(t, 1, retry[0].PriorFailures) +} + +// onlyApplier is the one applier the loop tracks. +func onlyApplier(t *testing.T, loop *branchWorkerEventLoop) *applierState { + t.Helper() + require.Len(t, loop.appliers, 1) + for _, state := range loop.appliers { + return state + } + return nil +} + +// recordingStatusWriter records what a worker publishes on its triggers' status, in order, and the +// operations the requester recorded. +type recordingStatusWriter struct { + mu sync.Mutex + published map[TriggerRef][]ApplyTriggerStatus + recorded map[TriggerRef][]*configv1alpha3.ApplyOperation +} + +func (r *recordingStatusWriter) RecordApplyOperation( + _ context.Context, trigger TriggerRef, operation *configv1alpha3.ApplyOperation, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.recorded == nil { + r.recorded = map[TriggerRef][]*configv1alpha3.ApplyOperation{} + } + r.recorded[trigger] = append(r.recorded[trigger], operation) + return nil +} + +func (r *recordingStatusWriter) PublishApplyTriggerStatus(trigger TriggerRef, status ApplyTriggerStatus) { + r.mu.Lock() + defer r.mu.Unlock() + if r.published == nil { + r.published = map[TriggerRef][]ApplyTriggerStatus{} + } + r.published[trigger] = append(r.published[trigger], status) +} + +// last is the last publication for trigger, and how many there were. +func (r *recordingStatusWriter) last(trigger TriggerRef) (ApplyTriggerStatus, int) { + r.mu.Lock() + defer r.mu.Unlock() + all := r.published[trigger] + if len(all) == 0 { + return ApplyTriggerStatus{}, 0 + } + return all[len(all)-1], len(all) +} + +// records are the operations recorded on trigger, in order. +func (r *recordingStatusWriter) records(trigger TriggerRef) []*configv1alpha3.ApplyOperation { + r.mu.Lock() + defer r.mu.Unlock() + return r.recorded[trigger] +} + +// TestApplier_TheTriggerStatusFollowsEveryTransition: the operation reaches the trigger's status +// first through the request's Record, which the requester writes through before it applies (so a +// restart finds every apply that may have started), and moves with every report, so `kubectl get +// fluxtriggers` shows where an apply stands without anything having to reconcile. Unresolved holds +// until the operation completes. +func TestApplier_TheTriggerStatusFollowsEveryTransition(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + writer := &recordingStatusWriter{} + loop.w.triggerStatus = writer + trigger := applierRef("alpha").Trigger + + observeForeignCommit(loop.w) + loop.serviceAppliers() + sent := requester.taken() + require.Len(t, sent, 1) + _, n := writer.last(trigger) + require.Zero(t, n, "nothing is published as the attempt is sent") + require.NotNil(t, sent[0].Record) + require.NoError(t, sent[0].Record(context.Background())) + records := writer.records(trigger) + require.Len(t, records, 1, "the request writes it through before it applies") + requested := records[0] + assert.Equal(t, configv1alpha3.ApplyOperationRequested, requested.Phase) + assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, requested.Reason) + assert.Equal(t, "bbb", requested.Commit) + assert.NotEmpty(t, requested.Token) + assert.Equal(t, requested.Token, sent[0].Token, "the request carries the token it is recorded with") + assert.False(t, requested.RequestedAt.IsZero()) + assert.Nil(t, requested.FinishedAt) + + phase := func(res ApplyResult) *configv1alpha3.ApplyOperation { + t.Helper() + res.ID = sent[0].ID + sent[0].Report(res) + loop.serviceAppliers() + status, _ := writer.last(trigger) + require.NotNil(t, status.Operation) + assert.Nil(t, status.Withdrawn) + return status.Operation + } + + assert.Equal(t, requested, phase(ApplyResult{Outcome: ApplyRequested}), "published once reported") + assert.Equal(t, configv1alpha3.ApplyOperationRunning, phase(ApplyResult{Outcome: ApplyRunning}).Phase) + assert.Equal(t, configv1alpha3.ApplyOperationUnresolved, phase(ApplyResult{Outcome: ApplyUnresolved}).Phase) + settling := phase(ApplyResult{Outcome: ApplySettling, SuspendToken: "T2"}) + assert.Equal(t, configv1alpha3.ApplyOperationUnresolved, settling.Phase, + "Unresolved holds until the operation completes") + assert.Equal(t, "T2", settling.SuspendToken) + + done := phase(ApplyResult{Outcome: ApplySucceeded, SuspendToken: "T2", AppliedRevision: "main@sha1:bbb"}) + assert.Equal(t, configv1alpha3.ApplyOperationSucceeded, done.Phase) + require.NotNil(t, done.FinishedAt) + assert.Equal(t, "main@sha1:bbb", done.AppliedRevision) + assert.Equal(t, requested.RequestedAt, done.RequestedAt, "one operation, one record") + assert.Equal(t, requested.Token, done.Token) + + _, before := writer.last(trigger) + loop.serviceAppliers() + _, after := writer.last(trigger) + assert.Equal(t, before, after, "an unchanged status is not published again") +} + +// TestApplier_AnAttemptEndedBeforeItsRecordLeavesTheTriggerAlone is the fourth review's first +// finding: the Requested operation was published as the attempt was sent, so an attempt the +// requester refused (an Argo CD Application with automated sync on: Rejected) left a Requested +// operation on the trigger that nothing would ever complete. Whatever ends an attempt before its +// Record, the trigger is told nothing about it. +func TestApplier_AnAttemptEndedBeforeItsRecordLeavesTheTriggerAlone(t *testing.T) { + for _, outcome := range []ApplyOutcome{ + ApplyRejected, ApplyDropped, ApplyStale, ApplyRequestFailed, ApplyCurrent, + } { + t.Run(string(outcome), func(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true, argoApplier)) + writer := &recordingStatusWriter{} + loop.w.triggerStatus = writer + trigger := argoApplierRef("alpha").Trigger + + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + report(first[0], ApplyRequested) + report(first[0], ApplySucceeded) + loop.serviceAppliers() + completed, published := writer.last(trigger) + require.Equal(t, configv1alpha3.ApplyOperationSucceeded, completed.Operation.Phase) + + observeForeignCommit(loop.w) + loop.serviceAppliers() + second := requester.taken() + require.Len(t, second, 1) + report(second[0], outcome) + loop.serviceAppliers() + + _, after := writer.last(trigger) + assert.Equal(t, published, after, "nothing is published for an attempt that was never recorded") + assert.Empty(t, writer.records(trigger)) + assert.Equal(t, completed.Operation, onlyApplier(t, loop).shownOperation(), + "the last operation stands") + }) + } +} + +// TestApplier_AnAttemptEndedAfterItsRecordWithdrawsIt: the requester recorded the operation, and +// then could not start the apply (the resume failed). The operation is withdrawn, so the trigger +// shows the one before it again, and a restart does not follow an apply that never started. +func TestApplier_AnAttemptEndedAfterItsRecordWithdrawsIt(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + writer := &recordingStatusWriter{} + loop.w.triggerStatus = writer + trigger := applierRef("alpha").Trigger + + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + report(first[0], ApplyRequested) + report(first[0], ApplySucceeded) + loop.serviceAppliers() + completed, _ := writer.last(trigger) + + observeForeignCommit(loop.w) + loop.serviceAppliers() + second := requester.taken() + require.Len(t, second, 1) + require.NoError(t, second[0].Record(context.Background())) + recorded := writer.records(trigger) + require.Len(t, recorded, 1) + report(second[0], ApplyRequestFailed) + loop.serviceAppliers() + + withdrawal, n := writer.last(trigger) + assert.Equal(t, recorded[0], withdrawal.Withdrawn, "the recorded operation is withdrawn") + assert.Equal(t, completed.Operation, withdrawal.Operation, "and the one before it shown again") + + loop.serviceAppliers() + _, again := writer.last(trigger) + assert.Equal(t, n, again, "withdrawn once") +} + +// TestApplier_TheTriggerStatusSaysARepairIsPending: repairPending is published from the withheld write +// until an operation that carried the repair succeeded. +func TestApplier_TheTriggerStatusSaysARepairIsPending(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + writer := &recordingStatusWriter{} + loop.w.triggerStatus = writer + trigger := applierRef("alpha").Trigger + + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + report(first[0], ApplyRequested) + loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit")) + require.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager))) + loop.serviceAppliers() + status, _ := writer.last(trigger) + assert.True(t, status.RepairPending, "the withheld write leaves a repair pending") + + report(first[0], ApplySucceeded) + loop.serviceAppliers() + repair := requester.taken() + require.Len(t, repair, 1) + status, _ = writer.last(trigger) + assert.True(t, status.RepairPending, "still pending while the repair is out") + + report(repair[0], ApplyRequested) + report(repair[0], ApplySucceeded) + loop.serviceAppliers() + status, _ = writer.last(trigger) + assert.False(t, status.RepairPending, "repaired") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, status.Operation.Reason) +} diff --git a/internal/git/apply_recovery.go b/internal/git/apply_recovery.go new file mode 100644 index 000000000..9d5e82dca --- /dev/null +++ b/internal/git/apply_recovery.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + "time" + + "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// Restart recovery (docs/design/apply-trigger.md, "Restart recovery"). The worker's attempts live +// in memory, and the trigger's status.operation is their durable record: the worker writes it +// before an apply can start (ApplyRequest.Record). So a process that starts reads it back, before +// its first live event is mirrored: +// +// - An operation that has not completed is followed to its end, by its recorded tokens, with its +// apply window open, exactly as if this process had sent it: the apply may still be running, or +// the Kustomization may have been left resumed, and the follow suspends it again. +// - repairPending is restored. +// +// The trigger's reconciler records an apply somebody else started (`flux resume`) the same way, as +// an operation with no request token. The worker follows such a record whenever it lists one, not +// only on start, so its apply window opens for an apply nobody checked. +// +// The first observation of the branch is still a baseline that leaves a CatchUp pending: a +// recovered operation decides nothing about which reasons are pending, and asks the applier for +// nothing. + +// followRecordedOperations reads the triggers' recorded status on every pass, and follows what it +// finds. Every pass, and not only while the branch has appliers: a GitTarget naming a trigger for +// the first time is seen here, and gets its first apply (syncAppliers). The listing is cached. A +// listing that fails is tried again on the next pass. +func (l *branchWorkerEventLoop) followRecordedOperations(now time.Time) { + w := l.w + if w.applyRequester == nil { + return + } + targets, err := w.appliersOnBranch(w.ctx) + if err != nil { + w.Log.Error(err, "Cannot read the apply triggers' recorded operations; retrying", "branch", w.Branch) + l.resetApplierTimer(applierRetryInterval) + return + } + l.syncAppliers(targets) + for _, ref := range l.sortedAppliers() { + l.followRecordedFor(l.appliers[ref], now) + } +} + +// followRecordedFor follows what one applier's trigger records: anything in flight, and the +// pending repair, the first time this process lists it; afterwards only an apply started elsewhere. +func (l *branchWorkerEventLoop) followRecordedFor(state *applierState, now time.Time) { + recorded, repairPending := recordedStatus(state) + inFlight := recorded != nil && !recorded.Phase.Completed() && state.inflight == nil + if !state.recovered { + state.recovered = true + state.repair = state.repair || repairPending + if inFlight { + l.followRecorded(state, recorded, now) + } + return + } + if inFlight && recorded.Token == "" && recorded.SuspendToken != state.followedSuspend { + l.followRecorded(state, recorded, now) + } +} + +// recordedStatus is the operation and repairPending recorded on an applier's trigger, as listed with +// any GitTarget naming it: they all name the same trigger. +func recordedStatus(state *applierState) (*v1alpha3.ApplyOperation, bool) { + for _, target := range state.targets { + return target.recorded, target.repairPending + } + return nil, false +} + +// followRecorded hands the requester an attempt that follows a recorded operation instead of asking +// for one, and opens its applier's apply window from now: the watches of this process see nothing +// from before it. The attempt carries no pending reason, so whatever it ends with, nothing is given +// back. +func (l *branchWorkerEventLoop) followRecorded(state *applierState, recorded *v1alpha3.ApplyOperation, now time.Time) { + w := l.w + l.nextApplyID++ + abandoned, abandon := context.WithCancel(w.ctx) + attempt := &inflightAttempt{ + id: l.nextApplyID, abandon: abandon, recovered: true, + previous: state.operation, previousID: state.operationID, + } + state.inflight, state.lastTurn = attempt, attempt.id + state.followedSuspend = recorded.SuspendToken + state.openApplyWindow(attempt.id, recorded.Commit, now) + state.operation, state.operationID = recorded.DeepCopy(), attempt.id + + targets := make([]ApplyTarget, 0, len(state.targets)) + for _, target := range state.targets { + targets = append(targets, ApplyTarget{Ref: target.ref, UID: target.uid, Reason: recorded.Reason}) + } + w.Log.Info("Following an apply recorded on the trigger, with its apply window open", + "branch", w.Branch, "applier", state.ref.String(), "phase", string(recorded.Phase), + "reason", string(recorded.Reason), "commit", recorded.Commit, "startedElsewhere", recorded.Token == "") + w.applyRequester.RequestApply(ApplyRequest{ + ID: attempt.id, + Branch: w.branchKey(), + Applier: state.ref, + Reason: recorded.Reason, + Commit: recorded.Commit, + Token: recorded.Token, + Targets: targets, + Recovered: recorded.DeepCopy(), + Report: w.reportApplyResult, + Abandoned: abandoned.Done(), + }) +} diff --git a/internal/git/apply_recovery_test.go b/internal/git/apply_recovery_test.go new file mode 100644 index 000000000..ec8cc9d48 --- /dev/null +++ b/internal/git/apply_recovery_test.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// Restart recovery (docs/design/apply-trigger.md, "Restart recovery"): the trigger's +// status.operation is the durable record of an apply in flight, written before the apply could +// start, and a worker that starts follows what it finds there, with the apply window open, before +// it mirrors a live event. + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" +) + +// recording gives a trigger a recorded status. +func recording(operation *configv1alpha3.ApplyOperation, repairPending bool) func(*configv1alpha3.FluxTrigger) { + return func(trigger *configv1alpha3.FluxTrigger) { + trigger.Status.Operation, trigger.Status.RepairPending = operation, repairPending + } +} + +// recordedOperation is a worker's Refusal operation at commit "bbb", with token T1. +func recordedOperation(phase configv1alpha3.ApplyOperationPhase) *configv1alpha3.ApplyOperation { + return &configv1alpha3.ApplyOperation{ + Phase: phase, Reason: configv1alpha3.ApplyReasonRefusal, Commit: "bbb", Token: "T1", + RequestedAt: metav1.NewTime(time.Now().Add(-time.Minute)), + } +} + +// TestRecovery_AnOperationLeftInFlightIsFollowedWithItsWindowOpen: the process stopped while an apply +// ran. The new one follows that operation by its recorded token, asks the applier for nothing, and +// holds the apply window open from the start, so the apply's late write over an edit made after the +// restart is withheld. +func TestRecovery_AnOperationLeftInFlightIsFollowedWithItsWindowOpen(t *testing.T) { + recorded := recordedOperation(configv1alpha3.ApplyOperationRunning) + loop, requester := manualLoop(t, applierTestTarget("alpha", true), + fluxTrigger("alpha", recording(recorded, false))) + writer := &recordingStatusWriter{} + loop.w.triggerStatus = writer + + loop.followRecordedOperations(time.Now()) + + followed := requester.taken() + require.Len(t, followed, 1, "the recorded operation is followed") + require.NotNil(t, followed[0].Recovered) + assert.Equal(t, "T1", followed[0].Recovered.Token) + assert.Equal(t, "T1", followed[0].Token) + assert.Equal(t, configv1alpha3.ApplyReasonRefusal, followed[0].Reason) + assert.Equal(t, "bbb", followed[0].Commit) + assert.Nil(t, followed[0].Record, "nothing is asked, so nothing is recorded first") + require.True(t, onlyApplier(t, loop).window.open(time.Now())) + + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit")), "the edit is mirrored") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "the recovered apply's late write is not") + + followed[0].Report(ApplyResult{ID: followed[0].ID, Outcome: ApplySettling, SuspendToken: "T2"}) + followed[0].Report(ApplyResult{ID: followed[0].ID, Outcome: ApplySucceeded, SuspendToken: "T2", + AppliedRevision: "main@sha1:bbb"}) + loop.openWindow = &openWindow{} // busy: the repair the withheld write left pending waits + loop.serviceAppliers() + status, _ := writer.last(applierRef("alpha").Trigger) + completed := status.Operation + require.NotNil(t, completed) + assert.Equal(t, configv1alpha3.ApplyOperationSucceeded, completed.Phase, "the record is completed") + assert.Equal(t, "T1", completed.Token, "one operation, one record") + assert.Equal(t, recorded.RequestedAt.Unix(), completed.RequestedAt.Unix()) + assert.Equal(t, "T2", completed.SuspendToken) + assert.Empty(t, requester.taken()) + + loop.openWindow, loop.quietSince = nil, time.Time{} + loop.serviceAppliers() + repair := requester.taken() + require.Len(t, repair, 1, "the withheld write leaves a repair pending, as in a process that never stopped") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) +} + +// TestRecovery_APendingRepairIsRestored: repairPending was recorded because a late apply was withheld; +// the cluster still runs the older value, so the new process sends the repair. +func TestRecovery_APendingRepairIsRestored(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true), + fluxTrigger("alpha", recording(recordedOperation(configv1alpha3.ApplyOperationSucceeded), true))) + loop.followRecordedOperations(time.Now()) + assert.True(t, onlyApplier(t, loop).repair) + loop.serviceAppliers() + repair := requester.taken() + require.Len(t, repair, 1, "the repair is sent on the quiet branch") + assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) + assert.Nil(t, repair[0].Recovered) +} + +// TestRecovery_OnlyAnOperationInFlightIsFollowed: a completed record is history, and asks for +// nothing; a worker with no trigger record says nothing. +func TestRecovery_OnlyAnOperationInFlightIsFollowed(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true), + fluxTrigger("alpha", recording(recordedOperation(configv1alpha3.ApplyOperationSucceeded), false))) + loop.followRecordedOperations(time.Now()) + assert.Empty(t, requester.taken()) + assert.Nil(t, onlyApplier(t, loop).inflight) +} + +// TestRecovery_ARecordIsReadOnceButAnApplyStartedElsewhereWhenever: a worker record is read back +// once, when this process first lists the applier, because afterwards the cache can lag the worker's +// own writes. The trigger's reconciler records an apply somebody else started (no request token) +// at any time, and the worker follows each such record once, so its window opens for it. +func TestRecovery_ARecordIsReadOnceButAnApplyStartedElsewhereWhenever(t *testing.T) { + loop, requester := manualLoop(t, applierTestTarget("alpha", true)) + loop.followRecordedOperations(time.Now()) + require.Empty(t, requester.taken()) + + setRecorded := func(operation *configv1alpha3.ApplyOperation) { + t.Helper() + trigger := &configv1alpha3.FluxTrigger{} + require.NoError( + t, + loop.w.Client.Get(t.Context(), client.ObjectKey{Namespace: "team-a", Name: "alpha"}, trigger), + ) + trigger.Status.Operation = operation + require.NoError(t, loop.w.Client.Status().Update(t.Context(), trigger)) + } + setRecorded(recordedOperation(configv1alpha3.ApplyOperationRunning)) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "a worker's own record is not read back while it runs") + + external := &configv1alpha3.ApplyOperation{ + Phase: configv1alpha3.ApplyOperationUnresolved, SuspendToken: "F1", RequestedAt: metav1.Now(), + } + setRecorded(external) + loop.serviceAppliers() + followed := requester.taken() + require.Len(t, followed, 1, "an apply started elsewhere is followed") + assert.Equal(t, "F1", followed[0].Recovered.SuspendToken) + assert.Empty(t, followed[0].Token) + + followed[0].Report(ApplyResult{ID: followed[0].ID, Outcome: ApplySucceeded, SuspendToken: "F1"}) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "once") +} diff --git a/internal/git/apply_request.go b/internal/git/apply_request.go index fce8622e5..a9cc8363b 100644 --- a/internal/git/apply_request.go +++ b/internal/git/apply_request.go @@ -3,24 +3,31 @@ package git import ( + "context" + "fmt" "slices" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/ConfigButler/gitops-reverser/api/v1alpha3" itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -// ApplierRef is one applier, as the GitTargets on a branch name it: a Flux Kustomization or an -// Argo CD Application in a cluster. Several GitTargets may name one applier, and it is asked once -// for all of them. +// ApplierRef is one applier, as the apply trigger that holds it names it: a Flux Kustomization or +// an Argo CD Application in a cluster. Several GitTargets may name one trigger, and its applier is +// asked once for all of them. type ApplierRef struct { - // Cluster is the ClusterProvider whose cluster the applier lives in (GitTarget.ApplierCluster()). + // Trigger is the apply trigger that holds the applier. + Trigger TriggerRef + // Cluster is the ClusterProvider whose cluster the applier lives in (the trigger's + // spec.clusterProviderRef). Cluster string // Kind is v1alpha3.ApplierKindKustomization or v1alpha3.ApplierKindApplication. Kind string - // Namespace and Name are the object's, the namespace defaulted to the GitTarget's own. + // Namespace and Name are the object's, the namespace defaulted to the trigger's own. Namespace string Name string } @@ -29,21 +36,51 @@ func (a ApplierRef) String() string { return a.Cluster + "/" + a.Kind + "/" + a.Namespace + "/" + a.Name } -// CatchesUp reports whether the applier is owed a CatchUp after GitOps Reverser's own pushes. An +// CatchesUp reports whether the applier is sent a CatchUp after GitOps Reverser's own pushes. An // Argo CD Application is not: with automated sync off it applies nothing on its own, so it holds no // stale state for a catch-up to correct. func (a ApplierRef) CatchesUp() bool { return a.Kind != v1alpha3.ApplierKindApplication } -// ApplierOf is the applier a GitTarget names. Call it only for a target with spec.applier. -func ApplierOf(target *v1alpha3.GitTarget) ApplierRef { - ref := target.Spec.Applier.Reference() - namespace := ref.Namespace - if namespace == "" { - namespace = target.Namespace - } +// TriggerRef is one apply trigger: its kind (v1alpha3.ApplyTriggerKind*), namespace and name. +type TriggerRef struct { + Kind string + Namespace string + Name string +} + +func (t TriggerRef) String() string { return t.Kind + "/" + t.Namespace + "/" + t.Name } + +// ApplierOf is the applier a trigger holds. +func ApplierOf(trigger v1alpha3.ApplyTrigger) ApplierRef { + applier := trigger.Applier() return ApplierRef{ - Cluster: target.ApplierCluster(), Kind: target.Spec.Applier.Kind(), Namespace: namespace, Name: ref.Name, + Trigger: TriggerRef{Kind: trigger.TriggerKind(), Namespace: trigger.GetNamespace(), Name: trigger.GetName()}, + Cluster: trigger.ClusterProvider(), Kind: trigger.ApplierKind(), + Namespace: applier.Namespace, Name: applier.Name, + } +} + +// ResolveApplyTrigger reads the apply trigger a GitTarget names, in the GitTarget's namespace. It +// returns nil, and no error, when the GitTarget names none, names a kind this release does not +// know, or names one that does not exist. +func ResolveApplyTrigger( + ctx context.Context, reader client.Reader, target *v1alpha3.GitTarget, +) (v1alpha3.ApplyTrigger, error) { + ref := target.Spec.ApplyTriggerRef + if ref == nil { + return nil, nil //nolint:nilnil // a GitTarget that names no trigger has none, and is no error + } + trigger := v1alpha3.NewApplyTrigger(ref.Kind) + if trigger == nil { + return nil, nil //nolint:nilnil // as above: a kind this release does not serve names none + } + if err := reader.Get(ctx, client.ObjectKey{Namespace: target.Namespace, Name: ref.Name}, trigger); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil //nolint:nilnil // as above: a trigger not created yet holds nothing + } + return nil, fmt.Errorf("read %s %s/%s: %w", ref.Kind, target.Namespace, ref.Name, err) } + return trigger, nil } // ApplyRequest is one attempt, decided by a branch worker on a quiet branch, to have one applier @@ -62,6 +99,9 @@ type ApplyRequest struct { // Commit is the commit to apply: the branch tip when the worker decided, which the worker has // pushed or proved by a fetch. Empty when the remote does not carry the branch. Commit string + // Token identifies the operation the attempt starts, in the applier's own status and in the + // trigger's status.operation, which Record writes with it. + Token string // Targets are the GitTargets the request is for, each the incarnation the worker decided for // and with its own reason. The requester asks nothing for a target that has changed since. Targets []ApplyTarget @@ -71,12 +111,30 @@ type ApplyRequest struct { PriorFailures int // Current reports whether the decision still holds: Commit is still the branch tip, and nothing // live is waiting to be published on this worker or a sibling. The requester asks it just before - // it patches, and asks nothing when it is false (ApplyStale): an apply of an older commit, or - // over an edit not yet pushed, is exactly what the quiet-branch rule exists to prevent. Nil is - // always current. It does not block. + // the write that starts the apply, and asks nothing when it is false (ApplyStale): an apply of an + // older commit, or over an edit not yet pushed, is exactly what the quiet-branch rule exists to + // prevent. Nil is always current. It does not block. Current func() bool - // Report receives the attempt's outcome. It is called exactly once, and does not block. + // Record writes the operation this attempt starts on its trigger's status, and returns once it + // is written: the only way the operation reaches the trigger before the requester reports it. + // The requester calls it just before the write that starts the apply, and asks nothing when it + // fails: a restart reads status.operation to find every apply that may have started, and a + // trigger that is being deleted, or that records an apply somebody else started, starts none. + // Nil records nothing. + Record func(ctx context.Context) error + // Report receives the attempt's reports, in order, and does not block. An attempt that asked the + // applier for an apply reports its operation as it moves (ApplyRequested first) until a report + // that ends it (ApplyOutcome.Ends); one that asked nothing reports once, with an outcome that + // ends it. Report func(ApplyResult) + // Recovered, when set, is an operation recorded on the trigger (by an earlier process, or by the + // trigger's reconciler for an apply somebody else started) that the attempt follows to its end + // instead of asking the applier for anything. Its token and suspend token are the ones to read. + Recovered *v1alpha3.ApplyOperation + // Abandoned is closed when the worker stops following the attempt: no GitTarget names its + // applier any more, or the worker stopped. The requester stops following the operation then, + // and reports nothing more. Nil is never closed. + Abandoned <-chan struct{} } // ApplyTarget is one GitTarget an ApplyRequest is for. @@ -86,25 +144,35 @@ type ApplyTarget struct { Reason v1alpha3.ApplyReason } -// TargetRefs lists the request's GitTargets. -func (r ApplyRequest) TargetRefs() []itypes.ResourceReference { - refs := make([]itypes.ResourceReference, 0, len(r.Targets)) - for _, target := range r.Targets { - refs = append(refs, target.Ref) - } - return refs -} - -// ApplyOutcome is how one attempt went. +// ApplyOutcome is where one attempt stands. The first four are the operation of an attempt that +// asked the applier for an apply, before it completed; they hold the attempt out and its applier's +// apply window open. Every other outcome ends the attempt. type ApplyOutcome string const ( - // ApplyDelivered asked the applier to apply. - ApplyDelivered ApplyOutcome = "Delivered" - // ApplyCurrent asked nothing: a CatchUp found the applier already at the commit. - ApplyCurrent ApplyOutcome = "Current" - // ApplyFailed could not ask; a later attempt may. + // ApplyRequested asked the applier to apply: the Kustomization was resumed or the sync + // operation set. + ApplyRequested ApplyOutcome = "Requested" + // ApplyRunning is an operation the applier is seen working on. + ApplyRunning ApplyOutcome = "Running" + // ApplySettling is an operation the applier has applied and GitOps Reverser has not yet seen + // settle: for a suspended Flux Kustomization, the suspend after the apply. + ApplySettling ApplyOutcome = "Settling" + // ApplyUnresolved is an operation that did not complete within the requester's budget. It is + // still followed, and resolves only when it completes or is abandoned: no timer stops an apply, + // so none closes its window. + ApplyUnresolved ApplyOutcome = "Unresolved" + + // ApplySucceeded is an operation that completed, and applied. + ApplySucceeded ApplyOutcome = "Succeeded" + // ApplyFailed is an operation that completed without applying the commit: the applier reported + // a failure, or the operation is gone. It may have applied part of it. ApplyFailed ApplyOutcome = "Failed" + // ApplyCurrent asked nothing: a CatchUp or ForeignCommit found the applier already at the + // commit (a suspended Kustomization that applied it), or the remote does not carry the branch. + ApplyCurrent ApplyOutcome = "Current" + // ApplyRequestFailed could not ask; a later attempt may. + ApplyRequestFailed ApplyOutcome = "RequestFailed" // ApplyRejected could not ask, and asking again will not help until the applier's // configuration changes: it applies another repository or branch, or no GitRepository. ApplyRejected ApplyOutcome = "Rejected" @@ -112,33 +180,94 @@ const ( // request carried: each changed, is no longer admitted by its ClusterProvider, or withdrew its // consent to a Refusal. The worker decides again from what the targets are now. ApplyDropped ApplyOutcome = "Dropped" - // ApplyStale asked nothing, because the decision no longer held when the patch was due: the + // ApplyStale asked nothing, because the decision no longer held when the write was due: the // branch moved on, or became busy (ApplyRequest.Current). It is no failure; the worker decides // again at the next quiet moment, at the tip as it is then. ApplyStale ApplyOutcome = "Stale" ) +// Ends reports whether the outcome ends its attempt. +func (o ApplyOutcome) Ends() bool { + switch o { + case ApplyRequested, ApplyRunning, ApplySettling, ApplyUnresolved: + return false + case ApplySucceeded, ApplyFailed, ApplyCurrent, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyStale: + } + return true +} + +// Operated reports whether the outcome completes an operation the applier ran, whatever its result. +func (o ApplyOutcome) Operated() bool { return o == ApplySucceeded || o == ApplyFailed } + // ApplyResult is the report of one attempt. type ApplyResult struct { ID uint64 Outcome ApplyOutcome - // Err is the cause of a Failed or Rejected attempt. + // SuspendToken is the requestedAt a Flux Kustomization was suspended again with after the apply + // (Settling carries it, and so do the reports after it). + SuspendToken string + // AppliedRevision is the revision the applier reports it applied (Succeeded and Failed carry + // it, when the applier reports one). + AppliedRevision string + // Message says what a Succeeded operation applied instead of the commit. + Message string + // Err is the cause of a RequestFailed, Rejected or Failed attempt. Err error } -// ApplyRequester carries out apply requests. RequestApply must not block: it is called on the -// branch worker's loop goroutine. +// ApplyTriggerStatus is what a branch worker publishes on an apply trigger: the operation it last +// started on the trigger's applier, and whether a Repair is pending. +type ApplyTriggerStatus struct { + // Operation is the operation status.operation shows, once it holds it: the writer moves an + // operation on, and writes one on a status that holds none, but never replaces another. Nil + // leaves it as it is: a worker that has started no operation since it began has nothing to say + // about the last one. + Operation *v1alpha3.ApplyOperation + // Withdrawn is an operation the worker recorded (ApplyRequest.Record) for an attempt that then + // asked for no apply. If status.operation still holds it, it is put back as it was before. + Withdrawn *v1alpha3.ApplyOperation + // RepairPending is status.repairPending. + RepairPending bool +} + +// ApplyTriggerStatusWriter writes what the branch workers publish on their triggers' status. +// PublishApplyTriggerStatus must not block: it is called on a branch worker's loop goroutine. A +// later publication for the same trigger supersedes one not yet written, except for what that one +// withdrew. +type ApplyTriggerStatusWriter interface { + PublishApplyTriggerStatus(trigger TriggerRef, status ApplyTriggerStatus) + // RecordApplyOperation writes the operation an attempt starts on the trigger's status now, and + // returns once it is written. It blocks, so it is never called on a worker's loop: the requester + // calls it (ApplyRequest.Record) just before it starts the apply. It fails when the trigger is + // gone or being deleted, or its status holds another operation that has not completed. + RecordApplyOperation(ctx context.Context, trigger TriggerRef, operation *v1alpha3.ApplyOperation) error +} + +// ApplyRequester carries out apply requests. RequestApply and PrefetchSource must not block: they +// are called on the branch worker's loop goroutine. type ApplyRequester interface { RequestApply(req ApplyRequest) + // PrefetchSource asks the source req.Applier applies to fetch req.Commit now, ahead of an apply, + // when its trigger says it may and no operation on that source is in flight. It applies nothing + // and reports nothing. + PrefetchSource(req PrefetchRequest) +} + +// PrefetchRequest asks the source an applier applies to fetch a commit ahead of its apply. +type PrefetchRequest struct { + Applier ApplierRef + Commit string } -// StrongerApplyReason is the reason a target owed two requests is owed. A CatchUp asks for the -// least: it stamps nothing when the artifact already holds the commit, where the others stamp the -// Kustomization, so it never replaces either. A ForeignCommit outranks a Refusal because it needs -// no consent. The empty reason ranks below all three. +// StrongerApplyReason is the reason a target is asked for when two reasons are pending for it. A +// CatchUp asks for the least: it asks nothing of a Kustomization that is suspended with the commit +// applied, where the others always run the operation, so it never replaces any of them. A Repair and a ForeignCommit +// outrank a Refusal because they need no consent, and a ForeignCommit outranks a Repair because the +// commit it applies is news to the applier. The empty reason ranks below all four. func StrongerApplyReason(a, b v1alpha3.ApplyReason) v1alpha3.ApplyReason { weakestFirst := []v1alpha3.ApplyReason{ - "", v1alpha3.ApplyReasonCatchUp, v1alpha3.ApplyReasonRefusal, v1alpha3.ApplyReasonForeignCommit, + "", v1alpha3.ApplyReasonCatchUp, v1alpha3.ApplyReasonRefusal, v1alpha3.ApplyReasonRepair, + v1alpha3.ApplyReasonForeignCommit, } if slices.Index(weakestFirst, b) > slices.Index(weakestFirst, a) { return b diff --git a/internal/git/apply_siblings.go b/internal/git/apply_siblings.go index 55d02cbd3..33bb8ab2b 100644 --- a/internal/git/apply_siblings.go +++ b/internal/git/apply_siblings.go @@ -64,20 +64,25 @@ func (s *branchSiblings) remove(w *BranchWorker) { s.wakeOthers(w) } -// othersBusy reports whether another worker on w's repository and branch is busy writing. -func (s *branchSiblings) othersBusy(w *BranchWorker) bool { +// othersBusy reports whether another worker on w's repository and branch is busy writing, and +// whether one has work items queued or in hand: a queued watch event is an edit that is live and +// not yet decided, and the item a loop is handling may be committing and pushing before the wake +// that publishes it ends. +func (s *branchSiblings) othersBusy(w *BranchWorker) (bool, bool) { key, ok := siblingKey(w) if !ok { - return false + return false, false } s.mu.Lock() defer s.mu.Unlock() + writing, working := false, false for sibling := range s.byKey[key] { - if sibling != w && sibling.busyWriting.Load() { - return true + if sibling != w { + writing = writing || sibling.busyWriting.Load() + working = working || sibling.inflightItems.Load() > 0 } } - return false + return writing, working } // wakeOthers wakes every other worker on w's repository and branch, so one that held back an @@ -108,7 +113,91 @@ func (w *BranchWorker) publishWriteBusy(busy bool) { } } -// siblingWriteBusy reports whether another worker on this repository and branch is busy writing. -func (w *BranchWorker) siblingWriteBusy() bool { - return w.siblings != nil && w.siblings.othersBusy(w) +// claimGroup records that w sends an attempt for an applier of group, and reports false, recording +// nothing, when another worker on its repository and branch has one out for that group. Checking +// and recording under one lock is what keeps two workers from sending for one group at once. +func (s *branchSiblings) claimGroup(w *BranchWorker, group string) bool { + key, ok := siblingKey(w) + if !ok { + return true + } + s.mu.Lock() + defer s.mu.Unlock() + if s.groupOutLocked(key, w, group) { + return false + } + if w.groupsOut == nil { + w.groupsOut = map[string]struct{}{} + } + w.groupsOut[group] = struct{}{} + return true +} + +// groupOutElsewhere reports whether another worker on w's repository and branch has an attempt out +// for an applier of group. +func (s *branchSiblings) groupOutElsewhere(w *BranchWorker, group string) bool { + key, ok := siblingKey(w) + if !ok { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return s.groupOutLocked(key, w, group) +} + +func (s *branchSiblings) groupOutLocked(key string, w *BranchWorker, group string) bool { + for sibling := range s.byKey[key] { + if _, out := sibling.groupsOut[group]; out && sibling != w { + return true + } + } + return false +} + +// publishGroupsOut replaces the groups w has attempts out for, and wakes the others when one of them +// is no longer out: they may have held an attempt back for it. +func (s *branchSiblings) publishGroupsOut(w *BranchWorker, groups map[string]struct{}) { + if _, ok := siblingKey(w); !ok { + return + } + s.mu.Lock() + released := false + for group := range w.groupsOut { + if _, out := groups[group]; !out { + released = true + } + } + w.groupsOut = groups + s.mu.Unlock() + if released { + s.wakeOthers(w) + } +} + +// claimGroup records that this worker sends an attempt for an applier of group, unless a sibling +// has one out for it (branchSiblings.claimGroup). An applier in no group claims nothing. +func (w *BranchWorker) claimGroup(group string) bool { + return group == "" || w.siblings == nil || w.siblings.claimGroup(w, group) +} + +// groupOutOnSibling reports whether a sibling has an attempt out for an applier of group. +func (w *BranchWorker) groupOutOnSibling(group string) bool { + return group != "" && w.siblings != nil && w.siblings.groupOutElsewhere(w, group) +} + +// releaseInflightItem takes one item off the in-flight count, and wakes the siblings when it was +// the last: they may have held an attempt back for it. +func (w *BranchWorker) releaseInflightItem() { + if w.inflightItems.Add(-1) == 0 && w.siblings != nil { + w.siblings.wakeOthers(w) + } +} + +// siblingBusy reports whether another worker on this repository and branch is busy writing, and +// whether one has work items queued or in hand (othersBusy). +func (w *BranchWorker) siblingBusy() (bool, bool) { + if w.siblings == nil { + return false, false + } + return w.siblings.othersBusy(w) } diff --git a/internal/git/apply_trigger.go b/internal/git/apply_trigger.go index def003d0e..233ba21b5 100644 --- a/internal/git/apply_trigger.go +++ b/internal/git/apply_trigger.go @@ -7,8 +7,11 @@ import ( "fmt" "sort" "sync" + "sync/atomic" "time" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -22,54 +25,64 @@ import ( // An apply is safe only when nothing live is missing from the commit being applied, and only the // workers on this branch can see the edits that are live and not yet pushed: their open commit // windows and pending writes. So an attempt goes out only on a QUIET branch, with neither on this -// worker or on any other worker writing the same repository and branch. Three things owe one: +// worker or on any other worker writing the same repository and branch. Four reasons ask for one: // // - ForeignCommit: an observation of the remote found the branch somewhere this worker did not // push it. Every applier on the branch is asked as soon as the branch is quiet, and an open // window is closed rather than waited out, because a user who keeps editing would otherwise // hold a Git-side change back for as long as they type. // - Refusal: a refused edit on a target with spec.onRefusal: Reapply (refusal_reapply.go). -// - CatchUp: this worker's own pushes. Flux applies on its own timers and on a restart from -// whatever artifact it holds, and an artifact behind these pushes would revert edits already in -// Git. Asked only once the branch has stayed quiet for the applier's quiet period, and never of -// an Argo CD Application: with automated sync off it applies nothing on its own, so it holds no -// stale state to correct (ApplierRef.CatchesUp). +// - CatchUp: this worker's own pushes. A suspended Kustomization applies nothing on its own, so +// this is convergence, not safety: Flux's inventory learns of an object only from an apply of a +// commit that holds it, and an object created through the API and never applied is not pruned +// when Git deletes it. Asked only once the branch has stayed quiet for the trigger's quiet +// period, and never of an Argo CD Application (decided without it; ApplierRef.CatchesUp). +// - Repair: the apply window withheld a late apply over a newer API edit (apply_window.go), so +// the cluster runs an older value than Git. Asked like a Refusal: on a quiet branch, with no +// quiet period, and of either kind of applier, because nothing else would put the edit back on +// an Argo CD Application. // // The three relations are kept apart on purpose. The BRANCH is the unit of quietness, because a // Kustomization applies the tip whichever GitTarget's folder an edit is in. The APPLIER is the unit -// of obligation: one attempt in flight at a time, its own retry clock, and one attempt for all the -// GitTargets that name it. The GITTARGET, by uid, is the unit of configuration and consent. +// of pending reasons: one attempt in flight at a time, its own retry clock, and one attempt for all +// the GitTargets that name it. The GITTARGET, by uid, is the unit of configuration and consent. // -// The worker owns every obligation until it is delivered. The requester carries out one attempt and -// reports back; it never retries, so a retry is decided here like any first attempt, on a quiet -// branch, against what the targets are now. - -// applyDebt is what the branch's appliers are owed because of what was OBSERVED about the remote, -// and the reports of attempts that came back. It is written from other goroutines (the first clone +// The worker keeps every pending reason until an operation answers it. The requester carries out one +// attempt and reports its operation as it moves; it never retries, so a retry is decided here like +// any first attempt, on a quiet branch, against what the targets are now. An operation is followed +// for as long as it takes: one past the requester's budget is Unresolved, and keeps its window open +// and its applier's next attempt back, because a timer that closes the window does not stop the +// apply. + +// applyInbox holds what other goroutines tell the loop about applies: the reasons pending for the +// branch's appliers because of what was OBSERVED about the remote, and the reports of attempts that +// came back. It is written from other goroutines (the first clone // publishes an observation before the loop starts; the requester reports from its own), hence the // lock. -type applyDebt struct { +type applyInbox struct { mu sync.Mutex // known is whether anything has been observed yet, and tip is the last commit observed. Empty // means the remote does not carry the branch. known bool tip string - // foreign and catchUp are owed until the loop takes them on a quiet branch. - foreign bool - catchUp bool + // foreign and catchUp are pending until the loop takes them on a quiet branch. prefetch is a push + // somebody else made, until the loop takes it (apply_groups.go). + foreign bool + catchUp bool + prefetch bool // results are the reports of finished attempts, until the loop reads them. results []ApplyResult } // observe classifies one observation. The first one is a baseline, not a move: nothing says what -// the appliers last applied, so they are owed a catch-up, never a ForeignCommit. A request on a +// the appliers last applied, so a CatchUp is pending for them, never a ForeignCommit. A request on a // restart would otherwise apply the tip before the watches have captured the edits made while the // operator was down. // // After that, a commit that differs from the last one observed is either this worker's push, proved // by the push itself, or somebody else's, because every other observation is a fetch. When in doubt // the classification errs towards foreign, which costs one apply of the tip on a quiet branch. -func (d *applyDebt) observe(observed RemoteObservation) { +func (d *applyInbox) observe(observed RemoteObservation) { d.mu.Lock() defer d.mu.Unlock() if !d.known { @@ -84,11 +97,21 @@ func (d *applyDebt) observe(observed RemoteObservation) { d.catchUp = true return } - d.foreign = true + d.foreign, d.prefetch = true, true } -// take returns what is owed and clears it. The caller restores it if it cannot act on it. -func (d *applyDebt) take() (string, bool, bool) { +// takePrefetch returns whether a push somebody else made is to be fetched ahead, with the tip, and +// clears it. +func (d *applyInbox) takePrefetch() (string, bool) { + d.mu.Lock() + defer d.mu.Unlock() + due := d.prefetch + d.prefetch = false + return d.tip, due +} + +// take returns the pending reasons and clears them. The caller restores it if it cannot act on it. +func (d *applyInbox) take() (string, bool, bool) { d.mu.Lock() defer d.mu.Unlock() tip, foreign, catchUp := d.tip, d.foreign, d.catchUp @@ -97,34 +120,34 @@ func (d *applyDebt) take() (string, bool, bool) { } // restore puts back what take returned, for a caller that could not act on it. -func (d *applyDebt) restore(foreign, catchUp bool) { +func (d *applyInbox) restore(foreign, catchUp bool) { d.mu.Lock() defer d.mu.Unlock() d.foreign = d.foreign || foreign d.catchUp = d.catchUp || catchUp } -// foreignOwed reports whether a ForeignCommit is owed, without taking it. -func (d *applyDebt) foreignOwed() bool { +// foreignPending reports whether a ForeignCommit is pending, without taking it. +func (d *applyInbox) foreignPending() bool { d.mu.Lock() defer d.mu.Unlock() return d.foreign } // tipIs reports whether the last commit observed is commit. -func (d *applyDebt) tipIs(commit string) bool { +func (d *applyInbox) tipIs(commit string) bool { d.mu.Lock() defer d.mu.Unlock() return d.known && d.tip == commit } -func (d *applyDebt) addResult(res ApplyResult) { +func (d *applyInbox) addResult(res ApplyResult) { d.mu.Lock() defer d.mu.Unlock() d.results = append(d.results, res) } -func (d *applyDebt) takeResults() []ApplyResult { +func (d *applyInbox) takeResults() []ApplyResult { d.mu.Lock() defer d.mu.Unlock() results := d.results @@ -143,11 +166,22 @@ type branchTarget struct { quietPeriod time.Duration // consents is whether it asks for a Refusal request now (ConsentsToReapply). consents bool -} - -// appliersOnBranch lists the GitTargets on this worker's (provider, branch) that name an applier. A -// GitTarget names its GitProvider in its own namespace, so the provider's namespace is the only one -// to look in. The client is cached, so this costs no round trip. + // recorded and repairPending are the trigger's status.operation and status.repairPending, as listed: + // what an earlier process, or the trigger's reconciler, recorded (apply_recovery.go). + recorded *v1alpha3.ApplyOperation + repairPending bool + // group names the GitRepository a Flux trigger's Kustomization applies, in its cluster; empty + // for an Argo CD Application or a source not read yet. prefetch is the trigger's + // status.source.prefetch (apply_groups.go). + group string + prefetch bool +} + +// appliersOnBranch lists the GitTargets on this worker's (provider, branch) that name an apply +// trigger that exists, with the applier each trigger holds. A GitTarget names its GitProvider and +// its trigger in its own namespace, so the provider's namespace is the only one to look in. The +// client is cached, so this costs no round trip. A GitTarget whose trigger does not exist (yet) +// names no applier: its ApplyTriggerReady condition says so. func (w *BranchWorker) appliersOnBranch(ctx context.Context) ([]branchTarget, error) { var targets v1alpha3.GitTargetList if err := w.Client.List(ctx, &targets, client.InNamespace(w.GitProviderNamespace)); err != nil { @@ -156,53 +190,104 @@ func (w *BranchWorker) appliersOnBranch(ctx context.Context) ([]branchTarget, er listed := make([]branchTarget, 0, len(targets.Items)) for i := range targets.Items { target := &targets.Items[i] - if !target.DeletionTimestamp.IsZero() || target.Spec.Applier == nil || + if !target.DeletionTimestamp.IsZero() || target.Spec.ApplyTriggerRef == nil || target.Spec.GitProviderRef.Name != w.GitProviderRef || target.Spec.Branch != w.Branch { continue } + trigger, err := ResolveApplyTrigger(ctx, w.Client, target) + if err != nil { + return nil, err + } + if trigger == nil || !trigger.GetDeletionTimestamp().IsZero() { + continue + } listed = append(listed, branchTarget{ - ref: itypes.NewResourceReference(target.Name, target.Namespace), - uid: target.UID, - applier: ApplierOf(target), - quietPeriod: target.Spec.Applier.EffectiveQuietPeriod(), - consents: ConsentsToReapply(target), + ref: itypes.NewResourceReference(target.Name, target.Namespace), + uid: target.UID, + applier: ApplierOf(trigger), + quietPeriod: trigger.EffectiveQuietPeriod(), + consents: ConsentsToReapply(target), + recorded: trigger.Operation(), + repairPending: trigger.RepairPending(), + group: sourceGroup(trigger), + prefetch: trigger.Prefetches(), }) } sort.Slice(listed, func(i, j int) bool { return listed[i].ref.String() < listed[j].ref.String() }) return listed, nil } -// applierState is what one applier on this branch is owed, and how its attempts are going. +// applierState is the reasons pending for one applier on this branch, and how its attempts are going. type applierState struct { ref ApplierRef // targets are the GitTargets that name it, as listed on the last pass. targets []branchTarget - // foreign and catchUp are owed until an attempt carries them. + // foreign, catchUp and repair are pending until an attempt carries them. foreign bool catchUp bool - // inflight is the attempt the requester has not reported yet. While it is out, nothing else is - // sent to this applier: what is owed meanwhile waits for the report. + repair bool + // inflight is the attempt whose report has not ended it yet. While it is out, nothing else is + // sent to this applier: a reason that becomes pending meanwhile waits for it. inflight *inflightAttempt - // failures counts attempts that did not deliver since the last that did, and retryAt is when the - // next attempt may go. + // failures counts attempts that did not succeed since the last that did, and retryAt is when + // the next attempt may go. failures int retryAt time.Time + // lastTurn is the id of the last attempt sent to it, zero before the first: a group's next + // attempt goes to the applier whose turn is longest ago (turnOrder). + lastTurn uint64 // window is the applier's apply window: open from sending an attempt until the applier - // finished it (apply_window.go). Nil when no attempt was sent, or the last asked for no apply. + // finished it (apply_window.go). Nil when no attempt was sent, or the last asked for no apply + // after the window before it had closed. window *applyWindow -} - -// inflightAttempt is what an attempt carries away from its applier's obligations. A failed attempt -// gives it back (restoreAttempt); a delivered one keeps it. + // operation is the last operation an attempt started, as its trigger's status.operation shows + // it, and operationID the attempt it belongs to. Nil until this worker started one. + operation *v1alpha3.ApplyOperation + operationID uint64 + // published is what was last handed to the trigger status writer, nil before the first. + published *ApplyTriggerStatus + // withdrawn is an operation an attempt recorded on the trigger and then did not start, until + // it is published as withdrawn. + withdrawn *v1alpha3.ApplyOperation + // recovered is whether the trigger's recorded status was read back for this applier, once, + // when this process first listed it; followedSuspend is the suspend token of the last operation + // the trigger's reconciler recorded that this worker followed (apply_recovery.go). + recovered bool + followedSuspend string +} + +// repairPending reports whether a Repair is pending: not yet carried, or carried by the attempt out. +func (s *applierState) repairPending() bool { + return s.repair || (s.inflight != nil && s.inflight.repair) +} + +// inflightAttempt is what an attempt carries away from its applier's pending reasons. One that did +// not succeed gives them back (restoreAttempt); one that succeeded keeps it. type inflightAttempt struct { id uint64 foreign bool catchUp bool - refusals map[refusalKey]owedRefusal -} - -// quietPeriod is how long the branch must be quiet before a CatchUp: the longest among the -// GitTargets naming the applier, because its apply reaches every one of their folders. + repair bool + refusals map[refusalKey]unsentRefusal + // phase is the last report of its operation, empty before the first. + phase ApplyOutcome + // previous is the operation recorded before this attempt's, with the attempt it belonged to, + // put back should this attempt ask for no apply. + previous *v1alpha3.ApplyOperation + previousID uint64 + // recovered is an attempt that asks nothing: it follows an operation recorded on the trigger, + // by an earlier process or by the trigger's reconciler, to its end (apply_recovery.go). + recovered bool + // recorded is set, on the requester's goroutine and before its reports, once the requester + // called the attempt's Record: from then on the trigger may hold its operation, so an ending + // that asked for no apply withdraws it. + recorded atomic.Bool + // abandon closes the request's Abandoned channel. + abandon context.CancelFunc +} + +// quietPeriod is how long the branch must be quiet before a CatchUp: the trigger's, as listed with +// each GitTarget naming it. The longest wins should a listing straddle an edit of the trigger. func (s *applierState) quietPeriod() time.Duration { var longest time.Duration for _, target := range s.targets { @@ -229,7 +314,7 @@ func (l *branchWorkerEventLoop) closeWindowForForeignCommit() { if l.openWindow == nil || l.w.applyRequester == nil { return } - if !hinted && !l.w.debt.foreignOwed() { + if !hinted && !l.w.inbox.foreignPending() { return } l.w.Log.Info("The branch moved, or may have; closing the commit window to publish it now", @@ -245,22 +330,30 @@ func (l *branchWorkerEventLoop) serviceAppliers() { if w.applyRequester == nil { return } + defer l.publishTriggerStatus() + defer l.publishGroupsOut() now := time.Now() l.settleApplyResults(now) - if !l.branchQuiet(now) || len(w.eventQueue) > 0 { - // Queued work: the next wake looks again. The branch is still quiet until that work opens - // a window, so quietSince is kept. + l.followRecordedOperations(now) + if !l.branchQuiet(now) || l.workWaiting() { + // Queued work, here or on a sibling: the next wake looks again, and a sibling's loop wakes + // this one once it has handled its last item. The branch is still quiet until that work + // opens a window, so quietSince is kept. A push observed meanwhile is fetched ahead of the + // apply. + l.prefetchSources() return } + // Quiet: the attempts sent now fetch what they apply themselves. + w.inbox.takePrefetch() - tip, foreign, catchUp := w.debt.take() - if !foreign && !catchUp && len(l.reapplyOwed) == 0 && !l.appliersOwe() { + tip, foreign, catchUp := w.inbox.take() + if !foreign && !catchUp && len(l.unsentRefusals) == 0 && !l.appliersHaveWork() { l.stopApplierTimer() return } targets, err := w.appliersOnBranch(w.ctx) if err != nil { - w.debt.restore(foreign, catchUp) + w.inbox.restore(foreign, catchUp) w.Log.Error(err, "Cannot list the appliers on this branch; retrying", "branch", w.Branch) l.resetApplierTimer(applierRetryInterval) return @@ -279,7 +372,8 @@ func (l *branchWorkerEventLoop) serviceAppliers() { func (l *branchWorkerEventLoop) branchQuiet(now time.Time) bool { busy := l.writeBusy() l.w.publishWriteBusy(busy) - if busy || l.w.siblingWriteBusy() { + siblingWriting, _ := l.w.siblingBusy() + if busy || siblingWriting { l.quietSince = time.Time{} l.stopApplierTimer() return false @@ -290,10 +384,10 @@ func (l *branchWorkerEventLoop) branchQuiet(now time.Time) bool { return true } -// appliersOwe reports whether any applier is owed something or has an attempt out. -func (l *branchWorkerEventLoop) appliersOwe() bool { +// appliersHaveWork reports whether any applier has a reason pending or an attempt out. +func (l *branchWorkerEventLoop) appliersHaveWork() bool { for _, state := range l.appliers { - if state.foreign || state.catchUp || state.inflight != nil { + if state.foreign || state.catchUp || state.repair || state.inflight != nil { return true } } @@ -301,8 +395,14 @@ func (l *branchWorkerEventLoop) appliersOwe() bool { } // syncAppliers makes the applier states match the GitTargets listed now. An applier no GitTarget -// names any more is forgotten, with whatever it was owed: nobody asks for it. A refusal owed for a -// GitTarget that is gone, or is now another incarnation, is dropped the same way. +// names any more is forgotten, with whatever was pending for it: nobody asks for it, and its +// operation, if one is out, is abandoned. An applier named for the first time starts where a first +// observation does, with a CatchUp pending where its kind catches up: a commit observed before it +// was named was pending for nobody, and was taken, so without one it would not be applied until +// some later push. For a +// Kustomization that already applied the tip it costs nothing (Current); an Application gets no +// CatchUp, and waits for the next push. An unsent refusal of a GitTarget that is gone, or is now +// another incarnation, is dropped the same way. func (l *branchWorkerEventLoop) syncAppliers(targets []branchTarget) { if l.appliers == nil { l.appliers = map[ApplierRef]*applierState{} @@ -313,28 +413,37 @@ func (l *branchWorkerEventLoop) syncAppliers(targets []branchTarget) { named[target.applier] = append(named[target.applier], target) uids[target.ref] = target.uid } - for ref := range l.appliers { + for ref, state := range l.appliers { if _, ok := named[ref]; !ok { + if state.inflight != nil { + state.inflight.abandon() + } + if state.withdrawn != nil { + l.publishStatusOf(state) + } delete(l.appliers, ref) } } + l.publishUnresolved() for ref, named := range named { state := l.appliers[ref] if state == nil { - state = &applierState{ref: ref} + state = &applierState{ref: ref, catchUp: ref.CatchesUp()} l.appliers[ref] = state } state.targets = named } - for key, owed := range l.reapplyOwed { - if uids[key.target] != owed.uid { - delete(l.reapplyOwed, key) + for key, refusal := range l.unsentRefusals { + if uids[key.target] != refusal.uid { + delete(l.unsentRefusals, key) } } } -// sendDue sends an attempt to every applier that is owed one, is not waiting on one, and whose -// retry and quiet periods have run out. It returns the earliest time one still waiting comes due. +// sendDue sends an attempt to every applier that has a reason pending, is not waiting on one, and whose +// retry and quiet periods have run out. The appliers are taken in turn order, so in a group, which +// has one attempt out at a time, the attempt goes to the applier whose turn is longest ago. It +// returns the earliest time one still waiting comes due. func (l *branchWorkerEventLoop) sendDue(tip string, now time.Time) time.Time { var earliest time.Time wait := func(at time.Time) { @@ -342,55 +451,66 @@ func (l *branchWorkerEventLoop) sendDue(tip string, now time.Time) time.Time { earliest = at } } - for _, ref := range l.sortedAppliers() { + for _, ref := range l.turnOrder() { state := l.appliers[ref] - if state.inflight != nil { + if l.attemptOut(state) { continue } - refusals := l.refusalsOwed(state) - if !state.foreign && len(refusals) == 0 && !state.catchUp { + refusals := l.unsentRefusalsFor(state) + if !state.foreign && !state.repair && len(refusals) == 0 && !state.catchUp { continue } - due := state.retryAt - if !state.foreign && len(refusals) == 0 { - due = latest(due, l.quietSince.Add(state.quietPeriod())) - } + due := l.dueAt(state, refusals) if now.Before(due) { wait(due) continue } + if !l.w.claimGroup(state.group()) { + // A sibling sent one for the group since attemptOut looked; its end wakes this worker. + continue + } l.sendAttempt(state, tip, refusals) } return earliest } -// refusalsOwed is the refusals owed for the GitTargets naming this applier that still consent. A -// refusal of a target that no longer consents is dropped: its consent was the obligation. -func (l *branchWorkerEventLoop) refusalsOwed(state *applierState) map[refusalKey]owedRefusal { +// dueAt is when an applier's next attempt may go: once its backoff has run out, and, when only a +// CatchUp is pending, once the branch has been quiet for the trigger's quiet period. +func (l *branchWorkerEventLoop) dueAt(state *applierState, refusals map[refusalKey]unsentRefusal) time.Time { + if state.foreign || state.repair || len(refusals) > 0 { + return state.retryAt + } + return latest(state.retryAt, l.quietSince.Add(state.quietPeriod())) +} + +// unsentRefusalsFor is the unsent refusals of the GitTargets naming this applier that still consent. +// A refusal of a target that no longer consents is dropped: its consent was the only reason to send it. +func (l *branchWorkerEventLoop) unsentRefusalsFor(state *applierState) map[refusalKey]unsentRefusal { consents := map[itypes.ResourceReference]bool{} for _, target := range state.targets { consents[target.ref] = target.consents } - owed := map[refusalKey]owedRefusal{} - for key, refusal := range l.reapplyOwed { + unsent := map[refusalKey]unsentRefusal{} + for key, refusal := range l.unsentRefusals { consenting, named := consents[key.target] if !named { continue } if !consenting { - delete(l.reapplyOwed, key) + delete(l.unsentRefusals, key) continue } - owed[key] = refusal + unsent[key] = refusal } - return owed + return unsent } // sendAttempt hands one attempt for an applier to the requester and moves what it carries out of -// the applier's obligations, into the attempt. Each target is asked for its own reason: every one -// for a ForeignCommit; otherwise a Refusal for a target with a refusal owed, and a CatchUp for the -// rest when one is owed. Any delivered attempt applies the tip, so it answers an owed CatchUp too. -func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, refusals map[refusalKey]owedRefusal) { +// the applier's pending reasons, into the attempt. Each target is asked for its own reason: every +// one for a ForeignCommit, then for a Repair; otherwise a Refusal for a target with an unsent +// refusal, and a CatchUp for the rest when one is pending. Any operation that succeeds applies the +// tip, so it answers a pending CatchUp or Repair too. +func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, refusals map[refusalKey]unsentRefusal) { refused := map[itypes.ResourceReference]bool{} for key := range refusals { refused[key.target] = true @@ -402,6 +522,8 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref switch { case state.foreign: own = v1alpha3.ApplyReasonForeignCommit + case state.repair: + own = v1alpha3.ApplyReasonRepair case refused[target.ref]: own = v1alpha3.ApplyReasonRefusal case state.catchUp: @@ -414,14 +536,29 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref } l.nextApplyID++ + abandoned, abandon := context.WithCancel(l.w.ctx) + sentAt := time.Now() attempt := &inflightAttempt{ - id: l.nextApplyID, foreign: state.foreign, catchUp: state.catchUp, refusals: refusals, + id: l.nextApplyID, foreign: state.foreign, catchUp: state.catchUp, repair: state.repair, + refusals: refusals, abandon: abandon, + previous: state.operation, previousID: state.operationID, } - state.foreign, state.catchUp, state.inflight = false, false, attempt - state.openApplyWindow(attempt.id, tip, time.Now()) + state.foreign, state.catchUp, state.repair, state.inflight = false, false, false, attempt + state.lastTurn = attempt.id + state.openApplyWindow(attempt.id, tip, sentAt) for key := range refusals { - delete(l.reapplyOwed, key) + delete(l.unsentRefusals, key) + } + // What the trigger shows now goes out before the request, so the record replaces the latest of + // it: the operation before this one may have completed on this very pass. + l.publishStatusOf(state) + token := operationToken(sentAt) + state.operation = &v1alpha3.ApplyOperation{ + Phase: v1alpha3.ApplyOperationRequested, Reason: reason, Commit: tip, Token: token, + RequestedAt: metav1.NewTime(sentAt), } + state.operationID = attempt.id + record := l.recorder(state, attempt) w := l.w w.Log.Info("Asking the applier to apply", @@ -429,65 +566,243 @@ func (l *branchWorkerEventLoop) sendAttempt(state *applierState, tip string, ref "gitTargets", len(targets), "priorFailures", state.failures) w.applyRequester.RequestApply(ApplyRequest{ ID: attempt.id, - Branch: BranchKey{RepoNamespace: w.GitProviderNamespace, RepoName: w.GitProviderRef, Branch: w.Branch}, + Branch: w.branchKey(), Applier: state.ref, Reason: reason, Commit: tip, + Token: token, + Record: record, Targets: targets, PriorFailures: state.failures, Current: func() bool { return w.decisionHolds(tip) }, Report: w.reportApplyResult, + Abandoned: abandoned.Done(), }) } -// settleApplyResults reads the reports that came back. A delivered attempt is done. One that did -// not deliver gives back what it carried and waits out its applier's backoff, so its retry is -// decided again on a quiet branch at whatever the tip is then, never replayed blind. A report for an -// applier no GitTarget names any more is dropped with it. +// settleApplyResults reads the reports that came back. A report of an operation still out records +// where it stands, and holds the attempt. An operation that succeeded is done. Any other ending +// gives back what the attempt carried and, unless it was only stale, waits out its applier's +// backoff, so its retry is decided again on a quiet branch at whatever the tip is then, never +// replayed blind. A report for an applier no GitTarget names any more is dropped with it. func (l *branchWorkerEventLoop) settleApplyResults(now time.Time) { - for _, res := range l.w.debt.takeResults() { + defer l.publishUnresolved() + for _, res := range l.w.inbox.takeResults() { state := l.applierAwaiting(res.ID) if state == nil { continue } attempt := state.inflight + state.recordOperation(attempt, res, now) + if !res.Outcome.Ends() { + l.advanceOperation(state, attempt, res.Outcome) + continue + } state.inflight = nil - state.settleApplyWindow(attempt.id, res.Outcome == ApplyDelivered, now) + attempt.abandon() + state.settleApplyWindow(attempt.id, res.Outcome.Operated(), now) + if attempt.phase == ApplyUnresolved { + l.w.Log.Info("An unresolved apply completed; its apply window closes after the grace", + "branch", l.w.Branch, "applier", state.ref.String(), "outcome", string(res.Outcome)) + } switch res.Outcome { - case ApplyDelivered, ApplyCurrent: + case ApplySucceeded, ApplyCurrent: state.failures, state.retryAt = 0, time.Time{} continue case ApplyStale: - // The decision went stale before the patch: nothing failed, and the next quiet moment + // The decision went stale before the write: nothing failed, and the next quiet moment // decides again, at the tip as it is then. l.restoreAttempt(state, attempt) continue - case ApplyFailed, ApplyRejected, ApplyDropped: + case ApplyFailed, ApplyRequestFailed, ApplyRejected, ApplyDropped: + case ApplyRequested, ApplyRunning, ApplySettling, ApplyUnresolved: } state.failures++ state.retryAt = now.Add(applierRetryDelay(state.failures, res.Outcome == ApplyRejected)) l.restoreAttempt(state, attempt) - l.w.Log.Info("An apply attempt did not deliver; deciding it again on a quiet branch", + message := "An apply attempt did not deliver; deciding it again on a quiet branch" + if res.Outcome == ApplyFailed { + message = "The applier did not complete the apply; deciding it again on a quiet branch" + } + l.w.Log.Info(message, "branch", l.w.Branch, "applier", state.ref.String(), "outcome", string(res.Outcome), "failures", state.failures, "retryIn", state.retryAt.Sub(now), "error", errString(res.Err)) } } -// restoreAttempt gives an applier back what an attempt that did not deliver carried. A refusal comes +// advanceOperation records where an attempt's operation stands. Unresolved is said once, and stays +// until the operation ends: it is the requester's verdict that the operation is overdue, which a +// later sight of it running does not take back. +func (l *branchWorkerEventLoop) advanceOperation(state *applierState, attempt *inflightAttempt, phase ApplyOutcome) { + if attempt.phase == ApplyUnresolved { + return + } + attempt.phase = phase + if phase != ApplyUnresolved { + return + } + l.w.Log.Info("The applier has not completed the apply in time; its apply window stays open, and "+ + "nothing more is asked of it until the apply completes", "branch", l.w.Branch, + "applier", state.ref.String(), "attempt", attempt.id) +} + +// operationPhase is the phase of the operation an attempt's report describes, and false for the +// outcomes that started no operation, which leave the last one recorded as it is. +func operationPhase(outcome ApplyOutcome) (v1alpha3.ApplyOperationPhase, bool) { + switch outcome { + case ApplyRequested: + return v1alpha3.ApplyOperationRequested, true + case ApplyRunning: + return v1alpha3.ApplyOperationRunning, true + case ApplySettling: + return v1alpha3.ApplyOperationSettling, true + case ApplyUnresolved: + return v1alpha3.ApplyOperationUnresolved, true + case ApplySucceeded: + return v1alpha3.ApplyOperationSucceeded, true + case ApplyFailed: + return v1alpha3.ApplyOperationFailed, true + case ApplyCurrent, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyStale: + } + return "", false +} + +// recordOperation records an attempt's report in the operation its trigger's status shows, which +// sendAttempt started as Requested when it sent the attempt. A report that ends an attempt that +// asked for no apply puts back the operation before it, and withdraws its own from the trigger if +// the requester recorded it there. Unresolved stays until the operation completes, as +// advanceOperation keeps it. +func (s *applierState) recordOperation(attempt *inflightAttempt, res ApplyResult, now time.Time) { + if s.operationID != attempt.id { + return + } + phase, operated := operationPhase(res.Outcome) + if !operated { + if res.Outcome.Ends() { + // It asked for no apply: its operation stands for nothing. + if attempt.recorded.Load() { + s.withdrawn = s.operation + } + s.operation, s.operationID = attempt.previous, attempt.previousID + } + return + } + operation := s.operation.DeepCopy() + if res.SuspendToken != "" { + operation.SuspendToken = res.SuspendToken + } + if operation.Phase != v1alpha3.ApplyOperationUnresolved || phase.Completed() { + operation.Phase = phase + } + switch { + case phase.Completed(): + finished := metav1.NewTime(now) + operation.FinishedAt, operation.AppliedRevision = &finished, res.AppliedRevision + operation.Message = errString(res.Err) + if res.Err == nil { + operation.Message = res.Message + } + case phase == v1alpha3.ApplyOperationUnresolved: + operation.Message = "the applier has not completed the apply in the time it was given; it is still " + + "followed, and nothing more is asked of it until it completes" + } + s.operation, s.operationID = operation, attempt.id +} + +// operationToken is the token an operation started at sentAt carries: the time, to the nanosecond, +// as `flux reconcile` stamps a requestedAt. +func operationToken(sentAt time.Time) string { + return sentAt.UTC().Format(time.RFC3339Nano) +} + +// recorder is the attempt's ApplyRequest.Record: the requester writes the operation the attempt +// starts through to the trigger, just before it starts the apply. That is the only way it reaches +// the trigger before the requester reports it, so an attempt that ends before it leaves the trigger +// as it was. Nil without a trigger status writer. +func (l *branchWorkerEventLoop) recorder(state *applierState, attempt *inflightAttempt) func(context.Context) error { + writer := l.w.triggerStatus + if writer == nil { + return nil + } + trigger, operation := state.ref.Trigger, state.operation.DeepCopy() + return func(ctx context.Context) error { + attempt.recorded.Store(true) + return writer.RecordApplyOperation(ctx, trigger, operation) + } +} + +// shownOperation is the operation the applier's trigger is to show: the last one an attempt +// started, once the requester reported it, and the one before it until then. An attempt that +// follows a recorded operation shows it from the start: the trigger holds it already. +func (s *applierState) shownOperation() *v1alpha3.ApplyOperation { + if attempt := s.inflight; attempt != nil && !attempt.recovered && attempt.phase == "" && + s.operationID == attempt.id { + return attempt.previous + } + return s.operation +} + +// publishTriggerStatus hands each applier's trigger what its status should show, when that changed +// since the last time. +func (l *branchWorkerEventLoop) publishTriggerStatus() { + for _, ref := range l.sortedAppliers() { + l.publishStatusOf(l.appliers[ref]) + } +} + +// publishStatusOf hands one applier's trigger what its status should show, and the operation it +// withdraws, when that changed since the last time. A worker that has started no operation and +// has no Repair pending has nothing to say yet, and says nothing: the status may hold what an earlier +// process recorded. +func (l *branchWorkerEventLoop) publishStatusOf(state *applierState) { + writer := l.w.triggerStatus + if writer == nil { + return + } + next := ApplyTriggerStatus{ + Operation: state.shownOperation(), Withdrawn: state.withdrawn, RepairPending: state.repairPending(), + } + if next.Withdrawn == nil { + if state.published == nil && next.Operation == nil && !next.RepairPending { + return + } + if state.published != nil && state.published.RepairPending == next.RepairPending && + equality.Semantic.DeepEqual(state.published.Operation, next.Operation) { + return + } + } + writer.PublishApplyTriggerStatus(state.ref.Trigger, next) + state.published, state.withdrawn = &next, nil +} + +// publishUnresolved publishes how many of this branch's operations are Unresolved, for the +// apply_operations_unresolved gauge. +func (l *branchWorkerEventLoop) publishUnresolved() { + var unresolved int64 + for _, state := range l.appliers { + if state.inflight != nil && state.inflight.phase == ApplyUnresolved { + unresolved++ + } + } + l.w.unresolvedOperations.Store(unresolved) +} + +// restoreAttempt gives an applier back what an attempt that did not succeed carried. A refusal comes // back only while it still stands: one that recovered while the attempt was out has nothing left to // revert. func (l *branchWorkerEventLoop) restoreAttempt(state *applierState, attempt *inflightAttempt) { state.foreign = state.foreign || attempt.foreign state.catchUp = state.catchUp || attempt.catchUp + state.repair = state.repair || attempt.repair for key, refusal := range attempt.refusals { if !l.w.refusalStands(key) { continue } - if l.reapplyOwed == nil { - l.reapplyOwed = map[refusalKey]owedRefusal{} + if l.unsentRefusals == nil { + l.unsentRefusals = map[refusalKey]unsentRefusal{} } - if _, owed := l.reapplyOwed[key]; !owed { - l.reapplyOwed[key] = refusal + if _, unsent := l.unsentRefusals[key]; !unsent { + l.unsentRefusals[key] = refusal } } } @@ -502,18 +817,39 @@ func (l *branchWorkerEventLoop) applierAwaiting(id uint64) *applierState { return nil } +// workWaiting reports whether work items are queued on this worker, or queued or in hand on another +// writing this repository and branch. +func (l *branchWorkerEventLoop) workWaiting() bool { + _, siblingWorking := l.w.siblingBusy() + return len(l.w.eventQueue) > 0 || siblingWorking +} + +// branchKey is the repository and branch this worker writes. +func (w *BranchWorker) branchKey() BranchKey { + return BranchKey{RepoNamespace: w.GitProviderNamespace, RepoName: w.GitProviderRef, Branch: w.Branch} +} + // decisionHolds is ApplyRequest.Current for an attempt decided at tip: the branch is still at that -// commit, and nothing live is waiting to be published on this worker or a sibling. Work still in the -// queue counts, because a queued watch event is an edit that is live and not yet decided. It is -// called from the requester's goroutine, so it reads only what is safe to read there. +// commit, and nothing live is waiting to be published on this worker or a sibling. A work item +// counts from its enqueue until its wake has published what it left behind (inflightItems), not +// only while it waits in the queue: a queued watch event is an edit that is live and not yet +// decided, and the item the loop is handling may be committing and pushing while busyWriting still +// says what the last wake left. It is called from the requester's goroutine, so it reads only what +// is safe to read there. +// +// An attempt sent from the wake of a queue item can find that item still counted, should the +// requester ask within moments. It is then Stale, which costs no backoff, and the report wakes the +// loop without an item, so the next attempt is not held by one of its own. func (w *BranchWorker) decisionHolds(tip string) bool { - return w.debt.tipIs(tip) && !w.busyWriting.Load() && len(w.eventQueue) == 0 && !w.siblingWriteBusy() + siblingWriting, siblingWorking := w.siblingBusy() + return w.inbox.tipIs(tip) && !w.busyWriting.Load() && w.inflightItems.Load() == 0 && + !siblingWriting && !siblingWorking } // reportApplyResult is ApplyRequest.Report: it records the report and wakes the loop to settle it. // It is called from the requester's goroutine. func (w *BranchWorker) reportApplyResult(res ApplyResult) { - w.debt.addResult(res) + w.inbox.addResult(res) w.wakeApplier() } diff --git a/internal/git/apply_trigger_test.go b/internal/git/apply_trigger_test.go index e48fa710c..a680157c9 100644 --- a/internal/git/apply_trigger_test.go +++ b/internal/git/apply_trigger_test.go @@ -2,8 +2,8 @@ package git -// The branch worker's half of the apply trigger, from docs/design/apply-trigger.md: what the appliers -// are owed, and the rule that nothing is asked while anything live is unpublished. +// The branch worker's half of the apply trigger, from docs/design/apply-trigger.md: which reasons are +// pending for the appliers, and the rule that nothing is asked while anything live is unpublished. import ( "context" @@ -21,18 +21,31 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "errors" + + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) +// targetRefs lists a request's GitTargets. +func targetRefs(req ApplyRequest) []itypes.ResourceReference { + refs := make([]itypes.ResourceReference, 0, len(req.Targets)) + for _, target := range req.Targets { + refs = append(refs, target.Ref) + } + return refs +} + func TestApplyDebt_FirstObservationIsABaseline(t *testing.T) { - var debt applyDebt - debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + var inbox applyInbox + inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) - tip, foreign, catchUp := debt.take() + tip, foreign, catchUp := inbox.take() assert.Equal(t, "aaa", tip) assert.False(t, foreign, "nothing says the applier has not applied what was there at startup") - assert.True(t, catchUp, "but nothing says it has, so it is owed a catch-up") + assert.True(t, catchUp, "but nothing says it has, so a catch-up is pending") } func TestApplyDebt_ClassifiesEachMove(t *testing.T) { @@ -42,22 +55,22 @@ func TestApplyDebt_ClassifiesEachMove(t *testing.T) { wantForeign bool wantCatchUp bool }{ - {name: "an unchanged tip owes nothing", next: RemoteObservation{Commit: "aaa", By: ObservedByFetch}}, - {name: "our own push owes a catch-up", next: RemoteObservation{Commit: "bbb", By: ObservedByPush}, + {name: "an unchanged tip leaves nothing pending", next: RemoteObservation{Commit: "aaa", By: ObservedByFetch}}, + {name: "our own push leaves a catch-up pending", next: RemoteObservation{Commit: "bbb", By: ObservedByPush}, wantCatchUp: true}, - {name: "a fetch that found the branch moved owes a ForeignCommit", + {name: "a fetch that found the branch moved leaves a ForeignCommit pending", next: RemoteObservation{Commit: "bbb", By: ObservedByFetch}, wantForeign: true}, {name: "a deleted branch is a move like any other", next: RemoteObservation{By: ObservedByFetch}, wantForeign: true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - var debt applyDebt - debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) - debt.take() + var inbox applyInbox + inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + inbox.take() - debt.observe(tc.next) - tip, foreign, catchUp := debt.take() + inbox.observe(tc.next) + tip, foreign, catchUp := inbox.take() assert.Equal(t, tc.next.Commit, tip) assert.Equal(t, tc.wantForeign, foreign) assert.Equal(t, tc.wantCatchUp, catchUp) @@ -66,24 +79,34 @@ func TestApplyDebt_ClassifiesEachMove(t *testing.T) { } func TestApplyDebt_RestorePutsBackWhatWasTaken(t *testing.T) { - var debt applyDebt - debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) - debt.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) - _, foreign, catchUp := debt.take() + var inbox applyInbox + inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) + _, foreign, catchUp := inbox.take() require.True(t, foreign) - debt.restore(foreign, catchUp) - assert.True(t, debt.foreignOwed()) + inbox.restore(foreign, catchUp) + assert.True(t, inbox.foreignPending()) } -// applierRef is the applier testApplier(name) names for a GitTarget in team-a. +// applierRef is the applier the trigger testApplier(name) names holds, for a GitTarget in team-a. func applierRef(name string) ApplierRef { return ApplierRef{ + Trigger: TriggerRef{Kind: configv1alpha3.ApplyTriggerKindFlux, Namespace: "team-a", Name: name}, Cluster: configv1alpha3.DefaultClusterProviderName, Kind: configv1alpha3.ApplierKindKustomization, Namespace: "team-a", Name: name, } } +// argoApplierRef is the applier the ArgoCDTrigger argoApplier names holds. +func argoApplierRef(name string) ApplierRef { + return ApplierRef{ + Trigger: TriggerRef{Kind: configv1alpha3.ApplyTriggerKindArgoCD, Namespace: "team-a", Name: name}, + Cluster: configv1alpha3.DefaultClusterProviderName, Kind: configv1alpha3.ApplierKindApplication, + Namespace: "argocd", Name: name, + } +} + // applierTestTarget is a GitTarget on the trigger worker's provider and branch unless overridden. func applierTestTarget(name string, applier bool, mutate ...func(*configv1alpha3.GitTarget)) *configv1alpha3.GitTarget { target := &configv1alpha3.GitTarget{ @@ -95,7 +118,7 @@ func applierTestTarget(name string, applier bool, mutate ...func(*configv1alpha3 }, } if applier { - target.Spec.Applier = testApplier(name) + target.Spec.ApplyTriggerRef = testApplier(name) } for _, m := range mutate { m(target) @@ -104,7 +127,8 @@ func applierTestTarget(name string, applier bool, mutate ...func(*configv1alpha3 } // applyTriggerLoop is an event loop on provider team-a/test-provider, branch main, whose fake client -// holds the given GitTargets and whose requests are recorded. +// holds the given objects and the trigger each GitTarget among them names, and whose requests are +// recorded. func applyTriggerLoop(t *testing.T, targets ...client.Object) (*branchWorkerEventLoop, *recordingRequester) { t.Helper() scheme := runtime.NewScheme() @@ -113,7 +137,8 @@ func applyTriggerLoop(t *testing.T, targets ...client.Object) (*branchWorkerEven w := newMetricsTestWorker() w.GitProviderNamespace = "team-a" - w.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(targets...).Build() + w.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(testTriggersFor(targets)...). + WithStatusSubresource(&configv1alpha3.FluxTrigger{}, &configv1alpha3.ArgoCDTrigger{}).Build() w.Log = logr.Discard() w.ctx = context.Background() requester := &recordingRequester{} @@ -123,11 +148,11 @@ func applyTriggerLoop(t *testing.T, targets ...client.Object) (*branchWorkerEven return loop, requester } -// owesForeign makes the worker's debt hold a ForeignCommit at commit "bbb". -func owesForeign(w *BranchWorker) { - w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) - w.debt.take() - w.debt.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) +// observeForeignCommit makes the worker's inbox hold a ForeignCommit at commit "bbb". +func observeForeignCommit(w *BranchWorker) { + w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + w.inbox.take() + w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByFetch}) } func TestServiceAppliers_ForeignCommitAsksEveryApplierOnTheBranch(t *testing.T) { @@ -140,7 +165,7 @@ func TestServiceAppliers_ForeignCommitAsksEveryApplierOnTheBranch(t *testing.T) g.Spec.GitProviderRef.Name = "elsewhere" }), ) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() @@ -153,7 +178,7 @@ func TestServiceAppliers_ForeignCommitAsksEveryApplierOnTheBranch(t *testing.T) requests[i].Branch) assert.Equal(t, applierRef(name), requests[i].Applier) assert.Equal(t, []itypes.ResourceReference{itypes.NewResourceReference(name, "team-a")}, - requests[i].TargetRefs(), "a suspended target stops writing, not deployments") + targetRefs(requests[i]), "a suspended target stops writing, not deployments") } } @@ -172,28 +197,24 @@ func TestServiceAppliers_NothingIsSentWhileAnythingLiveIsUnpublished(t *testing. for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - owesForeign(loop.w) + observeForeignCommit(loop.w) tc.busy(loop) loop.serviceAppliers() assert.Empty(t, requester.taken()) assert.True(t, loop.quietSince.IsZero(), "a busy branch is not quiet") - assert.True(t, loop.w.debt.foreignOwed(), "and the debt waits for it") + assert.True(t, loop.w.inbox.foreignPending(), "and the pending ForeignCommit waits for it") }) } } func TestServiceAppliers_CatchUpWaitsForTheQuietPeriod(t *testing.T) { loop, requester := applyTriggerLoop(t, - applierTestTarget("soon", true, func(g *configv1alpha3.GitTarget) { - g.Spec.Applier.QuietPeriod = &metav1.Duration{Duration: time.Minute} - }), - applierTestTarget("late", true, func(g *configv1alpha3.GitTarget) { - g.Spec.Applier.QuietPeriod = &metav1.Duration{Duration: time.Hour} - }), + applierTestTarget("soon", true), fluxTrigger("soon", quietFor(time.Minute)), + applierTestTarget("late", true), fluxTrigger("late", quietFor(time.Hour)), ) - loop.w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) loop.serviceAppliers() assert.Empty(t, requester.taken(), "the branch has only just become quiet") @@ -208,30 +229,31 @@ func TestServiceAppliers_CatchUpWaitsForTheQuietPeriod(t *testing.T) { assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) assert.Equal(t, "aaa", requests[0].Commit) assert.Equal(t, []itypes.ResourceReference{itypes.NewResourceReference("soon", "team-a")}, - requests[0].TargetRefs()) + targetRefs(requests[0])) assert.True(t, loop.appliers[applierRef("late")].catchUp, "the one-hour applier still waits") assert.NotNil(t, loop.applierTimer) } -// argoApplier makes a GitTarget's applier an Argo CD Application of its own name. +// argoApplier makes a GitTarget name the ArgoCDTrigger of its own name, which holds the Application +// argocd/. func argoApplier(g *configv1alpha3.GitTarget) { - g.Spec.Applier = &configv1alpha3.GitTargetApplier{ - ApplicationRef: &meta.NamespacedObjectReference{Namespace: "argocd", Name: g.Name}, + g.Spec.ApplyTriggerRef = &configv1alpha3.ApplyTriggerReference{ + Kind: configv1alpha3.ApplyTriggerKindArgoCD, Name: g.Name, } } -// TestServiceAppliers_AnApplicationIsNeverOwedACatchUp: with automated sync off Argo CD applies -// nothing on its own, so GitOps Reverser's own pushes (and the baseline after a start) owe an -// Application nothing, however long the branch stays quiet; a Kustomization beside it on the same +// TestServiceAppliers_AnApplicationIsNeverSentACatchUp: with automated sync off Argo CD applies +// nothing on its own, so GitOps Reverser's own pushes (and the baseline after a start) ask an +// Application for nothing, however long the branch stays quiet; a Kustomization beside it on the same // branch is still caught up. -func TestServiceAppliers_AnApplicationIsNeverOwedACatchUp(t *testing.T) { +func TestServiceAppliers_AnApplicationIsNeverSentACatchUp(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("argo", true, argoApplier), applierTestTarget("flux", true), ) - loop.w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // the baseline + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) // the baseline loop.serviceAppliers() - loop.w.debt.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) loop.serviceAppliers() loop.quietSince = time.Now().Add(-time.Hour) loop.serviceAppliers() @@ -240,10 +262,7 @@ func TestServiceAppliers_AnApplicationIsNeverOwedACatchUp(t *testing.T) { require.Len(t, requests, 1, "only the Kustomization catches up") assert.Equal(t, applierRef("flux"), requests[0].Applier) assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) - argo := ApplierRef{ - Cluster: configv1alpha3.DefaultClusterProviderName, Kind: configv1alpha3.ApplierKindApplication, - Namespace: "argocd", Name: "argo", - } + argo := argoApplierRef("argo") require.Contains(t, loop.appliers, argo) assert.False(t, loop.appliers[argo].catchUp) } @@ -251,14 +270,11 @@ func TestServiceAppliers_AnApplicationIsNeverOwedACatchUp(t *testing.T) { // TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal: the two reasons that are // not catching up reach an Application as they reach a Kustomization. func TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal(t *testing.T) { - argo := ApplierRef{ - Cluster: configv1alpha3.DefaultClusterProviderName, Kind: configv1alpha3.ApplierKindApplication, - Namespace: "argocd", Name: "argo", - } + argo := argoApplierRef("argo") t.Run("a foreign commit", func(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("argo", true, argoApplier)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() requests := requester.taken() require.Len(t, requests, 1) @@ -270,9 +286,9 @@ func TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal(t *test t.Run("a refusal", func(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("argo", true, argoApplier, func(g *configv1alpha3.GitTarget) { g.Spec.OnRefusal = configv1alpha3.RefusalActionReapply })) - loop.w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) - loop.oweReapply(refusalKeyFor(itypes.NewResourceReference("argo", "team-a"), "deployments"), - owedRefusal{detail: "refused", uid: "uid-argo"}, "obs") + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByFetch}) + loop.addUnsentRefusal(refusalKeyFor(itypes.NewResourceReference("argo", "team-a"), "deployments"), + unsentRefusal{detail: "refused", uid: "uid-argo"}, "obs") loop.serviceAppliers() requests := requester.taken() require.Len(t, requests, 1, "a refusal is asked at once, with no quiet period") @@ -283,7 +299,7 @@ func TestServiceAppliers_AnApplicationIsAskedOnAForeignCommitAndARefusal(t *test func TestServiceAppliers_AWriteRestartsTheQuietPeriod(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - loop.w.debt.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "aaa", By: ObservedByPush}) loop.serviceAppliers() loop.quietSince = time.Now().Add(-time.Hour) @@ -310,7 +326,7 @@ func TestServiceAppliers_AWriteDecidedAndPushedInOneWakeRestartsTheQuietPeriod(t loop.decide(PendingWrite{}) loop.pendingWrites, loop.pendingWritesBytes = nil, 0 // the same wake pushed it - loop.w.debt.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: "bbb", By: ObservedByPush}) loop.serviceAppliers() assert.Empty(t, requester.taken(), "the catch-up waits a quiet period from the push, not from before it") @@ -320,19 +336,21 @@ func TestServiceAppliers_AWriteDecidedAndPushedInOneWakeRestartsTheQuietPeriod(t // TestServiceAppliers_ASecondWriteInsideTheQuietPeriodRestartsIt is the other half of the race the // bi-directional e2e spec covers with real timers: two API edits a few seconds apart, each decided -// and pushed in one wake. The catch-up owed for the first push must not go out between them, and +// and pushed in one wake. The catch-up pending for the first push must not go out between them, and // when it goes out it names the second push. func TestServiceAppliers_ASecondWriteInsideTheQuietPeriodRestartsIt(t *testing.T) { - loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true, func(g *configv1alpha3.GitTarget) { - g.Spec.Applier.QuietPeriod = &metav1.Duration{Duration: time.Minute} - })) - loop.w.debt.observe(RemoteObservation{Commit: "base", By: ObservedByFetch}) - loop.w.debt.take() + loop, requester := applyTriggerLoop( + t, + applierTestTarget("alpha", true), + fluxTrigger("alpha", quietFor(time.Minute)), + ) + loop.w.inbox.observe(RemoteObservation{Commit: "base", By: ObservedByFetch}) + loop.w.inbox.take() pushInOneWake := func(commit string) { loop.decide(PendingWrite{}) loop.pendingWrites, loop.pendingWritesBytes = nil, 0 - loop.w.debt.observe(RemoteObservation{Commit: commit, By: ObservedByPush}) + loop.w.inbox.observe(RemoteObservation{Commit: commit, By: ObservedByPush}) loop.serviceAppliers() } @@ -354,42 +372,87 @@ func TestServiceAppliers_ASecondWriteInsideTheQuietPeriodRestartsIt(t *testing.T assert.Equal(t, "second", requests[0].Commit, "the catch-up names the later push, never the earlier") } -func TestServiceAppliers_AForeignCommitAnswersEverythingElseOwed(t *testing.T) { +func TestServiceAppliers_AForeignCommitAnswersEverythingElsePending(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("alpha", true)) - loop.oweReapply(refusalKeyFor(itypes.NewResourceReference("alpha", "team-a"), "deployments"), - owedRefusal{detail: "refused", uid: "uid-alpha"}, "obs") - owesForeign(loop.w) + loop.addUnsentRefusal(refusalKeyFor(itypes.NewResourceReference("alpha", "team-a"), "deployments"), + unsentRefusal{detail: "refused", uid: "uid-alpha"}, "obs") + observeForeignCommit(loop.w) loop.serviceAppliers() requests := requester.taken() require.Len(t, requests, 1, "one apply of the tip covers the refusal and the catch-up") assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, requests[0].Reason) - assert.Empty(t, loop.reapplyOwed) + assert.Empty(t, loop.unsentRefusals) assert.False(t, loop.appliers[applierRef("alpha")].catchUp) } -func TestServiceAppliers_NoApplierOnTheBranchClearsTheDebt(t *testing.T) { +func TestServiceAppliers_NoApplierOnTheBranchClearsThePendingReasons(t *testing.T) { loop, requester := applyTriggerLoop(t, applierTestTarget("plain", false)) - owesForeign(loop.w) - loop.w.debt.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) + observeForeignCommit(loop.w) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByPush}) loop.serviceAppliers() assert.Empty(t, requester.taken()) - assert.False(t, loop.w.debt.foreignOwed()) + assert.False(t, loop.w.inbox.foreignPending()) assert.Empty(t, loop.appliers) assert.Nil(t, loop.applierTimer) } +// TestServiceAppliers_ANewlyNamedApplierIsCaughtUp is review 7's W1: a merge to the branch is +// observed while no GitTarget names an applier, so the ForeignCommit is taken and dropped; then a +// GitTarget names a trigger, whose Kustomization was suspended before its GitRepository fetched the +// merge. The new applier had nothing pending, so the merge was never deployed until some later push. +// A newly named applier now starts where a first observation does, with a CatchUp pending (for +// Flux free when the commit is already applied); an Application, which gets no CatchUp, waits for the next push. +func TestServiceAppliers_ANewlyNamedApplierIsCaughtUp(t *testing.T) { + loop, requester := applyTriggerLoop(t, + applierTestTarget("flux", false), fluxTrigger("flux"), + applierTestTarget("argo", false)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + require.Empty(t, requester.taken()) + require.False(t, loop.w.inbox.foreignPending(), "taken, with no applier to send it to") + + for name, names := range map[string]func(*configv1alpha3.GitTarget){ + "flux": func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = testApplier("flux") }, + "argo": argoApplier, + } { + target := &configv1alpha3.GitTarget{} + require.NoError(t, loop.w.Client.Get(context.Background(), client.ObjectKey{Namespace: "team-a", Name: name}, + target)) + names(target) + require.NoError(t, loop.w.Client.Update(context.Background(), target)) + } + require.NoError(t, loop.w.Client.Create(context.Background(), &configv1alpha3.ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: "argo", Namespace: "team-a"}, + Spec: configv1alpha3.ArgoCDTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: configv1alpha3.DefaultClusterProviderName}, + ApplicationRef: meta.NamespacedObjectReference{Namespace: "argocd", Name: "argo"}, + }, + })) + loop.serviceAppliers() + loop.quietSince = time.Now().Add(-time.Hour) + loop.serviceAppliers() + + requests := requester.taken() + require.Len(t, requests, 1, "one apply, for the newly named Kustomization") + assert.Equal(t, applierRef("flux"), requests[0].Applier) + assert.Equal(t, configv1alpha3.ApplyReasonCatchUp, requests[0].Reason) + assert.Equal(t, "bbb", requests[0].Commit, "the merge it never fetched") + require.Contains(t, loop.appliers, argoApplierRef("argo"), "the Application is listed") + assert.False(t, loop.appliers[argoApplierRef("argo")].catchUp, "and waits for the next push") +} + func TestServiceAppliers_NoRequesterSwitchesTheTriggerOff(t *testing.T) { loop, _ := applyTriggerLoop(t, applierTestTarget("alpha", true)) loop.w.applyRequester = nil - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() - assert.True(t, loop.w.debt.foreignOwed(), "nothing took the debt") + assert.True(t, loop.w.inbox.foreignPending(), "nothing took the pending ForeignCommit") } // TestCloseWindowForForeignCommit_PublishesTheOpenWindowNow: a window closes on silence, so a user @@ -400,7 +463,7 @@ func TestCloseWindowForForeignCommit_PublishesTheOpenWindowNow(t *testing.T) { name string cause func(l *branchWorkerEventLoop) }{ - {name: "a foreign commit observed", cause: func(l *branchWorkerEventLoop) { owesForeign(l.w) }}, + {name: "a foreign commit observed", cause: func(l *branchWorkerEventLoop) { observeForeignCommit(l.w) }}, {name: "a Git host's push notice on a busy branch", cause: func(l *branchWorkerEventLoop) { l.handleRefreshRequest(&RefreshRequest{PushNotice: true}) }}, @@ -437,8 +500,9 @@ func applierLedgerFixture(t *testing.T, slug string) (*ledgerFixture, *branchWor target := &configv1alpha3.GitTarget{} require.NoError(t, f.worker.Client.Get(f.worker.ctx, client.ObjectKey{Name: ledgerTargetName, Namespace: "default"}, target)) - target.Spec.Applier = testApplier("apps") + target.Spec.ApplyTriggerRef = testApplier("apps") require.NoError(t, f.worker.Client.Update(f.worker.ctx, target)) + createTestTrigger(t, f.worker.Client, target) requester := &recordingRequester{} f.worker.applyRequester = requester @@ -474,7 +538,7 @@ func TestApplyTrigger_APushNoticeFindsAForeignCommit(t *testing.T) { require.Len(t, requests, 1) assert.Equal(t, gitOut(t, f.repoDir, "rev-parse", "main"), requests[0].Commit) assert.Equal(t, []itypes.ResourceReference{itypes.NewResourceReference(ledgerTargetName, "default")}, - requests[0].TargetRefs()) + targetRefs(requests[0])) // A duplicate delivery of the same push is one more advertisement and no second request. loop.handleRefreshRequest(&RefreshRequest{PushNotice: true}) @@ -482,10 +546,10 @@ func TestApplyTrigger_APushNoticeFindsAForeignCommit(t *testing.T) { assert.Empty(t, foreignRequests(requester.taken())) } -// TestApplyTrigger_ARejectedPushOwesTheReplayedTip is §3.4: the foreign commit is found by the +// TestApplyTrigger_ARejectedPushAsksForTheReplayedTip is §3.4: the foreign commit is found by the // compare-and-swap, the pending write is replayed onto it, and the request names the replayed tip, // which carries the API side's answer. -func TestApplyTrigger_ARejectedPushOwesTheReplayedTip(t *testing.T) { +func TestApplyTrigger_ARejectedPushAsksForTheReplayedTip(t *testing.T) { f, loop, requester := applierLedgerFixture(t, "apply-trigger-rejected") f.publish("prime") loop.endWake(0) @@ -503,32 +567,70 @@ func TestApplyTrigger_ARejectedPushOwesTheReplayedTip(t *testing.T) { assert.Equal(t, "from-another-writer", gitOut(t, f.repoDir, "show", head+":OUTSIDE.md")) } -// TestApplierOf_KeysTheApplierByItsOwnClusterAndKind: one name in two clusters is two appliers, and -// spec.applier.clusterProviderRef, not the source, decides which cluster. A Kustomization and an -// Application of one name are two appliers too. -func TestApplierOf_KeysTheApplierByItsOwnClusterAndKind(t *testing.T) { - target := &configv1alpha3.GitTarget{ - ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "apps"}, - Spec: configv1alpha3.GitTargetSpec{ - ClusterProviderRef: &meta.LocalObjectReference{Name: "prod-eu-1"}, - Applier: &configv1alpha3.GitTargetApplier{ - KustomizationRef: &meta.NamespacedObjectReference{Name: "apps"}, - }, - }, - } +// TestApplierOf_KeysTheApplierByItsTriggerClusterAndKind: one name in two clusters is two appliers, +// and the trigger's spec.clusterProviderRef, not the GitTarget's source, decides which cluster. A +// Kustomization and an Application of one name are two appliers too. +func TestApplierOf_KeysTheApplierByItsTriggerClusterAndKind(t *testing.T) { + flux := fluxTrigger("apps", func(t *configv1alpha3.FluxTrigger) { + t.Spec.ClusterProviderRef.Name = "prod-eu-1" + }) kustomization := configv1alpha3.ApplierKindKustomization - assert.Equal(t, ApplierRef{Cluster: "prod-eu-1", Kind: kustomization, Namespace: "team-a", Name: "apps"}, - ApplierOf(target)) - - target.Spec.Applier.ClusterProviderRef = &meta.LocalObjectReference{Name: "hub"} - assert.Equal(t, ApplierRef{Cluster: "hub", Kind: kustomization, Namespace: "team-a", Name: "apps"}, - ApplierOf(target)) - - target.Spec.Applier.KustomizationRef = nil - target.Spec.Applier.ApplicationRef = &meta.NamespacedObjectReference{Namespace: "argocd", Name: "apps"} - app := ApplierOf(target) - assert.Equal(t, ApplierRef{ - Cluster: "hub", Kind: configv1alpha3.ApplierKindApplication, Namespace: "argocd", Name: "apps", - }, app) + byFlux := TriggerRef{Kind: configv1alpha3.ApplyTriggerKindFlux, Namespace: "team-a", Name: "apps"} + assert.Equal(t, ApplierRef{Trigger: byFlux, Cluster: "prod-eu-1", Kind: kustomization, Namespace: "team-a", + Name: "apps"}, ApplierOf(flux)) + + flux.Spec.ClusterProviderRef.Name = "hub" + assert.NotEqual(t, ApplierRef{Trigger: byFlux, Cluster: "prod-eu-1", Kind: kustomization, + Namespace: "team-a", Name: "apps"}, ApplierOf(flux)) + + argo := testTrigger("team-a", &configv1alpha3.ApplyTriggerReference{ + Kind: configv1alpha3.ApplyTriggerKindArgoCD, Name: "apps", + }).(configv1alpha3.ApplyTrigger) + app := ApplierOf(argo) + assert.Equal(t, argoApplierRef("apps"), app) assert.False(t, app.CatchesUp(), "an Application is never asked to catch up") } + +// TestResolveApplyTrigger_NamesNothingItCannotFind: a GitTarget that names no trigger, a kind this +// release does not serve, or a trigger not created yet resolves to none, without an error; a read +// that fails is an error, so a blip never reads as "no applier". +func TestResolveApplyTrigger_NamesNothingItCannotFind(t *testing.T) { + loop, _ := applyTriggerLoop(t, fluxTrigger("apps")) + reader := loop.w.Client + resolve := func(ref *configv1alpha3.ApplyTriggerReference) (configv1alpha3.ApplyTrigger, error) { + target := applierTestTarget("alpha", false, func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = ref }) + return ResolveApplyTrigger(context.Background(), reader, target) + } + + for name, ref := range map[string]*configv1alpha3.ApplyTriggerReference{ + "no reference": nil, + "an unknown kind": {Kind: "WebhookTrigger", Name: "apps"}, + "a missing object": {Kind: configv1alpha3.ApplyTriggerKindArgoCD, Name: "apps"}, + } { + trigger, err := resolve(ref) + require.NoError(t, err, name) + assert.Nil(t, trigger, name) + } + + trigger, err := resolve(testApplier("apps")) + require.NoError(t, err) + require.NotNil(t, trigger) + assert.Equal(t, "team-a/apps", trigger.Applier().String()) + assert.Equal(t, "FluxTrigger/team-a/apps", ApplierOf(trigger).Trigger.String()) + + failing := interceptor.NewClient(reader.(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("the API server is away") + }, + }) + _, err = ResolveApplyTrigger( + context.Background(), + failing, + applierTestTarget( + "alpha", + false, + func(g *configv1alpha3.GitTarget) { g.Spec.ApplyTriggerRef = testApplier("apps") }, + ), + ) + require.ErrorContains(t, err, "read FluxTrigger team-a/apps") +} diff --git a/internal/git/apply_window.go b/internal/git/apply_window.go index f4e0afae9..27e1db883 100644 --- a/internal/git/apply_window.go +++ b/internal/git/apply_window.go @@ -4,6 +4,7 @@ package git import ( "context" + "maps" "slices" "time" @@ -23,13 +24,18 @@ import ( // The repository's rule for an object both sides changed is that the API value wins. Applied to // the window: a write by the applier's own field manager, during an attempt, to an object the API // side edited after that attempt was sent, is the apply arriving late with no intent behind it. It -// is withheld. Git keeps the edit, and the CatchUp that edit's push owes applies it back. +// is withheld. Git keeps the edit, and the cluster runs the reverted value until an apply puts it +// back, so the withheld write leaves a Repair pending for its applier (apply_trigger.go). // // What the rule must never do is withhold an API edit, which would lose it outright. So it acts -// only on proof: the object's latest managedFields entries are the applier's managers alone. When -// the second-resolution times tie with another manager the write could be either, and it is mirrored -// as before; the collision is logged and counted as ambiguous so it is still seen. A DELETE carries -// no managedFields of its deleter and is never withheld. +// only on proof: the object's latest managedFields entries are the applier's managers alone, and +// they are newer than the edit. Alone is not enough: an update that removes every field a manager +// owns deletes that manager's entry (structured-merge-diff, merge/update.go), so undoing an edit +// (`kubectl label cm/x tmp-` after `tmp=1`) can leave the applier's entry from an earlier apply as +// the newest. A late apply moves the applier's entry forward; an undo does not. When the +// second-resolution times tie, with another manager or with the edit, the write could be either, and +// it is mirrored as before; the collision is logged and counted as ambiguous so it is still seen. A +// DELETE carries no managedFields of its deleter and is never withheld. // FluxFieldManager is the field manager kustomize-controller applies with (operation Apply). It is // fixed in Flux, not configurable (allFluxManagers in external-sources/flux/flux-operator @@ -80,16 +86,23 @@ const ( collisionAmbiguous = "ambiguous" ) -// applyWindow is one applier's open apply window: from sending an attempt until the applier -// finished it, plus applyWindowGrace. +// applyWindow is one applier's open apply window: from sending an attempt until its operation +// completed, plus applyWindowGrace, and never before the window it replaced would have closed. An +// Unresolved operation holds it open. type applyWindow struct { attempt uint64 commit string sentAt time.Time // closesAt is zero while the attempt is out. closesAt time.Time - // edited holds the objects the API side wrote since the attempt was sent, by objectKey. - edited map[string]struct{} + // edited holds the objects the API side wrote since the attempt was sent, by objectKey, with the + // time of the latest such write's managedFields entries (zero when the event carried none). + edited map[string]time.Time + // carried holds the objects the window before this one had seen edited, when that window was + // still open as this attempt was sent, and carriedUntil is when it would have closed: the + // earlier apply's writes can arrive until then (openApplyWindow). + carried map[string]time.Time + carriedUntil time.Time } // open reports whether the window is open at now. @@ -103,22 +116,54 @@ func objectKey(id itypes.ResourceIdentifier) string { return id.Group + "/" + id.Resource + "/" + id.Namespace + "/" + id.Name } -// openApplyWindow starts an applier's window as its attempt is sent. +// editedSince reports whether the API side wrote an object since the attempt was sent, or, until +// the window before it would have closed, since that one was, and when it last did. +func (a *applyWindow) editedSince(key string, now time.Time) (time.Time, bool) { + at, edited := a.edited[key] + if carriedAt, carried := a.carried[key]; carried && now.Before(a.carriedUntil) { + return latest(at, carriedAt), true + } + return at, edited +} + +// recordEdit records an API-side write of an object at at, keeping the latest. +func (a *applyWindow) recordEdit(key string, at time.Time) { + a.edited[key] = latest(a.edited[key], at) +} + +// openApplyWindow starts an applier's window as its attempt is sent. The window before it can still +// be open: an attempt may follow one that completed within its grace, and that apply's writes are +// still arriving. Its edits are carried until it would have closed, so a late write of the earlier +// apply over one of them is still withheld. Replacing the window lost them, and the revert was +// mirrored. func (state *applierState) openApplyWindow(attempt uint64, commit string, now time.Time) { - state.window = &applyWindow{attempt: attempt, commit: commit, sentAt: now, edited: map[string]struct{}{}} + next := &applyWindow{attempt: attempt, commit: commit, sentAt: now, edited: map[string]time.Time{}} + if before := state.window; before.open(now) { + next.carried = maps.Clone(before.edited) + if now.Before(before.carriedUntil) { + maps.Copy(next.carried, before.carried) + } + next.carriedUntil = before.closesAt + } + state.window = next } -// settleApplyWindow ends an attempt's part in its applier's window: a delivered attempt's window -// stays open for applyWindowGrace, and any other attempt asked for no apply, so there is no window. -func (state *applierState) settleApplyWindow(attempt uint64, delivered bool, now time.Time) { - if state.window == nil || state.window.attempt != attempt { +// settleApplyWindow ends an attempt's part in its applier's window: once an operation completed, +// succeeded or not, its window stays open for applyWindowGrace; any other attempt asked for no +// apply, so its window stays open only while the one before it would have. +func (state *applierState) settleApplyWindow(attempt uint64, operated bool, now time.Time) { + window := state.window + if window == nil || window.attempt != attempt { return } - if !delivered { + switch { + case operated: + window.closesAt = latest(now.Add(applyWindowGrace), window.carriedUntil) + case now.Before(window.carriedUntil): + window.closesAt = window.carriedUntil + default: state.window = nil - return } - state.window.closesAt = now.Add(applyWindowGrace) } // covers reports whether the applier applies the GitTarget an event is for. @@ -145,20 +190,35 @@ func (l *branchWorkerEventLoop) withholdLateApply(event Event) bool { continue } byApplier, byApplierAlone := ref.writtenBy(event.LastFieldManagers) - _, editedSince := window.edited[key] + editedAt, editedSince := window.editedSince(key, now) + written := event.LastFieldManagedAt switch { case !byApplier || event.Operation == string(itypes.OperationDelete): - window.edited[key] = struct{}{} + window.recordEdit(key, written) + case !byApplierAlone: + // It may be the API edit itself, written in the same second: mirroring it is the + // choice that cannot lose an edit. It counts as one, too, even as the first write in the + // window, so the applier's write alone after it is still seen as late. It collides only + // with an edit made before it. + if editedSince { + l.recordApplyWindowCollision(applier, window, event, collisionAmbiguous) + } + window.recordEdit(key, written) case !editedSince: // The applier applying an object nobody edited since the attempt: its content is the // commit's, which Git already holds. Mirrored as ever. - case !byApplierAlone: + case written.Equal(editedAt): + // The applier's entry and the edit in the same second: it could be either. l.recordApplyWindowCollision(applier, window, event, collisionAmbiguous) - // It may be the API edit itself, written in the same second: mirroring it is the - // choice that cannot lose an edit. It counts as one, too. - window.edited[key] = struct{}{} + window.recordEdit(key, written) + case written.Before(editedAt): + // The applier's entry is older than the edit, so the applier did not write this: an API + // update that removed every field its manager owned took that entry with it, and left + // the applier's from an earlier apply as the newest. It is an edit, the undo of one. + window.recordEdit(key, written) default: l.recordApplyWindowCollision(applier, window, event, collisionWithheld) + applier.repair = true return true } } diff --git a/internal/git/apply_window_test.go b/internal/git/apply_window_test.go index 83e1e103e..d7bf33ca1 100644 --- a/internal/git/apply_window_test.go +++ b/internal/git/apply_window_test.go @@ -16,12 +16,24 @@ import ( sdkmetric "go.opentelemetry.io/otel/sdk/metric" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" "github.com/ConfigButler/gitops-reverser/internal/telemetry" itypes "github.com/ConfigButler/gitops-reverser/internal/types" ) -// windowEvent is a live UPDATE to the Deployment "api" in target's folder, written last by managers. +// windowClock is when the managedFields entries of the next windowEvent were written: a second +// after the previous one, as the API server stamps them. +var windowClock = time.Unix(1_700_000_000, 0) + +// windowEvent is a live UPDATE to the Deployment "api" in target's folder, written last by managers, +// a second after the previous one. func windowEvent(target, replicas string, managers ...string) Event { + windowClock = windowClock.Add(time.Second) + return windowEventAt(target, replicas, windowClock, managers...) +} + +// windowEventAt is windowEvent, its managers' latest entries written at. +func windowEventAt(target, replicas string, at time.Time, managers ...string) Event { obj := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{"name": "api", "namespace": "shop"}, @@ -32,6 +44,7 @@ func windowEvent(target, replicas string, managers ...string) Event { Identifier: itypes.NewResourceIdentifier("apps", "v1", "deployments", "shop", "api"), Operation: string(itypes.OperationUpdate), LastFieldManagers: managers, + LastFieldManagedAt: at, GitTargetName: target, GitTargetNamespace: "team-a", Path: "apps/" + target, @@ -42,7 +55,7 @@ func windowEvent(target, replicas string, managers ...string) Event { func sentAttempt(t *testing.T) (*branchWorkerEventLoop, ApplyRequest) { t.Helper() loop, requester := manualLoop(t, applierTestTarget("alpha", true), applierTestTarget("bravo", false)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() out := requester.taken() require.Len(t, out, 1) @@ -70,6 +83,30 @@ func TestApplyWindow_TheAppliersLateWriteOverANewerEditIsWithheld(t *testing.T) assert.Equal(t, int64(1), collisions(t, reader, collisionWithheld)) } +// TestApplyWindow_AnUndoOfAnEditIsMirrored is review 7's W2. Inside a window, `kubectl label cm/x +// tmp=1` is mirrored; `kubectl label cm/x tmp-` then removes every field kubectl-label owns, which +// deletes its managedFields entry, so the newest entry left is kustomize-controller's, from an +// earlier apply. That undo looked like the applier's write alone over a newer edit: it was withheld +// and a Repair sent, which put tmp=1 back. A real late apply moves the applier's entry forward; an +// undo does not. So only an applier entry strictly newer than the edit is withheld. +func TestApplyWindow_AnUndoOfAnEditIsMirrored(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, _ := sentAttempt(t) + earlierApply, edit := time.Unix(1_800_000_000, 0), time.Unix(1_800_000_100, 0) + + assert.False(t, loop.withholdLateApply(windowEventAt("alpha", "3", edit, "kubectl-label")), "the edit") + assert.False(t, loop.withholdLateApply(windowEventAt("alpha", "2", earlierApply, FluxFieldManager)), + "its undo leaves only an older entry of the applier's, and is mirrored") + assert.False(t, loop.appliers[applierRef("alpha")].repair, "so no Repair puts the edit back") + assert.False(t, loop.withholdLateApply(windowEventAt("alpha", "2", edit, FluxFieldManager)), + "a write of the applier's in the same second as the edit could be either, and is mirrored") + assert.Equal(t, int64(1), collisions(t, reader, collisionAmbiguous)) + assert.True(t, loop.withholdLateApply(windowEventAt("alpha", "2", edit.Add(time.Second), FluxFieldManager)), + "an apply arriving after the edit moves the applier's entry forward, and is withheld") + assert.Equal(t, int64(1), collisions(t, reader, collisionWithheld)) +} + // TestApplyWindow_OnlyAWithheldWriteIsKeptFromTheCommitWindow follows both events through the live // path: the commit window ends up holding the API edit, not Flux's revert of it. func TestApplyWindow_OnlyAWithheldWriteIsKeptFromTheCommitWindow(t *testing.T) { @@ -127,7 +164,7 @@ func TestApplyWindow_NoWindowBeforeAnAttempt(t *testing.T) { // closes. func TestApplyWindow_ClosesAGraceAfterFluxFinished(t *testing.T) { loop, req := sentAttempt(t) - report(req, ApplyDelivered) + report(req, ApplySucceeded) loop.serviceAppliers() assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit"))) @@ -142,7 +179,7 @@ func TestApplyWindow_ClosesAGraceAfterFluxFinished(t *testing.T) { // TestApplyWindow_AnAttemptThatAppliedNothingHasNoWindow: a stale, failed or dropped attempt // stamped nothing, so nothing the applier writes afterwards is the apply it asked for. func TestApplyWindow_AnAttemptThatAppliedNothingHasNoWindow(t *testing.T) { - for _, outcome := range []ApplyOutcome{ApplyStale, ApplyFailed, ApplyRejected, ApplyDropped, ApplyCurrent} { + for _, outcome := range []ApplyOutcome{ApplyStale, ApplyRequestFailed, ApplyRejected, ApplyDropped, ApplyCurrent} { t.Run(string(outcome), func(t *testing.T) { loop, req := sentAttempt(t) assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit"))) @@ -160,7 +197,7 @@ func TestApplyWindow_ArgoCDsLateWriteIsWithheldByItsOwnManagers(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) loop, requester := manualLoop(t, applierTestTarget("alpha", true, argoApplier)) - owesForeign(loop.w) + observeForeignCommit(loop.w) loop.serviceAppliers() require.Len(t, requester.taken(), 1) @@ -179,3 +216,124 @@ func TestApplyWindow_ArgoCDsLateWriteIsWithheldByItsOwnManagers(t *testing.T) { "a tie with another manager may be the edit itself") assert.Equal(t, int64(1), collisions(t, reader, collisionAmbiguous)) } + +// TestApplyWindow_AFirstTieIsRememberedAsAnEdit: a write by the applier and another manager in the +// same second may be the API edit, so it is mirrored. It must also be remembered as one: when the +// applier then writes the object alone a second later, that is its apply reverting the edit, and is +// withheld. A tie on an object nobody edited before is not a collision, so it is not counted. +func TestApplyWindow_AFirstTieIsRememberedAsAnEdit(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, _ := sentAttempt(t) + + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", FluxFieldManager, "kubectl-edit")), + "a tie may be the API edit itself") + assert.Equal(t, int64(0), collisions(t, reader, collisionAmbiguous), + "nothing was edited before it, so it collides with nothing") + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "the applier's write a second later is its apply over the tie's edit") + assert.Equal(t, int64(1), collisions(t, reader, collisionWithheld)) +} + +// completedThenNext is a loop whose applier, alpha's, completed attempt A over an API edit made +// while it ran, and was sent attempt B within A's grace: a push somebody else made asks it again. +func completedThenNext(t *testing.T) (*branchWorkerEventLoop, *recordingRequester, ApplyRequest) { + t.Helper() + loop, requester := manualLoop(t, applierTestTarget("alpha", true), applierTestTarget("bravo", false)) + observeForeignCommit(loop.w) + loop.serviceAppliers() + first := requester.taken() + require.Len(t, first, 1) + require.False(t, loop.withholdLateApply(windowEvent("alpha", "3", "kubectl-edit")), "the edit is mirrored") + + report(first[0], ApplySucceeded) + loop.w.inbox.observe(RemoteObservation{Commit: "ccc", By: ObservedByFetch}) + loop.serviceAppliers() + next := requester.taken() + require.Len(t, next, 1, "B goes out within A's grace") + require.NotEqual(t, first[0].ID, next[0].ID) + return loop, requester, next[0] +} + +// TestApplyWindow_ANewAttemptKeepsTheGraceOfTheOneBefore is the fifth review's third finding. A new +// attempt replaced its applier's window, and the edits the one before had seen went with it, while +// that earlier apply's writes could still arrive within its grace. Its late write over the edit was +// then mirrored, and the revert reached Git as the applier's commit. +func TestApplyWindow_ANewAttemptKeepsTheGraceOfTheOneBefore(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + loop, _, _ := completedThenNext(t) + + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "A's late write over the edit is withheld while B is out") + assert.Equal(t, int64(1), collisions(t, reader, collisionWithheld)) + assert.True(t, onlyApplier(t, loop).repair, "and has a Repair pending") +} + +// TestApplyWindow_AnAttemptThatAppliedNothingKeepsTheGraceBeforeIt: B ending without an apply has no +// window of its own, but A's grace still runs, so A's late write is withheld until it ends, and +// mirrored after. +func TestApplyWindow_AnAttemptThatAppliedNothingKeepsTheGraceBeforeIt(t *testing.T) { + for _, outcome := range []ApplyOutcome{ApplyCurrent, ApplyStale} { + t.Run(string(outcome), func(t *testing.T) { + loop, _, next := completedThenNext(t) + report(next, outcome) + loop.openWindow = &openWindow{} // a busy branch: a stale B is not sent again yet + loop.serviceAppliers() + state := onlyApplier(t, loop) + require.Nil(t, state.inflight) + + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), + "A's late write is withheld within A's grace") + assert.True(t, state.repair) + + state.window.closesAt = time.Now().Add(-time.Millisecond) + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager)), "after it") + }) + } +} + +// TestApplyWindow_TheCarriedGraceEndsWhenTheEarlierWindowWouldHave: the edits a new attempt carries +// from the window before it count only until that window would have closed. After that, the +// applier's write is B's apply of a commit that holds the edit, and is mirrored; an edit made since +// B was sent still counts. +func TestApplyWindow_TheCarriedGraceEndsWhenTheEarlierWindowWouldHave(t *testing.T) { + loop, _, _ := completedThenNext(t) + window := onlyApplier(t, loop).window + require.False(t, window.carriedUntil.IsZero()) + window.carriedUntil = time.Now().Add(-time.Millisecond) + + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "3", FluxFieldManager)), + "A's grace is over: the applier's write is B's apply") + assert.False(t, loop.withholdLateApply(windowEvent("alpha", "4", "kubectl-edit"))) + assert.True(t, loop.withholdLateApply(windowEvent("alpha", "3", FluxFieldManager)), + "an edit made while B runs is B's own") +} + +// TestApplyWindow_ConsecutiveOperationsOnOneApplier composes the steps: A completes, B is sent within +// A's grace, A's late write over the edit is withheld, B completes, and the Repair that write left +// pending goes out once, on the quiet branch, at the tip. Each window closes a grace after its own attempt. +func TestApplyWindow_ConsecutiveOperationsOnOneApplier(t *testing.T) { + loop, requester, second := completedThenNext(t) + assert.Equal(t, configv1alpha3.ApplyReasonForeignCommit, second.Reason) + + require.True(t, loop.withholdLateApply(windowEvent("alpha", "2", FluxFieldManager))) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "the Repair waits for B") + + report(second, ApplySucceeded) + loop.serviceAppliers() + state := onlyApplier(t, loop) + repair := requester.taken() + require.Len(t, repair, 1) + assert.Equal(t, configv1alpha3.ApplyReasonRepair, repair[0].Reason) + assert.Equal(t, "ccc", repair[0].Commit) + assert.WithinDuration(t, time.Now().Add(applyWindowGrace), state.window.carriedUntil, time.Second, + "the Repair carries B's grace, which ends a grace after B completed") + + report(repair[0], ApplySucceeded) + loop.serviceAppliers() + assert.Empty(t, requester.taken(), "the Repair is sent once") + assert.False(t, state.repairPending()) + assert.WithinDuration(t, time.Now().Add(applyWindowGrace), state.window.closesAt, time.Second) +} diff --git a/internal/git/branch_worker.go b/internal/git/branch_worker.go index 90e1da338..814f27e54 100644 --- a/internal/git/branch_worker.go +++ b/internal/git/branch_worker.go @@ -135,21 +135,31 @@ type BranchWorker struct { // simply not shared. remoteReporter RemoteReporter - // applyRequester delivers the apply requests this worker decides (apply_trigger.go), and debt is - // what the branch's appliers are owed from what was observed about the remote. Set by the + // applyRequester delivers the apply requests this worker decides (apply_trigger.go), and inbox + // holds the reasons pending for the branch's appliers from what was observed about the remote, + // and the reports of attempts that came back. Set by the // WorkerManager before Start; a nil requester switches the apply trigger off (the CLI, and tests // that do not exercise it). applyRequester ApplyRequester - debt applyDebt + // triggerStatus writes each applier's operation to its trigger's status; nil writes nothing. + triggerStatus ApplyTriggerStatusWriter + inbox applyInbox // applyWake wakes the loop to service the appliers without queueing a work item: an attempt // reported back, or another worker writing this repository and branch went quiet. applyWake chan struct{} // siblings are the workers writing the same repository and branch as this one, under another - // GitProvider; their open windows and pending writes make the branch busy too. busyWriting is - // this worker's own, published for them at the end of every wake. Nil siblings (the CLI, tests) - // means this worker is alone. See apply_siblings.go. + // GitProvider; their open windows, pending writes and work items make the branch busy too. + // busyWriting is this worker's own, published for them at the end of every wake; inflightItems + // below covers the wake in between. Nil siblings (the CLI, tests) means this worker is alone. + // See apply_siblings.go. siblings *branchSiblings busyWriting atomic.Bool + // groupsOut are the source groups this worker has an attempt out for, so a sibling sends none + // for the same GitRepository meanwhile; guarded by the siblings' lock (apply_groups.go). + groupsOut map[string]struct{} + // unresolvedOperations is how many of this branch's apply operations are Unresolved, as the + // loop last published it, for the apply_operations_unresolved gauge. + unresolvedOperations atomic.Int64 // Event processing eventQueue chan WorkItem @@ -201,7 +211,7 @@ type BranchWorker struct { // across every error return on the write path, and a lock there would put a second ordering // constraint on code whose only job is to record that something went wrong. // - // See the package's design note in docs/finished/push-notification-and-reconcile-trigger.md §1.5. + // See the package's design note in docs/design/push-notification-and-reconcile-trigger.md §1.5. baseTrustedState atomic.Bool // checkoutApplied is how many of the loop's retained writes the checkout holds: the checkout is @@ -221,14 +231,14 @@ type BranchWorker struct { // See docs/design/gittarget-branch-worker-pending-writes.md. checkoutApplied atomic.Int64 - // lastReapply records when this worker last owed a GitTarget's applier a Refusal request, keyed + // lastReapply records when this worker last decided a Refusal request for a GitTarget's applier, keyed // by "namespace/name", so refusalReapplyInterval can floor the rate. It is guarded by its own // mutex rather than repoMu: the decision is taken before any repository work starts, and // borrowing the repository lock for it would order the two for no reason. refusalMu sync.Mutex lastReapply map[string]time.Time // coveredRefusal records, per GitTarget AND WATCHED COLLECTION, the refusal observation the last - // reapply request was owed for: a digest of the objects that were refused together with the + // reapply request was decided for: a digest of the objects that were refused together with the // issues raised about them. A recheck that observes exactly the same thing is already covered // by that request, so it is not another trigger. It is dropped when a resync for that collection is // ACCEPTED — see refusalRecovered, which is what makes a live edit re-made after a revert @@ -605,7 +615,7 @@ func (w *BranchWorker) EnqueueWithdraw(req *AttachCommitRequest) { select { case w.eventQueue <- WorkItem{Withdraw: req}: default: - w.inflightItems.Add(-1) + w.releaseInflightItem() w.Log.V(1).Info("Queue full, CommitRequest withdraw dropped (the controller re-sends)", "request", req.Namespace+"/"+req.Name) } @@ -632,14 +642,14 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { w.pendingResyncsMu.Lock() if w.stoppingLocked() { w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalAttach) w.Log.V(1).Info("Worker is stopping, CommitRequest attach refused (the controller re-sends)") return } if !w.admitPayload(charge) { w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalAttach) w.Log.V(1).Info("CommitRequest attach refused while the branch has paused intake "+ "(the controller re-sends)", "request", req.Namespace+"/"+req.Name) @@ -651,7 +661,7 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { // gets here. The owner answers for it; registering it twice could commit it twice. w.intake.release(charge) w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.Log.Info("CommitRequest attach refused: another branch worker holds the request", "request", req.Namespace+"/"+req.Name) return @@ -674,7 +684,7 @@ func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) { w.uncountQueuedAttach(id) w.intake.release(charge) w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalAttach) w.Log.Error(nil, "Event queue full, CommitRequest attach refused (controller will re-send)") } @@ -765,7 +775,7 @@ func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool { "sourceCollection", sourceCollectionForLog(request.SourceCollection)) return true default: - w.inflightItems.Add(-1) + w.releaseInflightItem() delete(w.pendingResyncs, key) w.intake.release(charge) w.pendingResyncsMu.Unlock() @@ -896,7 +906,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { w.pendingResyncsMu.Lock() if w.stoppingLocked() { w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalWrite) w.Log.V(1).Info("Worker is stopping, request refused; the producer keeps it to redeliver", "events", len(request.Events), @@ -905,7 +915,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { } if !w.admitPayload(item.charge) { w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalWrite) w.Log.V(1).Info("Write refused while the branch has paused intake; "+ "the producer keeps it to redeliver", "events", len(request.Events), "gitTarget", request.GitTargetName) @@ -927,7 +937,7 @@ func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool { default: w.intake.release(item.charge) w.pendingResyncsMu.Unlock() - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalWrite) // Name the producing collection on a refusal. A saturated queue is diagnosed from what was // refused and by whom: the 595-in-16-seconds storm was one GitTarget, and the next @@ -1136,7 +1146,7 @@ type branchWorkerEventLoop struct { commitTimer *time.Timer // pushTimer fires the next push when the success cooldown ends. pushTimer *time.Timer - // retry is the one schedule of the next attempt at work a failed attempt left owed. See + // retry is the one schedule of the next attempt at work a failed attempt left pending. See // retry.go. retry retrySchedule @@ -1163,14 +1173,14 @@ type branchWorkerEventLoop struct { // overwrite another's pending work: A queues, B replaces it, B is then suspended, and A's // authorized request is gone with nothing to report it. refusalPending map[refusalKey]pendingRefusalReapply - // reapplyOwed holds the refusals whose target's applier is owed a Refusal request, until an - // attempt carries them on a quiet branch. It is keyed like refusalPending, by target AND - // collection, so a collection that recovers cancels its own obligation and no other + // unsentRefusals holds the refusals the worker decided to send their target's applier a Refusal + // request for, until an attempt carries them on a quiet branch. It is keyed like refusalPending, + // by target AND collection, so a collection that recovers cancels its own request and no other // (refusalRecovered). See apply_trigger.go. - reapplyOwed map[refusalKey]owedRefusal + unsentRefusals map[refusalKey]unsentRefusal - // appliers holds what each applier named on this branch is owed and how its attempts are going, - // and nextApplyID numbers the attempts. quietSince is when the branch last became quiet (zero + // appliers holds the reasons pending for each applier named on this branch and how its attempts + // are going, and nextApplyID numbers the attempts. quietSince is when the branch last became quiet (zero // while it is busy), and applierTimer fires when the earliest waiting attempt comes due. See // apply_trigger.go. appliers map[ApplierRef]*applierState @@ -1186,9 +1196,13 @@ type branchWorkerEventLoop struct { // its window's timers close it. attachTimer *time.Timer - // recovery is the obligation to write what a missing parent branch held back, and the probe + // recovery is the open task of writing what a missing parent branch held back, and the probe // that drives it. See parent_recovery.go. recovery parentRecovery + + // fidelityDropsLogged is, per GitTarget, the gate status under which its last drop was logged at + // Info, so a closed gate logs its first drop and counts the rest. See render_fidelity_drops.go. + fidelityDropsLogged map[string]renderFidelityDropMark } // commitWindowDefaults are a GitTarget's commit window timers when it declares none. @@ -1210,6 +1224,11 @@ func newBranchWorkerEventLoop(w *BranchWorker, defaultIdleTimeout time.Duration) func (l *branchWorkerEventLoop) run() { defer l.stopTimers() + // Before the first live event can be mirrored: an operation an earlier process left in flight + // is followed again, with its apply window open. + l.followRecordedOperations(time.Now()) + l.publishTriggerStatus() + l.publishGroupsOut() l.publishLoopState(0) for { commitC, pushC, attachC, refusalC := l.timerChannels() @@ -1220,8 +1239,9 @@ func (l *branchWorkerEventLoop) run() { return case item := <-l.w.eventQueue: l.handleQueueItem(item) - // Released after the wake has published what the item left behind: see queueDepth. - l.w.inflightItems.Add(-1) + // Released after the wake has published what the item left behind: see queueDepth, and + // decisionHolds, which reads busyWriting and this count the same way. + l.w.releaseInflightItem() continue case <-commitC: l.commitTimer = nil @@ -1265,7 +1285,7 @@ func (l *branchWorkerEventLoop) handleQueueItem(item WorkItem) { // nothing is parked. // - Close the open window early if the branch moved under it (closeWindowForForeignCommit). // - Drive the pending writes: commit what the wake decided, schedule its push (advance). -// - Send the apply requests owed, if the wake left the branch quiet (serviceAppliers). +// - Send the pending apply requests, if the wake left the branch quiet (serviceAppliers). // - Publish the result, moving the handled item's charge to what the loop holds. func (l *branchWorkerEventLoop) endWake(released int64) { l.serviceCommitRequests() @@ -1280,7 +1300,7 @@ func (l *branchWorkerEventLoop) endWake(released int64) { // authoritative state; the loop goroutine is the only writer of both. released is the charge of the // item the wake handled: see intakeGate.sync. // -// The flag is all the loop owes the depth gauge: queueDepth() reads it and inflightItems at scrape +// The flag is all the depth gauge needs from the loop: queueDepth() reads it and inflightItems at scrape // time, so nothing here publishes a value that could go stale while the loop is busy. The order is // the contract: the loop releases a handled item from the in-flight count only after this, because // publishing the flag second would let both read empty while a window is open, and a scrape landing @@ -1335,9 +1355,9 @@ func (l *branchWorkerEventLoop) dispatch(item WorkItem) int64 { case item.Request == nil: case item.Request.CommitMode == CommitModeAtomic: targetName, targetNamespace := atomicRefusalTarget(item.Request) - if !l.w.normalWritesAllowed(targetName, targetNamespace) { - l.w.Log.V(1).Info("Dropping atomic write while render fidelity is not established", - "gitTarget", targetNamespace+"/"+targetName) + if status, closed := l.w.renderFidelityClosed(targetName, targetNamespace); closed { + l.dropAtRenderFidelity(itypes.NewResourceReference(targetName, targetNamespace), status, + writeDroppedAtomic, "") break } l.handleAtomicRequest(item.Request) @@ -1351,9 +1371,9 @@ func (l *branchWorkerEventLoop) dispatch(item WorkItem) int64 { // finalizing it as identity, the byte cap or a zero window require. func (l *branchWorkerEventLoop) handleLiveEvents(request *WriteRequest) { for _, event := range request.Events { - if !l.w.normalWritesAllowed(event.GitTargetName, event.GitTargetNamespace) { - l.w.Log.V(1).Info("Dropping live event while render fidelity is not established", - "gitTarget", event.GitTargetNamespace+"/"+event.GitTargetName) + if status, closed := l.w.renderFidelityClosed(event.GitTargetName, event.GitTargetNamespace); closed { + l.dropAtRenderFidelity(itypes.NewResourceReference(event.GitTargetName, event.GitTargetNamespace), + status, writeDroppedEvent, event.Identifier.String()) continue } if l.withholdLateApply(event) { @@ -1520,7 +1540,7 @@ func (w *BranchWorker) drainQueue() { _, charge = w.takePendingResync(item.Resync) } w.intake.release(charge) - w.inflightItems.Add(-1) + w.releaseInflightItem() default: return } @@ -1916,7 +1936,7 @@ func (w *BranchWorker) stoppingLocked() bool { return w.stoppingState } // // The push session reads the remote's ref advertisement on a connection the cycle was making // anyway, and a cycle that commits nothing still reaches it, so a trusted base needs no fetch to -// plan against. That is the whole saving. See docs/finished/push-notification-and-reconcile-trigger.md §1.3 and §1.5. +// plan against. That is the whole saving. See docs/design/push-notification-and-reconcile-trigger.md §1.3 and §1.5. // // Only a checkout holding no retained writes may fetch here at all: a reset would destroy the // local commits those writes produced. With writes retained, the event loop's materialize has @@ -3102,7 +3122,7 @@ func (w *BranchWorker) publishObservation(observed RemoteObservation) { // Stamped with the repository it is about, so that whoever publishes it can tell whether it // still describes the repository the GitTarget points at. See RemoteObservation.Repo. w.lastObservation.Store(&observed) - w.debt.observe(observed) + w.inbox.observe(observed) // Delivered here, at the one point where an observation is made, rather than by each caller // against whichever GitTargets it happened to be holding. The fact is about the BRANCH, so it // goes to the branch, and every target on it reads the same answer with the time it was diff --git a/internal/git/branch_worker_base_trust_test.go b/internal/git/branch_worker_base_trust_test.go index 300f733f7..37800fd9f 100644 --- a/internal/git/branch_worker_base_trust_test.go +++ b/internal/git/branch_worker_base_trust_test.go @@ -2,7 +2,7 @@ package git -// The base-trust state machine from docs/finished/push-notification-and-reconcile-trigger.md §1.5. +// The base-trust state machine from docs/design/push-notification-and-reconcile-trigger.md §1.5. // // Nothing reads these flags to make a decision yet and the setter is hard-wired off, so none of // this can change behavior — which is exactly why the tests are written now. The hard part of the diff --git a/internal/git/branch_worker_fetch_metric_test.go b/internal/git/branch_worker_fetch_metric_test.go index 90519a4a2..25fa2505b 100644 --- a/internal/git/branch_worker_fetch_metric_test.go +++ b/internal/git/branch_worker_fetch_metric_test.go @@ -2,7 +2,7 @@ package git -// Red-first, per docs/finished/push-notification-and-reconcile-trigger.md §1.7: today's behavior is pinned as a +// Red-first, per docs/design/push-notification-and-reconcile-trigger.md §1.7: today's behavior is pinned as a // passing assertion, so the change that removes the head-of-cycle fetch is proved by this // assertion flipping rather than by argument. // diff --git a/internal/git/deleted_remote_branch_test.go b/internal/git/deleted_remote_branch_test.go index 53619da2d..de85cadb8 100644 --- a/internal/git/deleted_remote_branch_test.go +++ b/internal/git/deleted_remote_branch_test.go @@ -16,7 +16,7 @@ package git // before the push, so the push simply re-created the branch. These tests are what keep that // recovery once the prefetch is conditional. // -// See docs/finished/push-notification-and-reconcile-trigger.md §1.3 and §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md §1.3 and §1.5. import ( "context" diff --git a/internal/git/dirty_worktree_recovery_test.go b/internal/git/dirty_worktree_recovery_test.go index da53f9022..fda8bad5e 100644 --- a/internal/git/dirty_worktree_recovery_test.go +++ b/internal/git/dirty_worktree_recovery_test.go @@ -11,7 +11,7 @@ package git // refuses a checkout that is not the projection of the retained writes, so a path that skipped // materialize fails rather than committing leftovers. // -// See docs/finished/push-notification-and-reconcile-trigger.md §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md §1.5. import ( "errors" diff --git a/internal/git/git_path_refusal.go b/internal/git/git_path_refusal.go index 35a9a5183..58021895f 100644 --- a/internal/git/git_path_refusal.go +++ b/internal/git/git_path_refusal.go @@ -10,10 +10,18 @@ import ( ) func (w *BranchWorker) normalWritesAllowed(targetName, targetNamespace string) bool { + _, closed := w.renderFidelityClosed(targetName, targetNamespace) + return !closed +} + +// renderFidelityClosed reports whether the render-fidelity gate closes a target's normal writes, +// and the gate's status for the target when it does: the status says which scopes it waits for. +func (w *BranchWorker) renderFidelityClosed(targetName, targetNamespace string) (RenderFidelityStatus, bool) { if w.renderFidelityGate == nil || targetName == "" || targetNamespace == "" { - return true + return RenderFidelityStatus{}, false } - return w.renderFidelityGate.AllowsWrites(itypes.NewResourceReference(targetName, targetNamespace)) + status := w.renderFidelityGate.Status(itypes.NewResourceReference(targetName, targetNamespace)) + return status, status.State != RenderFidelityTrue } // PathRefusalReporter surfaces a refused write plan to the layer that owns GitTarget diff --git a/internal/git/git_push_advertisement_test.go b/internal/git/git_push_advertisement_test.go index 981829b68..1a25cf6d6 100644 --- a/internal/git/git_push_advertisement_test.go +++ b/internal/git/git_push_advertisement_test.go @@ -6,7 +6,7 @@ package git // the worker now listens, and that it still falls back to the network when the remote said // nothing. // -// See docs/finished/push-notification-and-reconcile-trigger.md §1.3. +// See docs/design/push-notification-and-reconcile-trigger.md §1.3. import ( "context" diff --git a/internal/git/git_request_ledger_test.go b/internal/git/git_request_ledger_test.go index dce43f93f..43a6a4f0e 100644 --- a/internal/git/git_request_ledger_test.go +++ b/internal/git/git_request_ledger_test.go @@ -10,7 +10,7 @@ package git // every row carries request and response bytes beside its count, and the golden file publishes a // coarse size class for them — see sizeClass for why the class rather than the number. // -// See docs/finished/push-notification-and-reconcile-trigger.md §1.6, which specifies this ledger. +// See docs/design/push-notification-and-reconcile-trigger.md §1.6, which specifies this ledger. import ( "fmt" diff --git a/internal/git/git_roundtrip_ledger_test.go b/internal/git/git_roundtrip_ledger_test.go index 94100e658..04df94eba 100644 --- a/internal/git/git_roundtrip_ledger_test.go +++ b/internal/git/git_roundtrip_ledger_test.go @@ -2,7 +2,7 @@ package git -// This is the measurement docs/finished/push-notification-and-reconcile-trigger.md §1.6 asks for, before any of +// This is the measurement docs/design/push-notification-and-reconcile-trigger.md §1.6 asks for, before any of // the changes that page argues for. Every connection count in §4 of that page is a prediction // derived from reading code; this file replaces reading with observing. // @@ -557,7 +557,7 @@ func renderLedger(rows []ledgerRow) string { var b strings.Builder b.WriteString("# Round trips to the Git host, per operation.\n") b.WriteString("# Generated by TestGitRoundTripLedger; specified by\n") - b.WriteString("# docs/finished/push-notification-and-reconcile-trigger.md §1.6.\n") + b.WriteString("# docs/design/push-notification-and-reconcile-trigger.md §1.6.\n") b.WriteString("#\n") b.WriteString("# Regenerate with: go test ./internal/git -run TestGitRoundTripLedger -update\n") b.WriteString("#\n") diff --git a/internal/git/inherited_delete.go b/internal/git/inherited_delete.go index a8ca71cc3..1b52bdcf7 100644 --- a/internal/git/inherited_delete.go +++ b/internal/git/inherited_delete.go @@ -185,7 +185,7 @@ func (wb *writeBatch) authorInheritedDelete( // cluster again: the patch file goes, and so does its patches: entry. It returns the base document's // path when this batch has brought the object back into the render, now or by an earlier event, so // the caller plans the upsert against the staged render; retired reports that it happened now, in -// which case the caller also owes the oracle an intent that the object renders again. +// which case the caller must also give the oracle an intent that the object renders again. // // It runs before the upsert, so the upsert edits the base document the render will hold rather // than one it hides. diff --git a/internal/git/intake.go b/internal/git/intake.go index bf48e7030..b975e73c9 100644 --- a/internal/git/intake.go +++ b/internal/git/intake.go @@ -24,8 +24,8 @@ import ( // many items are kept as well as their bytes. The charge is an estimate of the serialized YAML, not // of the heap: an object's in-memory form is several times larger. // -// Pausing is a latch, released only when the retry clears: a publication landed, or nothing is owed -// any more. Room under the budget again, a partial replay, or a remote that can be read but still +// Pausing is a latch, released only when the retry clears: a publication landed, or no pending +// writes remain. Room under the budget again, a partial replay, or a remote that can be read but still // refuses the push prove nothing about whether the backlog can drain. A refused producer keeps its // work: the watch keeps its cursor and delivers again, the controller sends a save again, a refused // resync's collection is gathered again. Producers wait on IntakePaused for the reopening instead diff --git a/internal/git/parent_recovery.go b/internal/git/parent_recovery.go index 7258a0d96..c96d2f033 100644 --- a/internal/git/parent_recovery.go +++ b/internal/git/parent_recovery.go @@ -10,20 +10,20 @@ import ( "github.com/go-git/go-git/v6/plumbing" ) -// This file is the worker's recovery from a parent branch that is missing or does not resolve: the -// obligation to write what was decided while it was gone, once it exists. +// This file is the worker's recovery from a parent branch that is missing or does not resolve: once +// the parent exists again, it still has to write what was decided while it was gone. // -// An observation flip is not an obligation. "The parent is Found again" says nothing about whether -// the writes decided meanwhile have landed: the first attempt after it can fail transiently, the +// An observation flip does not end the recovery. "The parent is Found again" says nothing about +// whether the writes decided meanwhile have landed: the first attempt after it can fail transiently, the // refresher skips a branch that holds work, and with --git-refresh-interval=0 there is no refresher -// at all. So the worker latches an obligation the moment an attempt fails on the parent, and clears -// it only when nothing is owed any more: the pending writes were published, or every one of them -// settled without needing a push (endOutage). +// at all. So the worker latches parent recovery the moment an attempt fails on the parent, and clears +// it only when no pending writes remain: they were published, or every one of them settled without +// needing a push (endOutage). // -// The obligation is the pending writes themselves. A write decided while the parent is missing stays -// pending like any other a remote failure holds back (pending_writes_loop.go), and admission -// backpressure bounds them however long the parent stays away, so nothing is dropped and no snapshot -// is owed. +// What the recovery waits on is the pending writes themselves. A write decided while the parent is +// missing stays pending like any other a remote failure holds back (pending_writes_loop.go), and +// admission backpressure bounds them however long the parent stays away, so nothing is dropped and +// no snapshot has to be requested again. // // The worker probes for the parent itself, on the event loop, with one advertisement per deadline of // its one retry schedule (retry.go), shared by every target on the worker. Until the parent is @@ -42,7 +42,7 @@ func isParentUnavailable(err error) bool { return errors.Is(err, ErrParentBranchNotFound) || errors.Is(err, ErrDefaultBranchUnresolved) } -// parentRecovery is the loop's latch on the obligation. Loop-goroutine only. +// parentRecovery is the loop's latch on parent recovery. Loop-goroutine only. type parentRecovery struct { // active is the latch. active bool @@ -60,9 +60,9 @@ func (w *BranchWorker) now() time.Time { return time.Now() } -// noteParentUnavailable latches the obligation when err was caused by the parent and pending writes -// are owed. Any other error is not this obligation's concern, and neither is a failure that left -// nothing owed: an obligation with nothing to publish would never close. It does not schedule the +// noteParentUnavailable latches parent recovery when err was caused by the parent and pending writes +// remain. Any other error is not this recovery's concern, and neither is a failure that left no +// pending writes: a recovery with nothing to publish would never close. It does not schedule the // next attempt: whoever observed the failure does, once (scheduleRetry), and a new latch starts that // schedule over so the first probe is one initial backoff away. func (l *branchWorkerEventLoop) noteParentUnavailable(err error) { @@ -88,7 +88,7 @@ func (l *branchWorkerEventLoop) probeDue() bool { return r.active && (!l.w.now().Before(l.retry.due) || r.gen != l.w.parentSnapshot().gen) } -// runParentProbe is the attempt while the obligation is open: while the parent is missing it spends +// runParentProbe is the attempt while parent recovery is open: while the parent is missing it spends // one advertisement, and once it is found it publishes the pending writes. A publication that fails schedules // the next attempt itself. func (l *branchWorkerEventLoop) runParentProbe() { @@ -172,8 +172,8 @@ func (w *BranchWorker) awaitingParentProbe() bool { return hold != nil && hold.gen == w.parentSnapshot().gen && w.now().Before(hold.until) } -// ParentRecovery reports whether the worker still owes work decided while the parent branch was -// missing (the first result), and whether the parent has been found since (the second). +// ParentRecovery reports whether work decided while the parent branch was missing is still pending +// on the worker (the first result), and whether the parent has been found since (the second). func (w *BranchWorker) ParentRecovery() (bool, bool) { return w.parentRecoveryOpen.Load(), w.parentRecoveryFound.Load() } diff --git a/internal/git/parent_recovery_test.go b/internal/git/parent_recovery_test.go index 13acef572..57a87d2f0 100644 --- a/internal/git/parent_recovery_test.go +++ b/internal/git/parent_recovery_test.go @@ -2,10 +2,10 @@ package git -// Work decided while the parent branch is missing is an obligation, not an observation: it stays -// pending until it is published, so a transient failure after the parent reappears cannot strand it, -// and it is driven by the worker on its own deadline, whatever the refresh configuration. These pin -// the obligation, that nothing decided meanwhile is dropped, and the probe budget. +// Work decided while the parent branch is missing stays pending until it is published, whatever the +// parent's observed state: a transient failure after the parent reappears cannot strand it, and the +// worker drives it on its own deadline, whatever the refresh configuration. These pin how long +// parent recovery stays open, that nothing decided meanwhile is dropped, and the probe budget. import ( "context" @@ -114,7 +114,7 @@ func TestParentRecovery_RetainedWritesLandAfterAFailedFirstAttempt(t *testing.T) assert.True(t, f.ref("feature").IsZero(), "the first attempt failed") require.Len(t, f.loop.pendingWrites, 1, "and the work is still held") open, found = f.worker.ParentRecovery() - require.True(t, open, "the obligation outlives a failed attempt") + require.True(t, open, "parent recovery outlives a failed attempt") assert.True(t, found) clock.advance(retryMaxBackoff) @@ -125,7 +125,7 @@ func TestParentRecovery_RetainedWritesLandAfterAFailedFirstAttempt(t *testing.T) assert.Equal(t, []plumbing.Hash{release}, f.parentOf(tip)) assert.Empty(t, f.loop.pendingWrites) open, _ = f.worker.ParentRecovery() - assert.False(t, open, "published: nothing is owed") + assert.False(t, open, "published: no pending writes remain") }) } } @@ -137,7 +137,7 @@ func refreshTarget() itypes.ResourceReference { // Test 2. Writes for two targets on one worker, decided while the parent is missing, stay pending // with their authors and collections, and so does a resync decided meanwhile, whose caller is told // it could not be applied yet. The probe that finds the parent publishes all of it, and nothing is -// owed after that: no snapshot has to re-derive anything. +// left pending after that: no snapshot has to re-derive anything. func TestParentRecovery_WritesDecidedWhileTheParentIsMissingAreKept(t *testing.T) { f := newRealServerNewBranch(t, "recovery-kept-writes", seedMain(t)) clock := useTestClock(f.worker) @@ -176,7 +176,7 @@ func TestParentRecovery_WritesDecidedWhileTheParentIsMissingAreKept(t *testing.T assert.Empty(t, loop.pendingWrites, "all of it was published") open, _ = f.worker.ParentRecovery() - assert.False(t, open, "and nothing is owed") + assert.False(t, open, "and nothing is left to publish") names := gitIn(t, f.repoDir, "ls-tree", "-r", "--name-only", "refs/heads/feature") for _, name := range []string{"a1", "b1", "a2"} { assert.Contains(t, names, name) @@ -249,7 +249,7 @@ func TestParentRecovery_ProbeBudgetIsPerWorker(t *testing.T) { } // A second outage while the first outage's writes still wait to be published. The write decided -// during it is kept with them, and the obligation stays open until all of it is published. +// during it is kept with them, and parent recovery stays open until all of it is published. func TestParentRecovery_ASecondOutageKeepsItsWritesToo(t *testing.T) { f := newRealServerNewBranch(t, "recovery-second-outage", func(dir string) { simulateClientCommitOnDisk(t, dir, "main", "README.md", "main\n") @@ -411,10 +411,10 @@ func TestParentRecovery_AnEmptySaveWaitsForTheParent(t *testing.T) { assert.False(t, open) } -// An obligation with nothing to publish would never close, so a parent failure that left nothing +// A recovery with nothing to publish would never close, so a parent failure that left nothing // pending opens none. Found in review of #413, when a failed empty save left the target -// RecoveringParentBranch with nothing owed. -func TestParentRecovery_NothingOwedOpensNoObligation(t *testing.T) { +// RecoveringParentBranch with no pending writes. +func TestParentRecovery_NothingPendingOpensNoRecovery(t *testing.T) { w := &BranchWorker{} loop := newBranchWorkerEventLoop(w, time.Second) defer loop.stopTimers() @@ -425,7 +425,7 @@ func TestParentRecovery_NothingOwedOpensNoObligation(t *testing.T) { assert.False(t, open) } -// Recovery ends when nothing is owed, not only when a push lands. A resync held while the parent +// Recovery ends when no pending writes remain, not only when a push lands. A resync held while the parent // was missing can find nothing to change once it is back, and leave nothing pending without a push: // recovery and its retry then stayed open, and intake stayed paused for good. func TestParentRecovery_EndsWhenTheHeldWorkNeedsNoCommit(t *testing.T) { @@ -447,7 +447,7 @@ func TestParentRecovery_EndsWhenTheHeldWorkNeedsNoCommit(t *testing.T) { fireRetry(f.loop) require.Empty(t, f.loop.pendingWrites, "the resync had nothing to change") - assert.False(t, f.loop.recovery.active, "nothing is owed, so there is nothing to recover") + assert.False(t, f.loop.recovery.active, "nothing is left to publish, so there is nothing to recover") assert.False(t, f.loop.retry.pending(), "or to retry") assert.Nil(t, f.worker.IntakePaused(), "and intake reopens") } diff --git a/internal/git/pending_writes_loop.go b/internal/git/pending_writes_loop.go index f4d6f4786..61426015f 100644 --- a/internal/git/pending_writes_loop.go +++ b/internal/git/pending_writes_loop.go @@ -21,7 +21,7 @@ import ( // - decide appends it to the pending writes. The write gates and building the write have passed, so the // decision stands from here on. Deciding does nothing else: no commit, no push, no settlement. // - advance is the one driver, run once at the end of every wake of the loop. It commits what was -// decided, schedules the push, and closes the outage once nothing is owed. +// decided, schedules the push, and closes the outage once no pending writes remain. // - materialize makes the checkout the projection of the pending writes: the remote tip the writes were // planned on plus one commit for each, in order. It rebuilds the writes committed before when // the checkout is behind them, then commits each decided write not committed yet. @@ -30,8 +30,9 @@ import ( // because the remote could not be reached. A write left pending needs no settling, only its // waiting resync callers an answer (replyToDeferredResyncs). // -// Settling an outcome never decides more work directly: a refused write owes its applier a reapply -// request (reapplyForRefusal), which is sent once the branch is quiet, outside the pending writes. +// Settling an outcome never decides more work directly: a refused write leaves a reapply request +// due to its applier (reapplyForRefusal), which is sent once the branch is quiet, outside the +// pending writes. // errWriteFailed marks a failure of the write itself, as opposed to reaching the remote or reading // the GitProvider: the plan was refused or could not be applied. It is terminal for the write, while @@ -104,7 +105,7 @@ func (l *branchWorkerEventLoop) decide(pendingWrite PendingWrite) { } // advance drives the pending writes once the wake's handlers have decided what they decide: it -// commits them, schedules their push, and ends the outage once nothing is owed. It is the only +// commits them, schedules their push, and ends the outage once none remain. It is the only // caller of materialize outside a publication, and the only place where having no pending writes // closes the retry and parent recovery, so no exit that settles the last one can leave intake // paused. @@ -335,6 +336,10 @@ func (l *branchWorkerEventLoop) settleResyncApplied(i int) { // because a ConfigMap resync succeeding is no evidence about a Deployment that is still refused. l.refusalRecovered( itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), req.refusalCollection()) + // Recorded here, on the loop and before the caller hears anything, rather than only by the + // caller once it has the answer: the next item off the queue may be a live event, and it must + // meet the gate this resync produced. See recordReplayClean. + l.recordReplayClean(req) l.w.Log.Info("Resync request applied", "committed", committed, "created", stats.Created, diff --git a/internal/git/placement_metrics_test.go b/internal/git/placement_metrics_test.go index 1faa438d9..267744563 100644 --- a/internal/git/placement_metrics_test.go +++ b/internal/git/placement_metrics_test.go @@ -86,7 +86,7 @@ func flushWithPolicy( require.NoError(t, err) } -// The signal the Option C deletion owes its users: a repository whose layout this operator +// The signal the Option C deletion must give its users: a repository whose layout this operator // cannot derive gets the built-in path, and `source="builtin"` on placements_total names // the GitTarget and the type that needs one `placement.byType` line. Without the labels the // counter would only say a fall-back happened somewhere, which is not a fix anybody can act on. diff --git a/internal/git/placement_test.go b/internal/git/placement_test.go index b2105c620..e097397de 100644 --- a/internal/git/placement_test.go +++ b/internal/git/placement_test.go @@ -121,7 +121,7 @@ func TestPlacement_DeclaredBundle_AppendsToExistingMultiDocFile(t *testing.T) { // A declared destination inside a kustomize context must not write metadata.namespace: // the kustomization's namespace: transformer supplies it, and repeating it would break the // "no namespace: in this file" convention every other document in the bundle follows. The -// obligation used to come from reading the siblings' bytes; it now comes from the +// rule to omit it used to come from reading the siblings' bytes; it now comes from the // governing kustomization, which is the thing that actually decides the rendered namespace. func TestPlacement_DeclaredBundle_OmitsNamespaceInKustomizeContext(t *testing.T) { worktree := newWorktreeForTest(t) diff --git a/internal/git/plan_flush.go b/internal/git/plan_flush.go index 95c3a5306..21fe6ac41 100644 --- a/internal/git/plan_flush.go +++ b/internal/git/plan_flush.go @@ -524,7 +524,7 @@ func (wb *writeBatch) rewriteSensitive(ctx context.Context, event Event, filePat } // wroteBytes reports whether an upsert actually changed the worktree, which is the only -// case that owes the oracle an intent. +// case that must give the oracle an intent. func wroteBytes(o upsertOutcome) bool { return o == upsertCreated || o == upsertUpdated } diff --git a/internal/git/publication.go b/internal/git/publication.go index 16b0182da..ea661de6a 100644 --- a/internal/git/publication.go +++ b/internal/git/publication.go @@ -151,6 +151,9 @@ func (m *WorkerManager) setPublicationGaugeSources() { telemetry.SetGaugeSource(telemetry.GaugeGitNextRetry, source(func(w *BranchWorker) (int64, bool) { return dated(w.backlog.nextRetry.Load()) })) + telemetry.SetGaugeSource(telemetry.GaugeApplyOperationsUnresolved, source(func(w *BranchWorker) (int64, bool) { + return always(w.unresolvedOperations.Load()) + })) } // publicationGauges are the gauges setPublicationGaugeSources installs. @@ -162,6 +165,7 @@ var publicationGauges = []string{ telemetry.GaugeGitIntakePaused, telemetry.GaugeGitOldestRetainedWrite, telemetry.GaugeGitNextRetry, + telemetry.GaugeApplyOperationsUnresolved, } // publicationEventBuffer bounds the GitTarget events a burst of report changes can queue. A full diff --git a/internal/git/publication_test.go b/internal/git/publication_test.go index d3b6c0767..b41daefae 100644 --- a/internal/git/publication_test.go +++ b/internal/git/publication_test.go @@ -108,6 +108,11 @@ func TestPublication_MetricsDescribeTheBacklog(t *testing.T) { require.True(t, ok) assert.Equal(t, loop.retry.due.Unix(), nextRetry) + f.worker.unresolvedOperations.Store(1) + unresolved, ok := gauge(telemetry.GaugeApplyOperationsUnresolved) + require.True(t, ok) + assert.Equal(t, int64(1), unresolved, "an operation past its budget is still followed, and seen") + f.worker.branchBufferMaxBytes = 1 loop.publishLoopState(0) paused, _ = gauge(telemetry.GaugeGitIntakePaused) @@ -121,7 +126,7 @@ func TestPublication_MetricsDescribeTheBacklog(t *testing.T) { _, ok = gauge(telemetry.GaugeGitOldestRetainedWrite) assert.False(t, ok, "nothing kept, nothing to date") _, ok = gauge(telemetry.GaugeGitNextRetry) - assert.False(t, ok, "nothing owed, no retry due") + assert.False(t, ok, "nothing pending, no retry due") } // A change of the report reaches every GitTarget on the branch, and none on another branch: they diff --git a/internal/git/push_notification.go b/internal/git/push_notification.go index edf2e269a..bb37cface 100644 --- a/internal/git/push_notification.go +++ b/internal/git/push_notification.go @@ -7,7 +7,7 @@ package git // The report is a hint and never a fact: after never decides whether the branch moved. It is queued // as a refresh // that reaches the remote whatever was proved recently, so the worker reads the advertisement itself -// and the observation it records decides whether the branch moved (applyDebt.observe). That one +// and the observation it records decides whether the branch moved (applyInbox.observe). That one // rule settles every awkward delivery without bookkeeping: a late delivery for an earlier push of // this worker's finds the branch at the worker's latest push, a rollback onto an older one of its // commits finds a tip it did not push last, and a duplicate costs one more advertisement. diff --git a/internal/git/refresh.go b/internal/git/refresh.go index 99efaa8d9..7da9fc52a 100644 --- a/internal/git/refresh.go +++ b/internal/git/refresh.go @@ -61,14 +61,14 @@ func (w *BranchWorker) EnqueueRefresh(req *RefreshRequest) { w.pendingResyncsMu.Lock() defer w.pendingResyncsMu.Unlock() if w.stoppingLocked() { - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalRefresh) return } select { case w.eventQueue <- WorkItem{Refresh: req}: default: - w.inflightItems.Add(-1) + w.releaseInflightItem() w.recordQueueRefusal(queueRefusalRefresh) w.Log.V(1).Info("Event queue full, refresh dropped; the next reconcile asks again", "branch", w.Branch, "gitTarget", req.Target.String()) diff --git a/internal/git/refusal_observation_test.go b/internal/git/refusal_observation_test.go index a62692019..eb766fcc7 100644 --- a/internal/git/refusal_observation_test.go +++ b/internal/git/refusal_observation_test.go @@ -45,8 +45,9 @@ func standingRefusalFixture(t *testing.T, slug string) *ledgerFixture { require.NoError(t, f.worker.Client.Get(f.worker.ctx, client.ObjectKey{Name: ledgerTargetName, Namespace: "default"}, target)) target.Spec.OnRefusal = configv1alpha3.RefusalActionReapply - target.Spec.Applier = testApplier(ledgerTargetName) + target.Spec.ApplyTriggerRef = testApplier(ledgerTargetName) require.NoError(t, f.worker.Client.Update(f.worker.ctx, target)) + createTestTrigger(t, f.worker.Client, target) f.worker.applyRequester = &recordingRequester{} f.worker.mapper = deploymentMapper() @@ -143,7 +144,7 @@ func TestRefusalReapply_ASecondRefusedEditEarnsItsOwnRequest(t *testing.T) { "a genuinely different refused edit must still be requested") } -// TestRefusalReapply_AResyncsRefusalIsRequestedByTheLoopItself: a refusal a resync discovers is owed +// TestRefusalReapply_AResyncsRefusalIsRequestedByTheLoopItself: a refusal a resync discovers is decided // while the resync's outcome is being settled, inside the commit pass, and the wake that ends the // pass has to send it. The fixture's refuseResync pushes by hand, which is how a pushed-by-hand unit // test once missed a trigger that never left the worker. @@ -211,9 +212,9 @@ func TestRefusalObservation_IsContentAddressedAndOrderIndependent(t *testing.T) "nothing to digest is not an observation, and must never count as covered") } -// TestRefusalObservation_NothingIsCoveredUntilARequestIsOwed. The memory is written where the -// request is owed, and nowhere else. -func TestRefusalObservation_NothingIsCoveredUntilARequestIsOwed(t *testing.T) { +// TestRefusalObservation_NothingIsCoveredUntilARequestIsDecided. The memory is written where the +// request is decided, and nowhere else. +func TestRefusalObservation_NothingIsCoveredUntilARequestIsDecided(t *testing.T) { w := refusalTouchWorker(t, configv1alpha3.GitTargetSpec{ OnRefusal: configv1alpha3.RefusalActionReapply, }) @@ -279,7 +280,7 @@ func TestRefusalObservation_RecoveryIsScopedToTheCollectionThatRecovered(t *test // // Dropping the digest is not enough: a request already queued for that collection finds nothing // covering it when its timer fires and asks for a reapply anyway, for a refusal that has since been -// accepted. The obligation has to go with the memory — and only that collection's obligation. +// accepted. The queued request has to go with the memory, and only that collection's request. func TestRefusalObservation_RecoveryCancelsTheQueuedRequestForThatCollection(t *testing.T) { w := refusalTouchWorker(t, configv1alpha3.GitTargetSpec{ OnRefusal: configv1alpha3.RefusalActionReapply, @@ -298,7 +299,7 @@ func TestRefusalObservation_RecoveryCancelsTheQueuedRequestForThatCollection(t * assert.NotContains(t, loop.refusalPending, deployments, "a queued request must not survive the acceptance of the object it was queued for") assert.Contains(t, loop.refusalPending, configMaps, - "and the collection that is still refused keeps its obligation") + "and the collection that is still refused keeps its request") // Nothing was requested for the cancelled entry, so it must not have spent the window either. limited, _ := w.refusalRateLimited(editingRef()) @@ -358,11 +359,11 @@ func TestRefusalReapply_ARemovalAnywhereInTheFlushDisqualifiesIt(t *testing.T) { } // TestRefusalReapply_AQueuedRequestIsDroppedOnceSomethingElseCoveredIt closes the gap between arming -// a trailing request and owing it. +// a trailing request and deciding it. // -// A minute passes between the two, and another request for the same target can be owed in it. If +// A minute passes between the two, and another request for the same target can be decided in it. If // that request covered the same observation, this entry is asking for a second one that says -// exactly the same thing to the same applier, so it is dropped rather than owed. Consent is +// exactly the same thing to the same applier, so it is dropped rather than decided. Consent is // re-checked at the same point and for the same reason: a delayed action must act on what is true // when it fires. func TestRefusalReapply_AQueuedRequestIsDroppedOnceSomethingElseCoveredIt(t *testing.T) { @@ -521,8 +522,9 @@ func overlayRefusalFixture(t *testing.T, slug string) *ledgerFixture { require.NoError(t, f.worker.Client.Get(f.worker.ctx, client.ObjectKey{Name: ledgerTargetName, Namespace: "default"}, target)) target.Spec.OnRefusal = configv1alpha3.RefusalActionReapply - target.Spec.Applier = testApplier(ledgerTargetName) + target.Spec.ApplyTriggerRef = testApplier(ledgerTargetName) require.NoError(t, f.worker.Client.Update(f.worker.ctx, target)) + createTestTrigger(t, f.worker.Client, target) f.worker.applyRequester = &recordingRequester{} f.worker.mapper = deploymentMapper() return f diff --git a/internal/git/refusal_reapply.go b/internal/git/refusal_reapply.go index 9ef9a8717..3f7b235c9 100644 --- a/internal/git/refusal_reapply.go +++ b/internal/git/refusal_reapply.go @@ -88,19 +88,20 @@ func (w *BranchWorker) refusalConsent(ctx context.Context, target itypes.Resourc return gitTarget.UID, ConsentsToReapply(gitTarget) } -// owedRefusal is one refusal its target's applier is owed a Refusal request for: what was refused, -// for the logs, and the incarnation of the GitTarget that consented. A GitTarget deleted and -// recreated under the same name is another incarnation, and inherits nothing owed to the first. -type owedRefusal struct { +// unsentRefusal is one refusal the worker decided to send its target's applier a Refusal request +// for, not sent yet: what was refused, for the logs, and the incarnation of the GitTarget that +// consented. A GitTarget deleted and recreated under the same name is another incarnation, and +// inherits no unsent refusal of the first. +type unsentRefusal struct { detail string uid k8stypes.UID } // ConsentsToReapply reports whether a GitTarget, as it is now, asks for its applier to reapply on a -// refusal. The worker asks before it owes a Refusal request and again before it sends one, and the +// refusal. The worker asks before it decides a Refusal request and again before it sends one, and the // applier dispatcher asks before every attempt to deliver one. func ConsentsToReapply(gitTarget *v1alpha3.GitTarget) bool { - if gitTarget.EffectiveOnRefusal() != v1alpha3.RefusalActionReapply || gitTarget.Spec.Applier == nil { + if gitTarget.EffectiveOnRefusal() != v1alpha3.RefusalActionReapply || gitTarget.Spec.ApplyTriggerRef == nil { return false } // A suspended target has stopped publishing the live edits a reapply would be weighed against. @@ -131,8 +132,8 @@ func (w *BranchWorker) refusalRateLimited(target itypes.ResourceReference) (bool return false, 0 } -// reapplyForRefusal is the loop-side half: it turns a reported refusal into a Refusal apply request -// owed to the target's applier, sent once the branch is quiet (serviceAppliers). +// reapplyForRefusal is the loop-side half: it turns a reported refusal into an unsent Refusal apply +// request for the target's applier, sent once the branch is quiet (serviceAppliers). // // It runs on the event loop, after the refusal has already been reported, and it is deliberately // best-effort. A refusal is already surfaced as GitPathAccepted=False; failing to ask for a reapply @@ -178,7 +179,7 @@ func (l *branchWorkerEventLoop) reapplyForRefusal( return } - l.oweReapply(key, owedRefusal{detail: detail, uid: uid}, observation) + l.addUnsentRefusal(key, unsentRefusal{detail: detail, uid: uid}, observation) } // refusalObservation digests what was refused: the objects the write would have produced, and the @@ -263,7 +264,7 @@ func (w *BranchWorker) refusalAlreadyCovered(key refusalKey, observation string) return w.coveredRefusal[newRefusalKey(key.target, key.collection)] == observation } -// recordRefusalObservation remembers what the request just owed covers. An empty observation is +// recordRefusalObservation remembers what the request just decided covers. An empty observation is // recorded too, though it covers nothing (refusalAlreadyCovered): the entry is also the record that // the refusal stands until it recovers (refusalStands). func (w *BranchWorker) recordRefusalObservation(key refusalKey, observation string) { @@ -292,7 +293,7 @@ func (w *BranchWorker) recordRefusalObservation(key refusalKey, observation stri // Dropping the memory is what keeps the fence from becoming a permanent mute: a refused live edit // earns a request, the applier reverts it, and the same edit is made again — sanitized content // cannot tell the second from the first, so only the acceptance in between can. Cancelling the -// queued request is the other half: an obligation whose object has since been accepted has nothing +// queued request is the other half: a request whose object has since been accepted has nothing // left to ask for, and firing it anyway would move the branch for a refusal that no longer exists. func (l *branchWorkerEventLoop) refusalRecovered(target itypes.ResourceReference, collection itypes.CollectionKey) { if target.Name == "" || target.Namespace == "" { @@ -308,18 +309,18 @@ func (l *branchWorkerEventLoop) refusalRecovered(target itypes.ResourceReference delete(l.refusalPending, key) } } - // An obligation already owed, waiting for a quiet branch, is cancelled the same way. Keyed by + // An unsent refusal, decided and waiting for a quiet branch, is cancelled the same way. Keyed by // target alone it could not be told apart from a sibling collection's, so it outlived the // recovery and asked for an apply for a refusal that no longer existed. - for key := range l.reapplyOwed { + for key := range l.unsentRefusals { if recovered(key) { - delete(l.reapplyOwed, key) + delete(l.unsentRefusals, key) } } l.rearmRefusalTimer() } -// refusalStands reports whether a request was owed for this refusal and it has not recovered since. +// refusalStands reports whether a request was decided for this refusal and it has not recovered since. func (w *BranchWorker) refusalStands(key refusalKey) bool { w.refusalMu.Lock() defer w.refusalMu.Unlock() @@ -390,7 +391,7 @@ func (l *branchWorkerEventLoop) rearmRefusalTimer() { l.refusalTimer = time.NewTimer(max(time.Until(earliest), 0)) } -// flushPendingReapply owes every coalesced request that has come due. +// flushPendingReapply decides every coalesced request that has come due, as an unsent refusal. // // Consent is re-checked per entry rather than trusted from when the refusal arrived, because a // minute is long enough for a target to have been suspended or set back to Ignore, and acting on @@ -409,7 +410,7 @@ func (l *branchWorkerEventLoop) flushPendingReapply() { if !consents { continue } - // Re-checked here as well as on arrival: another request for this collection may have been owed + // Re-checked here as well as on arrival: another request for this collection may have been decided // while this entry waited, and if it covered the same observation this one has nothing // left to ask for. An entry whose collection RECOVERED in the meantime is not here at all — // refusalRecovered removed it. @@ -422,13 +423,13 @@ func (l *branchWorkerEventLoop) flushPendingReapply() { l.armTrailingReapply(entry.key, entry.detail, entry.observation, wait) continue } - l.oweReapply(entry.key, owedRefusal{detail: entry.detail, uid: uid}, entry.observation) + l.addUnsentRefusal(entry.key, unsentRefusal{detail: entry.detail, uid: uid}, entry.observation) } l.rearmRefusalTimer() } // sortedRefusalKeys makes the flush order deterministic, so a test that queues two targets sees -// the same one owed first every run. +// the same one decided first every run. func sortedRefusalKeys(pending map[refusalKey]pendingRefusalReapply) []refusalKey { keys := make([]refusalKey, 0, len(pending)) for key := range pending { @@ -438,19 +439,20 @@ func sortedRefusalKeys(pending map[refusalKey]pendingRefusalReapply) []refusalKe return keys } -// oweReapply records that the target's applier is owed a Refusal request, which serviceAppliers -// sends once the branch is quiet. Several refusals for one target before then share one request. -func (l *branchWorkerEventLoop) oweReapply(key refusalKey, refusal owedRefusal, observation string) { - // Recorded as covered now, when the request is owed, rather than when it is delivered: the +// addUnsentRefusal records that the worker decided a Refusal request for the target's applier, +// which serviceAppliers sends once the branch is quiet. Several refusals for one target before then +// share one request. +func (l *branchWorkerEventLoop) addUnsentRefusal(key refusalKey, refusal unsentRefusal, observation string) { + // Recorded as covered now, when the request is decided, rather than when it is delivered: the // request is the worker's decision, and a recheck of the same refusal while the branch is still - // busy must not owe a second one. An attempt that does not deliver gives the refusal back while + // busy must not decide a second one. An attempt that does not deliver gives the refusal back while // it stands (restoreAttempt). l.w.recordRefusalObservation(key, observation) - if l.reapplyOwed == nil { - l.reapplyOwed = map[refusalKey]owedRefusal{} + if l.unsentRefusals == nil { + l.unsentRefusals = map[refusalKey]unsentRefusal{} } - l.reapplyOwed[key] = refusal - l.w.Log.Info("Refused edit owes its applier a reapply, sent once the branch is quiet", + l.unsentRefusals[key] = refusal + l.w.Log.Info("Refused edit: asking its applier to reapply once the branch is quiet", "gitTarget", key.target.String(), "branch", l.w.Branch, "refused", refusal.detail) } diff --git a/internal/git/refusal_reapply_test.go b/internal/git/refusal_reapply_test.go index d8f0934b3..5aec47b47 100644 --- a/internal/git/refusal_reapply_test.go +++ b/internal/git/refusal_reapply_test.go @@ -24,6 +24,7 @@ import ( "github.com/go-logr/logr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -48,14 +49,17 @@ func refusalTouchWorker(t *testing.T, spec configv1alpha3.GitTargetSpec) *Branch if spec.Path == "" { spec.Path = "apps" } - if spec.OnRefusal == configv1alpha3.RefusalActionReapply && spec.Applier == nil { - spec.Applier = testApplier("editing") + if spec.OnRefusal == configv1alpha3.RefusalActionReapply && spec.ApplyTriggerRef == nil { + spec.ApplyTriggerRef = testApplier("editing") } target := &configv1alpha3.GitTarget{ ObjectMeta: metav1.ObjectMeta{Name: "editing", Namespace: "team-a", UID: editingUID}, Spec: spec, } - client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(target).Build() + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(testTriggersFor([]client.Object{target})...). + Build() w := newMetricsTestWorker() // A GitTarget names its GitProvider in its own namespace, which is where a worker looks for @@ -180,7 +184,7 @@ func TestRefusalReapply_OwedRequestWaitsForAQuietBranch(t *testing.T) { t.Cleanup(loop.stopTimers) loop.openWindow = &openWindow{} - loop.oweReapply(editingKey(), owedRefusal{detail: "a base-owned field", uid: editingUID}, "observation-1") + loop.addUnsentRefusal(editingKey(), unsentRefusal{detail: "a base-owned field", uid: editingUID}, "observation-1") loop.serviceAppliers() assert.Empty(t, requester.taken(), "nothing is sent while a commit window is open") @@ -189,10 +193,10 @@ func TestRefusalReapply_OwedRequestWaitsForAQuietBranch(t *testing.T) { requests := requester.taken() require.Len(t, requests, 1) assert.Equal(t, configv1alpha3.ApplyReasonRefusal, requests[0].Reason) - assert.Equal(t, []itypes.ResourceReference{editingRef()}, requests[0].TargetRefs(), + assert.Equal(t, []itypes.ResourceReference{editingRef()}, targetRefs(requests[0]), "only the refusing target's applier is asked") assert.True(t, w.refusalAlreadyCovered(editingKey(), "observation-1"), - "the observation is covered once the request is owed") + "the observation is covered once the request is decided") } // gitOut runs git against a repository on disk and returns its trimmed output. @@ -403,9 +407,9 @@ func TestRefusalReapply_OneTargetDoesNotDropAnothersPendingRequest(t *testing.T) ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "team-a"}, Spec: configv1alpha3.GitTargetSpec{ Branch: "main", Path: "apps/" + name, - OnRefusal: configv1alpha3.RefusalActionReapply, - Applier: testApplier(name), - Suspend: suspend, + OnRefusal: configv1alpha3.RefusalActionReapply, + ApplyTriggerRef: testApplier(name), + Suspend: suspend, }, } target.Spec.GitProviderRef.Name = "test-provider" @@ -413,7 +417,7 @@ func TestRefusalReapply_OneTargetDoesNotDropAnothersPendingRequest(t *testing.T) } // B is suspended, so its delayed request must be declined when the timer fires. A is not. client := fake.NewClientBuilder().WithScheme(scheme). - WithObjects(newTarget("alpha", false), newTarget("bravo", true)).Build() + WithObjects(testTriggersFor([]client.Object{newTarget("alpha", false), newTarget("bravo", true)})...).Build() w := newMetricsTestWorker() w.Client = client @@ -440,7 +444,7 @@ func TestRefusalReapply_OneTargetDoesNotDropAnothersPendingRequest(t *testing.T) assert.Empty(t, loop.refusalPending, "every due entry is consumed exactly once") limitedAlpha, _ := w.refusalRateLimited(alpha) assert.True(t, limitedAlpha, - "alpha's request must have been owed, consuming its rate-limit window") + "alpha's request must have been decided, consuming its rate-limit window") limitedBravo, _ := w.refusalRateLimited(bravo) assert.False(t, limitedBravo, "bravo's was declined for lack of consent, so it must not have consumed a window") @@ -469,9 +473,83 @@ func TestRefusalReapply_TheTimerTracksTheEarliestDeadline(t *testing.T) { } } -// testApplier is a GitTarget applier naming a Kustomization of the given name. -func testApplier(name string) *configv1alpha3.GitTargetApplier { - return &configv1alpha3.GitTargetApplier{KustomizationRef: &meta.NamespacedObjectReference{Name: name}} +// testApplier is a GitTarget's reference to the FluxTrigger of the given name. testTriggersFor +// creates the trigger, holding a Kustomization of the same name. +func testApplier(name string) *configv1alpha3.ApplyTriggerReference { + return &configv1alpha3.ApplyTriggerReference{Kind: configv1alpha3.ApplyTriggerKindFlux, Name: name} +} + +// fluxTrigger is a FluxTrigger in team-a holding the Kustomization of its own name, in the default +// cluster. +func fluxTrigger(name string, mutate ...func(*configv1alpha3.FluxTrigger)) *configv1alpha3.FluxTrigger { + trigger := &configv1alpha3.FluxTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "team-a"}, + Spec: configv1alpha3.FluxTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: configv1alpha3.DefaultClusterProviderName}, + KustomizationRef: meta.NamespacedObjectReference{Name: name}, + }, + } + for _, m := range mutate { + m(trigger) + } + return trigger +} + +// quietFor sets a FluxTrigger's quietPeriod. +func quietFor(d time.Duration) func(*configv1alpha3.FluxTrigger) { + return func(t *configv1alpha3.FluxTrigger) { t.Spec.QuietPeriod = &metav1.Duration{Duration: d} } +} + +// testTriggersFor adds to objects the trigger each GitTarget among them names that objects does not +// hold already: a FluxTrigger as fluxTrigger builds it, or an ArgoCDTrigger holding the Application +// argocd/, each in the GitTarget's namespace. +func testTriggersFor(objects []client.Object) []client.Object { + held := map[string]bool{} + for _, obj := range objects { + if trigger, ok := obj.(configv1alpha3.ApplyTrigger); ok { + held[trigger.TriggerKind()+"/"+obj.GetNamespace()+"/"+obj.GetName()] = true + } + } + out := append([]client.Object(nil), objects...) + for _, obj := range objects { + target, ok := obj.(*configv1alpha3.GitTarget) + if !ok || target.Spec.ApplyTriggerRef == nil { + continue + } + ref := target.Spec.ApplyTriggerRef + key := ref.Kind + "/" + target.Namespace + "/" + ref.Name + if held[key] { + continue + } + held[key] = true + out = append(out, testTrigger(target.Namespace, ref)) + } + return out +} + +// testTrigger is the trigger testTriggersFor creates for ref, in namespace. +func testTrigger(namespace string, ref *configv1alpha3.ApplyTriggerReference) client.Object { + if ref.Kind == configv1alpha3.ApplyTriggerKindArgoCD { + return &configv1alpha3.ArgoCDTrigger{ + ObjectMeta: metav1.ObjectMeta{Name: ref.Name, Namespace: namespace}, + Spec: configv1alpha3.ArgoCDTriggerSpec{ + ClusterProviderRef: &meta.LocalObjectReference{Name: configv1alpha3.DefaultClusterProviderName}, + ApplicationRef: meta.NamespacedObjectReference{Namespace: "argocd", Name: ref.Name}, + }, + } + } + trigger := fluxTrigger(ref.Name) + trigger.Namespace = namespace + return trigger +} + +// createTestTrigger creates, through c, the trigger target names, unless it exists. +func createTestTrigger(t *testing.T, c client.Client, target *configv1alpha3.GitTarget) { + t.Helper() + err := c.Create(context.Background(), testTrigger(target.Namespace, target.Spec.ApplyTriggerRef)) + if err != nil && !apierrors.IsAlreadyExists(err) { + require.NoError(t, err) + } } // recordingRequester records every apply request a worker sends, and reports each one at once with @@ -480,12 +558,20 @@ func testApplier(name string) *configv1alpha3.GitTargetApplier { type recordingRequester struct { mu sync.Mutex requests []ApplyRequest + // prefetches are the PrefetchSource requests, in order. + prefetches []PrefetchRequest // counted keeps the reasons of the requests taken, so refusals still sees them. counted map[configv1alpha3.ApplyReason]int outcome ApplyOutcome manual bool } +func (r *recordingRequester) PrefetchSource(req PrefetchRequest) { + r.mu.Lock() + defer r.mu.Unlock() + r.prefetches = append(r.prefetches, req) +} + func (r *recordingRequester) RequestApply(req ApplyRequest) { r.mu.Lock() r.requests = append(r.requests, req) @@ -495,7 +581,7 @@ func (r *recordingRequester) RequestApply(req ApplyRequest) { return } if outcome == "" { - outcome = ApplyDelivered + outcome = ApplySucceeded } req.Report(ApplyResult{ID: req.ID, Outcome: outcome}) } diff --git a/internal/git/render_fidelity_drops.go b/internal/git/render_fidelity_drops.go new file mode 100644 index 000000000..58506db6c --- /dev/null +++ b/internal/git/render_fidelity_drops.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// This file is the branch worker's side of the render-fidelity gate: what it records when a replay +// resync applies, and what it says when the gate drops a write. +// +// The gate is per GitTarget and its scopes are per collection: one scope that has not reported +// under its current revision closes normal writes for every collection of the target. A live +// event or atomic write that meets a closed gate is DROPPED, and nothing replays it: the watch +// cursor has already moved past it. So the drop is counted (git_writes_dropped_total) and the first +// one under each gate status is logged at Info, naming the scopes the gate waits for. + +// The kind label of git_writes_dropped_total. +const ( + writeDroppedEvent = "event" + writeDroppedAtomic = "atomic" +) + +// The reason label of git_writes_dropped_total. +const ( + writeDropRenderFidelityPending = "render_fidelity_pending" + writeDropRenderFidelityDiverged = "render_fidelity_diverged" +) + +// renderFidelityDropMark is the gate status a GitTarget's last Info-logged drop was made under. +type renderFidelityDropMark struct { + revision uint64 + state RenderFidelityState +} + +// recordReplayClean records the scope a replay resync measured as clean, now: on the loop, once +// its plan was accepted, and before its caller is answered. +// +// The caller records the same result once it has the answer (the watch layer's resync drain), and +// that is where the condition is published. But the caller runs on another goroutine, and the loop +// takes its next item the moment it has answered. A live event queued behind the resync would then +// meet the gate before the caller recorded anything, find it closed, and be dropped for good. The +// e2e spec for a rules[].sourceNamespace override lost its ConfigMap this way, on a sibling scope +// whose resync sat ahead of the event in the queue. +// +// Recording it twice is harmless: the gate accepts a second clean result under the same revision +// unchanged, and ignores a revision the plan has moved past. A divergence is left to the caller, +// because a scope that diverged keeps the gate closed exactly as a pending one does. +func (l *branchWorkerEventLoop) recordReplayClean(req *ResyncRequest) { + if l.w.renderFidelityGate == nil || req.RenderFidelityRevision == 0 || req.Scope == nil { + return + } + l.w.renderFidelityGate.RecordScopeClean( + itypes.NewResourceReference(req.GitTargetName, req.GitTargetNamespace), + req.RenderFidelityRevision, req.Scope.Collection) +} + +// dropAtRenderFidelity counts one write the closed gate dropped, and logs it: at Info for the first +// drop of target under this gate status, at V(1) for the rest, so a target whose gate stays closed +// through a burst of edits says so once without logging every edit. resource names the dropped +// object, empty for an atomic write. +func (l *branchWorkerEventLoop) dropAtRenderFidelity( + target itypes.ResourceReference, + status RenderFidelityStatus, + kind, resource string, +) { + reason := writeDropRenderFidelityPending + if status.State == RenderFidelityFalse { + reason = writeDropRenderFidelityDiverged + } + log := l.w.Log.WithValues("gitTarget", target.String(), "kind", kind, "reason", reason, + "renderFidelity", status.Message) + if resource != "" { + log = log.WithValues("resource", resource) + } + mark := renderFidelityDropMark{revision: status.Revision, state: status.State} + if l.fidelityDropsLogged[target.Key()] != mark { + if l.fidelityDropsLogged == nil { + l.fidelityDropsLogged = map[string]renderFidelityDropMark{} + } + l.fidelityDropsLogged[target.Key()] = mark + log.Info("Dropped a write because the GitTarget's render fidelity is not established; it is " + + "missing from Git until the object changes again or its collection replays. Further " + + "drops under this gate status are counted in git_writes_dropped_total and logged at V(1)") + } else { + log.V(1).Info("Dropped a write because the GitTarget's render fidelity is not established") + } + if telemetry.GitWritesDroppedTotal == nil { + return + } + ctx := l.w.ctx + if ctx == nil { + ctx = context.Background() + } + telemetry.GitWritesDroppedTotal.Add(ctx, 1, metric.WithAttributes( + l.w.providerAttrs(attribute.String("kind", kind), attribute.String("reason", reason))...)) +} diff --git a/internal/git/render_fidelity_drops_test.go b/internal/git/render_fidelity_drops_test.go new file mode 100644 index 000000000..09eafaf3e --- /dev/null +++ b/internal/git/render_fidelity_drops_test.go @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +// The branch worker's side of the render-fidelity gate (render_fidelity_drops.go): a replay +// resync records its scope clean on the loop, before its caller hears of it, and a write the +// closed gate drops is counted and logged. + +import ( + "strings" + "sync" + "testing" + "time" + + "github.com/go-logr/logr/funcr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer" + "github.com/ConfigButler/gitops-reverser/internal/telemetry" + itypes "github.com/ConfigButler/gitops-reverser/internal/types" +) + +// replayFixture is a ledger target with a render-fidelity gate whose one scope, ConfigMaps in +// "default", was just started and has not reported. +type replayFixture struct { + *ledgerFixture + + loop *branchWorkerEventLoop + gate *RenderFidelityGate + target itypes.ResourceReference + scope ResyncScope + revision uint64 +} + +func newReplayFixture(t *testing.T, slug string) *replayFixture { + t.Helper() + f := newLedgerFixture(t, slug, true) + f.createLedgerTarget("team-a", nil) + f.publish("prime") + gate := NewRenderFidelityGate() + f.worker.renderFidelityGate = gate + target := itypes.NewResourceReference(ledgerTargetName, "default") + scope := ResyncScopeFor(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, "default") + _, revisions := gate.Reconcile(target, + []itypes.CollectionKey{scope.Collection}, []itypes.CollectionKey{scope.Collection}) + require.False(t, gate.AllowsWrites(target), "precondition: the new scope has not reported") + loop := newBranchWorkerEventLoop(f.worker, time.Hour) + t.Cleanup(loop.stopTimers) + return &replayFixture{ledgerFixture: f, loop: loop, gate: gate, target: target, scope: scope, + revision: revisions[scope.Collection]} +} + +// replay is the replay resync the scope's stream sends once its initial list is complete. +func (f *replayFixture) replay() *ResyncRequest { + return &ResyncRequest{ + Desired: []manifestanalyzer.DesiredResource{desiredCMIn("keep", "default", "blue")}, + GitTargetName: ledgerTargetName, + GitTargetNamespace: "default", + Scope: &f.scope, + RenderFidelityRevision: f.revision, + Result: make(chan ResyncResult, 1), + } +} + +func (f *replayFixture) liveEvent(name string) { + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent(name, "alice", ledgerTargetName)}, + CommitMode: CommitModePerEvent, + }}) +} + +func writesDropped(t *testing.T, reader *sdkmetric.ManualReader, kind, reason string) int64 { + t.Helper() + got, _ := telemetry.CollectInt64Sum(reader, "gitopsreverser_git_writes_dropped_total", + map[string]string{"kind": kind, "reason": reason}) + return got +} + +// TestRenderFidelity_ALiveEventQueuedBehindAReplayResyncIsKept is the race the e2e spec for a +// rules[].sourceNamespace override lost its ConfigMap to. The stream queues its replay resync and +// then a live event behind it. The loop applies the resync, answers, and takes the event at once, +// while the caller that used to record the scope clean has not read the answer yet. The event must +// meet the gate the resync produced. +func TestRenderFidelity_ALiveEventQueuedBehindAReplayResyncIsKept(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newReplayFixture(t, "render-fidelity-replay-then-event") + + req := f.replay() + f.loop.handleResyncRequest(req) + f.loop.endWake(0) + // Nothing has read req.Result: the watch layer's drain, on its own goroutine, is still behind. + f.liveEvent("created-after-the-replay") + + assert.NotNil(t, f.loop.openWindow, "the event queued behind the replay is kept") + assert.True(t, f.gate.AllowsWrites(f.target), "the loop recorded the scope before answering") + assert.Zero(t, writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityPending)) + require.NoError(t, (<-req.Result).Err) + + // The drain records the same result once it has the answer, and the gate takes it unchanged. + status, applied := f.gate.RecordScopeClean(f.target, f.revision, f.scope.Collection) + assert.True(t, applied) + assert.Equal(t, RenderFidelityTrue, status.State) +} + +// A replay resync held back by an unreachable remote is answered at once with the failure, and +// applied later with nobody waiting. Its caller saw only the failure, so it never recorded the +// scope, and the GitTarget stayed closed to live writes after the snapshot had landed. The loop +// records it when it applies. +func TestRenderFidelity_AReplayAppliedAfterAnOutageOpensTheGate(t *testing.T) { + f := newReplayFixture(t, "render-fidelity-replay-after-outage") + _, restoreSyncs := failSyncs(t) + + req := f.replay() + f.loop.handleResyncRequest(req) + f.loop.endWake(0) + require.Error(t, (<-req.Result).Err, "the caller hears the outage, and records nothing") + require.False(t, f.gate.AllowsWrites(f.target)) + + restoreSyncs() + fireRetry(f.loop) + assert.Empty(t, f.loop.pendingWrites, "the retry applied the held resync") + assert.True(t, f.gate.AllowsWrites(f.target), "applying it recorded the scope clean") +} + +// A stream the plan has replaced still reports under the revision it started with. Its replay +// must not open the gate for the scope's new stream. +func TestRenderFidelity_AReplayFromARetiredStreamDoesNotOpenTheGate(t *testing.T) { + f := newReplayFixture(t, "render-fidelity-retired-replay") + stale := f.replay() + _, revisions := f.gate.Reconcile(f.target, + []itypes.CollectionKey{f.scope.Collection}, []itypes.CollectionKey{f.scope.Collection}) + require.NotEqual(t, f.revision, revisions[f.scope.Collection], "precondition: the scope restarted") + + f.loop.handleResyncRequest(stale) + f.loop.endWake(0) + require.NoError(t, (<-stale.Result).Err) + assert.False(t, f.gate.AllowsWrites(f.target), "only the current stream's replay opens the gate") +} + +// TestRenderFidelity_ADroppedWriteIsCountedAndLoggedOncePerGateStatus: a write the closed gate +// drops is gone for good, so every drop is counted, and the first under each gate status is +// logged at Info with what the gate waits for. A burst of edits against one closed gate logs once. +func TestRenderFidelity_ADroppedWriteIsCountedAndLoggedOncePerGateStatus(t *testing.T) { + reader, err := telemetry.InitTestExporter() + require.NoError(t, err) + f := newReplayFixture(t, "render-fidelity-drops") + logged := captureInfo(f.worker) + + f.liveEvent("first") + f.liveEvent("second") + f.loop.handleQueueItem(WorkItem{Request: &WriteRequest{ + Events: []Event{configMapTargetEvent("snapshot", "alice", ledgerTargetName)}, + CommitMode: CommitModeAtomic, + }}) + + assert.Nil(t, f.loop.openWindow, "a pending scope closes the GitTarget") + assert.Equal(t, int64(2), writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityPending)) + assert.Equal(t, int64(1), writesDropped(t, reader, writeDroppedAtomic, writeDropRenderFidelityPending)) + drops := logged.matching("Dropped a write") + require.Len(t, drops, 1, "one Info line for the burst") + assert.Contains(t, drops[0], "Waiting for 1 of 1", "it says which scopes the gate waits for") + assert.Contains(t, drops[0], "configmaps/first") + + _, applied := f.gate.RecordScopeDivergence(f.target, f.revision, f.scope.Collection, + manifestanalyzer.RenderDivergence{Field: "data.region", Token: "${REGION}"}) + require.True(t, applied) + f.liveEvent("third") + assert.Equal(t, int64(1), writesDropped(t, reader, writeDroppedEvent, writeDropRenderFidelityDiverged)) + assert.Len(t, logged.matching("Dropped a write"), 2, "a new gate status logs again") +} + +// infoLines collects the worker's Info-level log lines. +type infoLines struct { + mu sync.Mutex + lines []string +} + +func captureInfo(w *BranchWorker) *infoLines { + captured := &infoLines{} + w.Log = funcr.New(func(prefix, args string) { + captured.mu.Lock() + defer captured.mu.Unlock() + captured.lines = append(captured.lines, prefix+" "+args) + }, funcr.Options{Verbosity: 0}) + return captured +} + +func (c *infoLines) matching(substring string) []string { + c.mu.Lock() + defer c.mu.Unlock() + var out []string + for _, line := range c.lines { + if strings.Contains(line, substring) { + out = append(out, line) + } + } + return out +} diff --git a/internal/git/render_fidelity_gate.go b/internal/git/render_fidelity_gate.go index 6998c6e11..270a08f6f 100644 --- a/internal/git/render_fidelity_gate.go +++ b/internal/git/render_fidelity_gate.go @@ -362,7 +362,7 @@ func pendingScopesMessage(state renderFidelityTargetState, scopes []types.Collec if result.finished && result.clean { continue } - name := fmt.Sprintf("%s (owes revision %d", scope.String(), result.revision) + name := fmt.Sprintf("%s (waiting for revision %d", scope.String(), result.revision) if result.refusedRevision != 0 { name += fmt.Sprintf("; refused a report carrying revision %d", result.refusedRevision) } diff --git a/internal/git/render_fidelity_gate_test.go b/internal/git/render_fidelity_gate_test.go index 17e013082..b226b7b70 100644 --- a/internal/git/render_fidelity_gate_test.go +++ b/internal/git/render_fidelity_gate_test.go @@ -298,7 +298,7 @@ func TestRenderFidelityGate_PendingMessageNamesTheScope(t *testing.T) { require.True(t, applied) require.Equal(t, RenderFidelityUnknown, status.State) // The one that reported must drop out of the message; the one still pending must stay, - // carrying the revision it owes a report under. + // carrying the revision it is waiting for a report at. assert.Contains(t, status.Message, "1 of 2") assert.NotContains(t, status.Message, deployment.String()) assert.Contains(t, status.Message, configMap.String()) @@ -345,7 +345,7 @@ func TestRenderFidelityGate_PendingMessageNamesARefusedReport(t *testing.T) { status := gate.Status(target) require.Equal(t, RenderFidelityUnknown, status.State) - assert.Contains(t, status.Message, fmt.Sprintf("owes revision %d", current)) + assert.Contains(t, status.Message, fmt.Sprintf("waiting for revision %d", current)) assert.Contains(t, status.Message, fmt.Sprintf("refused a report carrying revision %d", first[scope])) // The correct report still converges the target, and clears the scope from the message. @@ -376,7 +376,7 @@ func TestRenderFidelityGate_InvalidateClosesWritesUntilAFreshMeasurement(t *test // A pass that restarts every scope is the fresh measurement, and only its result reopens. _, after := restartAll(gate, target, scope) - assert.False(t, gate.AllowsWrites(target), "the fresh scope still owes a report") + assert.False(t, gate.AllowsWrites(target), "the fresh scope is still waiting for a report") _, applied = gate.RecordScopeClean(target, after[scope], scope) require.True(t, applied) assert.True(t, gate.AllowsWrites(target)) diff --git a/internal/git/replay_failure_test.go b/internal/git/replay_failure_test.go index ad6e3957e..d0e90eace 100644 --- a/internal/git/replay_failure_test.go +++ b/internal/git/replay_failure_test.go @@ -18,7 +18,7 @@ package git // worktreeDirty is false throughout: the reset cleared it, and the worktree really is clean. It is // the retained writes that are stale, which is why this needs an invariant of its own. // -// See docs/finished/push-notification-and-reconcile-trigger.md §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md §1.5. import ( "context" diff --git a/internal/git/replay_refusal_test.go b/internal/git/replay_refusal_test.go index 0d8c80448..e3deddf26 100644 --- a/internal/git/replay_refusal_test.go +++ b/internal/git/replay_refusal_test.go @@ -88,7 +88,7 @@ func assertOnlyTheRefusedWriteWasDropped( assert.NotContains(t, names, "refused-on-replay", "the refused write is not") assert.Contains(t, names, "team-a/kustomization.yaml", "and the other writer's commit is kept") assert.Empty(t, loop.pendingWrites, "nothing is left pending to block the writes after it") - assert.False(t, loop.retry.pending(), "nothing is owed, so no retry is scheduled") + assert.False(t, loop.retry.pending(), "nothing is left to publish, so no retry is scheduled") require.Len(t, *refusals, 1, "the refusal is reported on the target's status, once") assert.Equal(t, itypes.NewResourceReference(ledgerTargetName, "default"), (*refusals)[0].target) diff --git a/internal/git/reset_cleanliness_test.go b/internal/git/reset_cleanliness_test.go index 71aae9e21..44c9c487d 100644 --- a/internal/git/reset_cleanliness_test.go +++ b/internal/git/reset_cleanliness_test.go @@ -12,7 +12,7 @@ package git // reaches the server, so the compare-and-swap caveat that forces other tests onto // startRealGitServer does not apply. // -// See docs/finished/push-notification-and-reconcile-trigger.md §1.5. +// See docs/design/push-notification-and-reconcile-trigger.md §1.5. import ( "context" diff --git a/internal/git/retry.go b/internal/git/retry.go index 45c50e145..13e1df5bc 100644 --- a/internal/git/retry.go +++ b/internal/git/retry.go @@ -4,8 +4,8 @@ package git import "time" -// This file is the worker's one retry schedule: the deadline at which the work it still owes is -// attempted again, whatever made the last attempt fail. +// This file is the worker's one retry schedule: the deadline at which the worker attempts its +// pending writes again, whatever made the last attempt fail. // // The cooldown in maybeSchedulePush spaces SUCCESSFUL pushes; it says nothing about failed ones, // because lastPushAt advances only on success. Without a schedule of its own a failure had two @@ -15,7 +15,7 @@ import "time" // // So a failed attempt schedules the next one: due one backoff later, 10s doubling to 5m, on a timer // of its own, whether or not anything else arrives. Until then new commits accumulate instead of -// pushing. A successful publication, or nothing left owed, clears it. +// pushing. A successful publication, or no pending writes left, clears it. // // It used to be two schedules, one for a failed push and one for parent recovery's probe, and the // two had to hand the work back and forth: a failure while recovery was open deferred to recovery's @@ -31,7 +31,7 @@ var ( const retryBackoffFactor = 2 -// retrySchedule is the loop's one retry deadline. The zero value is "nothing owed a retry". +// retrySchedule is the loop's one retry deadline. The zero value is "no retry scheduled". // Loop-goroutine only. type retrySchedule struct { backoff time.Duration @@ -71,8 +71,8 @@ func (l *branchWorkerEventLoop) scheduleRetry(cause error) { l.publishRecovery() } -// endOutage closes the retry and parent recovery together: a publication landed, or nothing is owed -// any more. Either ends whatever the last failed attempt was waiting out, and with it the pause on +// endOutage closes the retry and parent recovery together: a publication landed, or no pending +// writes remain. Either ends whatever the last failed attempt was waiting out, and with it the pause on // intake (intake.go). func (l *branchWorkerEventLoop) endOutage() { l.recovery = parentRecovery{} diff --git a/internal/git/retry_test.go b/internal/git/retry_test.go index a3c6db3f7..18ed1438a 100644 --- a/internal/git/retry_test.go +++ b/internal/git/retry_test.go @@ -165,7 +165,7 @@ func TestPublicationRetry_AMissingParentIsProbedOnTheOneSchedule(t *testing.T) { assert.Nil(t, f.loop.pushTimer, "the push timer is only the success cooldown") } -// Once the parent is back, recovery's missing-parent hold is gone but its obligation stays open +// Once the parent is back, recovery's missing-parent hold is gone but the recovery stays open // until the work publishes. A publication that fails then must not be retried by every new commit: // the commits wait for recovery's next deadline. Found in review of #412; it failed on main too. func TestPublicationRetry_AFailureAfterTheParentReturnsWaitsForRecovery(t *testing.T) { diff --git a/internal/git/testdata/git-roundtrip-ledger.golden b/internal/git/testdata/git-roundtrip-ledger.golden index 4a97bc80a..1084d44f9 100644 --- a/internal/git/testdata/git-roundtrip-ledger.golden +++ b/internal/git/testdata/git-roundtrip-ledger.golden @@ -1,6 +1,6 @@ # Round trips to the Git host, per operation. # Generated by TestGitRoundTripLedger; specified by -# docs/finished/push-notification-and-reconcile-trigger.md §1.6. +# docs/design/push-notification-and-reconcile-trigger.md §1.6. # # Regenerate with: go test ./internal/git -run TestGitRoundTripLedger -update # diff --git a/internal/git/types.go b/internal/git/types.go index af77b9de8..bc2f4dec4 100644 --- a/internal/git/types.go +++ b/internal/git/types.go @@ -520,6 +520,10 @@ type ResyncRequest struct { // whole-GitTarget resync, which speaks for no single collection. Diagnostic only: nothing // filters the queue on it. See source_collection.go. SourceCollection types.CollectionKey + // RenderFidelityRevision is the render-fidelity revision of the target-watch stream whose + // replay this is, for the scope Scope.Collection names. Zero for a resync no stream replayed. + // The worker records that scope clean once the resync applies (recordReplayClean). + RenderFidelityRevision uint64 // Result receives exactly one reply. It is buffered (cap 1) by the emitter so // the worker never blocks delivering it. Result chan ResyncResult @@ -668,6 +672,11 @@ type Event struct { // apply window reads it to tell an applier's write from an API edit (apply_window.go). LastFieldManagers []string + // LastFieldManagedAt is when those entries were written, to the second; zero with no + // LastFieldManagers. The apply window compares it with the time of an API edit: a late apply + // moves the applier's entry forward, and an undo that deletes the editor's entry does not. + LastFieldManagedAt time.Time + // UserInfo contains user information for commit messages. UserInfo UserInfo diff --git a/internal/git/worker_manager.go b/internal/git/worker_manager.go index 1f13c8724..7fb31f9ab 100644 --- a/internal/git/worker_manager.go +++ b/internal/git/worker_manager.go @@ -172,6 +172,10 @@ type WorkerManager struct { // (SetApplyRequester) before any worker is created; nil in the CLI and in tests, which switches // the apply trigger off. applyRequester ApplyRequester + + // triggerStatus writes what every worker publishes on its apply triggers' status. Set once at + // startup (SetApplyTriggerStatusWriter); nil in the CLI and in tests, which publishes nothing. + triggerStatus ApplyTriggerStatusWriter } // NewWorkerManager creates a new worker manager. limits bounds every worker this manager @@ -274,6 +278,14 @@ func (m *WorkerManager) SetApplyRequester(requester ApplyRequester) { m.applyRequester = requester } +// SetApplyTriggerStatusWriter injects what writes every worker's apply trigger status. Like +// SetApplyRequester, it is called once at startup before any worker is created. +func (m *WorkerManager) SetApplyTriggerStatusWriter(writer ApplyTriggerStatusWriter) { + m.mu.Lock() + defer m.mu.Unlock() + m.triggerStatus = writer +} + // recordRemoteObservation is the delivery point for one branch's confirmed observation. Every // worker is given it bound to its own key; it is called on the worker's loop goroutine. func (m *WorkerManager) recordRemoteObservation(key BranchKey, observed RemoteObservation) { @@ -392,6 +404,7 @@ func (m *WorkerManager) EnsureWorker( worker.publicationReporter = func() { go m.notifyPublication(managerCtx, key) } worker.scanAcceptance = m.scanAcceptance worker.applyRequester = m.applyRequester + worker.triggerStatus = m.triggerStatus m.siblings.add(worker) worker.renderFidelityGate = m.renderFidelityGate worker.crOwners = m.commitRequests diff --git a/internal/git/worker_manager_test.go b/internal/git/worker_manager_test.go index 3ce329b9c..0a3b5d183 100644 --- a/internal/git/worker_manager_test.go +++ b/internal/git/worker_manager_test.go @@ -600,5 +600,5 @@ func TestWorkerLifecycle_AReplacementDoesNotReclaimItsPredecessorsCheckout(t *te require.NoError(t, err, "and left the replacement's alone") assert.True(t, manager.ReplacementPending( BranchKey{RepoNamespace: "gitops-system", RepoName: "repo1", Branch: "main"}), - "and the targets on the branch still owe a rebuild in the new one") + "and the targets on the branch still wait to be rebuilt in the new one") } diff --git a/internal/git/worker_repo_identity_test.go b/internal/git/worker_repo_identity_test.go index 6fd70ab09..b61169669 100644 --- a/internal/git/worker_repo_identity_test.go +++ b/internal/git/worker_repo_identity_test.go @@ -259,7 +259,7 @@ func TestEnqueue_RefusesOnceTheWorkerIsStopping(t *testing.T) { "an event the shutdown drain would discard must not be reported as accepted") assert.False(t, worker.EnqueueResync(&ResyncRequest{}), "and a resync must be answered rather than left waiting on a reply that never comes") - // The two fire-and-forget paths report nothing, so what they owe is to leave no trace: an + // The two fire-and-forget paths report nothing, so they must at least leave no trace: an // increment here would be work the depth gauge counts and nothing will ever finish. worker.EnqueueAttach(&AttachCommitRequest{Namespace: "ns", Name: "cr"}) worker.EnqueueRefresh(&RefreshRequest{}) diff --git a/internal/git/write_gate_test.go b/internal/git/write_gate_test.go index 53259c077..49f01db20 100644 --- a/internal/git/write_gate_test.go +++ b/internal/git/write_gate_test.go @@ -256,7 +256,7 @@ func TestWriteGate_AGateClosingAfterTheLocalCommitDoesNotRetractIt(t *testing.T) gate.close(t, worker) loop.pushPending() _, resolved := worker.LookupCommitRequestOutcome("default", req.Name, req.UID) - require.False(t, resolved, "the gate closing does not resolve a request the push still owes") + require.False(t, resolved, "the gate closing does not resolve a request still waiting for its push") restore() loop.pushPending() diff --git a/internal/manifestanalyzer/placement_test.go b/internal/manifestanalyzer/placement_test.go index e01d7590c..4819dde8c 100644 --- a/internal/manifestanalyzer/placement_test.go +++ b/internal/manifestanalyzer/placement_test.go @@ -314,7 +314,7 @@ func TestLocateNew_KustomizeContextNamespaceMismatch_NewFileKeepsItsOwnNamespace } } -// A DECLARED template pointing into a governed directory carries the same obligation as +// A DECLARED template pointing into a governed directory follows the same rule as // the kustomize-root fallback: the context supplies the namespace, so the file must not // repeat it. Before the Option C deletion this only ever came out of a sibling's own // bytes, so a declared placement silently wrote a namespace: line the folder omits. diff --git a/internal/queue/fact_index_test.go b/internal/queue/fact_index_test.go index 3a4667790..b79cf5d67 100644 --- a/internal/queue/fact_index_test.go +++ b/internal/queue/fact_index_test.go @@ -230,10 +230,14 @@ func TestFactIndex_RewritingOneKeyDoesNotStrandTheEvictionOrder(t *testing.T) { harness.publish(factIndexTestStream("prod-eu-1"), facts...) // Waiting on the ORDER rather than on Len is the only barrier available here: Len reaches 2 on - // the first fact and stays there, so it cannot say the batch has finished applying. + // the first fact and stays there, so it cannot say the batch has finished applying. Its LENGTH, + // not its capacity: every rewrite appends one exact and one latest reference, and the follower + // stores the batch one fact at a time, so the capacity passes rewrites about halfway through. + // A sweep run then compacts what has landed, and the rest of the batch grows the order again. scope := factScope{route: "prod-eu-1", groupResource: groupResourceKey("apps", "deployments")} - require.Eventually(t, func() bool { return harness.orderCapacity(scope) > rewrites }, - factIndexTestGrace, factIndexTestBlock, + require.Eventually(t, func() bool { return harness.orderLen(scope) == 2*rewrites }, + factIndexTestGrace, factIndexTestBlock, "the follower never applied the whole batch") + require.Greater(t, harness.orderCapacity(scope), rewrites, "the references are what the sweep has to reclaim, so the test is only meaningful if they piled up") require.Equal(t, 2, harness.index.Len(), "one exact and one latest entry, however many rewrites landed") @@ -259,6 +263,18 @@ func (h *factIndexHarness) orderCapacity(scope factScope) int { return cap(facts.order) } +// orderLen reads how many references one scope's eviction order holds, under the index lock. +func (h *factIndexHarness) orderLen(scope factScope) int { + h.t.Helper() + h.index.mu.Lock() + defer h.index.mu.Unlock() + facts, ok := h.index.scopes[scope] + if !ok { + return 0 + } + return len(facts.order) +} + func TestFactIndex_PerTypeCapEvictsOldestFirstAndCountsIt(t *testing.T) { reader, err := telemetry.InitTestExporter() require.NoError(t, err) diff --git a/internal/telemetry/apply_requests.go b/internal/telemetry/apply_requests.go index 0eaf2fa03..1a7623250 100644 --- a/internal/telemetry/apply_requests.go +++ b/internal/telemetry/apply_requests.go @@ -8,14 +8,15 @@ import "go.opentelemetry.io/otel/metric" // object, labelled by {applier_kind, reason, outcome}. // // applier_kind names the applier: "kustomization" (a Flux Kustomization) or "application" (an Argo -// CD Application). reason is the attempt's ApplyReason (ForeignCommit, Refusal, CatchUp; never -// CatchUp for an application). outcome is a frozen enum: +// CD Application). reason is the attempt's ApplyReason (ForeignCommit, Refusal, Repair, CatchUp; +// never CatchUp for an application). outcome is a frozen enum: // -// sent — the applier was asked and the ask landed: a requestedAt stamp was patched or a sync -// operation set, or, for a CatchUp, the applier's source already held the commit and -// nothing was owed. +// sent — the applier was asked and the ask landed: a Kustomization was resumed with a +// requestedAt token or a sync operation set. Or it needed no request: a ForeignCommit or +// a CatchUp found the Kustomization suspended with the commit applied already, or the +// remote no longer carries the branch, so there is no commit to apply. // failed — the attempt could not ask, including an applier that applies another repository or -// branch. The branch worker owns the obligation and decides a retry on a quiet branch, +// branch. The branch worker keeps the reason pending and decides a retry on a quiet branch, // with a backoff of its own, so a failure that persists keeps counting. An // ApplyRequestFailed Event marks the first failure of a run. // dropped — the attempt asked nothing, because no GitTarget may still ask for its reason: each @@ -26,10 +27,19 @@ import "go.opentelemetry.io/otel/metric" // moment. Frequent stale attempts mean edits keep arriving within an attempt's lifetime. // // It counts attempts, so a request that fails twice and then lands is two "failed" and one "sent". -// It says nothing about whether the applier APPLIED: a request asks and never reports, and the -// applier's own status is the record of what it did. +// It says nothing about whether the applier APPLIED: the operation a sent request starts is +// followed by the branch worker, and an ApplyFailed or ApplyUnresolved Event marks one that did not +// complete, or not in time (apply_operations_unresolved counts the latter). // // No GitTarget identity: one attempt is for one applier, which several targets can share, so a // target label would have to invent an owner. The ApplyRequested and ApplyRequestFailed Events on // each GitTarget carry that identity instead. var ApplyRequestsTotal metric.Int64Counter + +// ApplyRevisionMismatchesTotal counts apply operations that succeeded on another revision than the +// commit GitOps Reverser checked, labelled by {applier_kind}. A Flux Kustomization applies whatever +// its GitRepository's artifact holds when it reconciles; the artifact is read again just before the +// resume, so this is somebody stamping the GitRepository between that read and the reconcile (a +// person, or the source's own interval). The apply window still withholds a late write over a +// newer API edit; a steady rate means something else keeps fetching the source. +var ApplyRevisionMismatchesTotal metric.Int64Counter diff --git a/internal/telemetry/apply_window.go b/internal/telemetry/apply_window.go index 89d6f429e..12171c670 100644 --- a/internal/telemetry/apply_window.go +++ b/internal/telemetry/apply_window.go @@ -5,13 +5,13 @@ package telemetry import "go.opentelemetry.io/otel/metric" // ApplyWindowCollisionsTotal counts writes by an applier, inside its apply window (from the attempt -// being sent until Flux finished it, plus a short grace), to an object the API side edited after the +// being sent until its operation completed, plus a short grace), to an object the API side edited after the // attempt was sent: the apply arriving late over a newer edit. Labelled by {provider_namespace, // provider_name, branch, outcome}. outcome is a frozen enum: // // withheld — the write is proven to be the applier's (its field manager alone wrote last), so it -// was not mirrored. Git keeps the API's value, and the next apply puts it back on the -// cluster. The edit was reverted on the cluster until then. +// was not mirrored. Git keeps the API's value, and a Repair is pending for the applier, an +// apply that puts it back on the cluster. The edit was reverted on the cluster until then. // ambiguous — the applier and another field manager wrote in the same second, so the write may be // the API edit itself. It was mirrored as ever, which cannot lose an edit; if it was the // apply, the revert is in Git as the applier's commit. diff --git a/internal/telemetry/exporter.go b/internal/telemetry/exporter.go index ca6f0f41d..b4251c925 100644 --- a/internal/telemetry/exporter.go +++ b/internal/telemetry/exporter.go @@ -240,6 +240,14 @@ var ( // every object in that window until the next resync re-derives them, and until now the only // trace was a log line (or, for a refusal, a GitTarget condition nobody is alerting on). GitCommitFailuresTotal metric.Int64Counter + // GitWritesDroppedTotal counts the live events and atomic writes a branch worker dropped + // because their GitTarget's render fidelity was not established, labelled by + // {provider_namespace, provider_name, branch, kind, reason}. kind is `event` (one live event) + // or `atomic` (an atomic write request); reason is `render_fidelity_pending` (a scope of the + // GitTarget has not reported under its current revision yet) or `render_fidelity_diverged` (a + // rendered value does not match the live object). A dropped write is not retried: the edit is + // missing from Git until the object changes again or its collection replays. + GitWritesDroppedTotal metric.Int64Counter // GitCommitWindowsTotal counts closed commit windows, labelled by {gittarget_namespace, // gittarget_name, close_reason, timer_source}. close_reason says what ended the window // (idle_timeout, max_duration, attach_next, identity_change, buffer_limit, resync_before_apply, @@ -562,6 +570,7 @@ func registerCounters() error { {"gitopsreverser_git_commits_total", &GitCommitsTotal}, {"gitopsreverser_commit_requests_total", &CommitRequestsTotal}, {"gitopsreverser_git_commit_failures_total", &GitCommitFailuresTotal}, + {"gitopsreverser_git_writes_dropped_total", &GitWritesDroppedTotal}, {"gitopsreverser_git_commit_windows_total", &GitCommitWindowsTotal}, {"gitopsreverser_git_pushes_total", &GitPushesTotal}, {"gitopsreverser_git_push_retries_total", &GitPushRetriesTotal}, @@ -597,6 +606,7 @@ func registerCounters() error { {"gitopsreverser_watch_plan_passes_total", &WatchPlanPassesTotal}, {"gitopsreverser_watch_plan_triggers_total", &WatchPlanTriggersTotal}, {"gitopsreverser_apply_requests_total", &ApplyRequestsTotal}, + {"gitopsreverser_apply_revision_mismatches_total", &ApplyRevisionMismatchesTotal}, {"gitopsreverser_apply_window_collisions_total", &ApplyWindowCollisionsTotal}, {"gitopsreverser_receiver_deliveries_total", &ReceiverDeliveriesTotal}, } @@ -730,6 +740,7 @@ func registerObservableGauges() error { }, {"gitopsreverser_resource_condition", GaugeResourceCondition}, {"gitopsreverser_git_branch_targets", GaugeGitBranchTargets}, + {"gitopsreverser_apply_operations_unresolved", GaugeApplyOperationsUnresolved}, } for _, o := range observable { if _, err := otelMeter.Int64ObservableGauge( diff --git a/internal/telemetry/gauges.go b/internal/telemetry/gauges.go index 34987c195..7c63ad675 100644 --- a/internal/telemetry/gauges.go +++ b/internal/telemetry/gauges.go @@ -60,6 +60,10 @@ const ( // target's join series ends at the next scrape rather than attributing a live branch's // failures to an object that is gone. See branch_targets.go. GaugeGitBranchTargets = "git_branch_targets" + // GaugeApplyOperationsUnresolved is how many apply operations on a branch did not complete + // within the dispatcher's budget and are still followed, each holding its applier's apply + // window open and its next attempt back. Read from what the branch worker's loop last published. + GaugeApplyOperationsUnresolved = "apply_operations_unresolved" ) var ( diff --git a/internal/watch/applier_dynamic_client_test.go b/internal/watch/applier_dynamic_client_test.go new file mode 100644 index 000000000..23fbb5464 --- /dev/null +++ b/internal/watch/applier_dynamic_client_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package watch + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/rest" +) + +// TestApplierDynamicClient_RefusesTheConfigPlane: the empty id is the operator's own cluster, and a +// caller asking for an applier's cluster by an empty name has a bug, not a cluster. +func TestApplierDynamicClient_RefusesTheConfigPlane(t *testing.T) { + m := &Manager{Log: logr.Discard()} + + _, err := m.ApplierDynamicClient(context.Background(), configPlaneClusterID) + require.Error(t, err, "the config plane is not a ClusterProvider") +} + +// TestApplierDynamicClient_ServesAnUndeclaredClusterWithoutKeepingIt: an applier may live in a +// cluster no GitTarget mirrors from. It is served, from the provider as it resolves now, and no +// cluster context is created, so nothing outlives the call. +func TestApplierDynamicClient_ServesAnUndeclaredClusterWithoutKeepingIt(t *testing.T) { + m := &Manager{ + Log: logr.Discard(), + SourceClusters: stubSourceClusterResolver{cfg: &rest.Config{Host: "https://hub.example"}, version: "v1"}, + } + + dc, err := m.ApplierDynamicClient(context.Background(), "hub") + require.NoError(t, err) + require.NotNil(t, dc) + assert.Nil(t, m.clusterContextByID("hub"), "an applier-only cluster creates no cluster context") +} + +// TestApplierDynamicClient_ReusesADeclaredClustersClient: a cluster some GitTarget mirrors from +// already has a client, with the same identity; the applier is asked through that one, built once +// and cached. +func TestApplierDynamicClient_ReusesADeclaredClustersClient(t *testing.T) { + m := &Manager{ + Log: logr.Discard(), + SourceClusters: stubSourceClusterResolver{cfg: &rest.Config{Host: "https://prod.example"}, version: "v1"}, + } + m.rememberGitTargetCluster(gd("checkout"), "prod-eu-1") + + first, err := m.ApplierDynamicClient(context.Background(), "prod-eu-1") + require.NoError(t, err) + second, err := m.ApplierDynamicClient(context.Background(), "prod-eu-1") + require.NoError(t, err) + assert.Same(t, first, second, "the cluster's cached client is reused, not rebuilt per call") + + cc := m.clusterContextByID("prod-eu-1") + require.NotNil(t, cc) + assert.Same(t, first, cc.dynamicClient, "it is the client the cluster's watches run on") +} diff --git a/internal/watch/cluster_context.go b/internal/watch/cluster_context.go index cecb35ca9..7b979ed32 100644 --- a/internal/watch/cluster_context.go +++ b/internal/watch/cluster_context.go @@ -696,35 +696,19 @@ func (m *Manager) clusterDynamicClient(ctx context.Context, clusterID string) (d return dc, nil } -// SourceDynamicClient returns the dynamic client for a source cluster, named by its -// ClusterProvider (GitTarget.SourceCluster()). It is the client the cluster's watches already run -// on, with the ClusterProvider's identity, so anything asked through it (the applier's apply -// requests) is asked as that identity and against that cluster, never the operator's own. +// ApplierDynamicClient returns the client an applier in clusterID is asked through, with that +// ClusterProvider's identity, so anything asked through it is asked as that identity and against +// that cluster, never the operator's own. A cluster some GitTarget mirrors from is asked through +// the client its watches already run on. // -// It serves only a cluster some GitTarget has declared. clusterDynamicClient creates a context for -// any id on first use, and only forgetGitTargetCluster tears one down, when the last GitTarget that -// declared it goes. A context created for an id no GitTarget declared would therefore live for the -// process: a resolved credential and a client set nobody ever releases. An undeclared id is also -// the moment a caller has run ahead of the GitTarget controller (startup, or a target created a -// moment ago), and the error says so; its caller retries, by which time the Declare has landed. +// An applier may also live in a cluster no GitTarget mirrors from, such as a management cluster +// that runs Flux for remote ones. No context is created for such a cluster: clusterDynamicClient +// creates one for any id on first use, and only forgetGitTargetCluster tears one down, when the last +// GitTarget that declared it goes, so a context for an id no GitTarget declared would live for the +// process. The client is built for this call from the provider as it resolves now, and dropped +// with it. // -// The empty id is the config plane, which is not a source cluster and is refused rather than -// silently served as one. -func (m *Manager) SourceDynamicClient(ctx context.Context, clusterID string) (dynamic.Interface, error) { - if clusterID == configPlaneClusterID { - return nil, errors.New("no source cluster named: the config plane is not a source cluster") - } - if !m.sourceClusterDeclared(clusterID) { - return nil, fmt.Errorf("source cluster %q is not declared by any GitTarget yet", clusterID) - } - return m.clusterDynamicClient(ctx, clusterID) -} - -// ApplierDynamicClient returns the client an applier in clusterID is asked through, with that -// ClusterProvider's identity, as SourceDynamicClient's is. An applier may live in a cluster no -// GitTarget mirrors from, such as a management cluster that runs Flux for remote ones. No context -// is created for such a cluster, because nothing would release it (see SourceDynamicClient): the -// client is built for this call from the provider as it resolves now, and dropped with it. +// The empty id is the config plane, which is not a ClusterProvider, and is refused. func (m *Manager) ApplierDynamicClient(ctx context.Context, clusterID string) (dynamic.Interface, error) { if clusterID == configPlaneClusterID { return nil, errors.New("no cluster named: the config plane is not a ClusterProvider") diff --git a/internal/watch/event_router.go b/internal/watch/event_router.go index 8e65bfb08..33549f6f5 100644 --- a/internal/watch/event_router.go +++ b/internal/watch/event_router.go @@ -274,6 +274,7 @@ func (r *EventRouter) enqueueScopedResync( desired []manifestanalyzer.DesiredResource, resourceVersion string, heal bool, + renderFidelityRevision uint64, ) (chan git.ResyncResult, bool, error) { worker, err := r.resyncTarget(ctx, gitDest) if err != nil { @@ -288,7 +289,10 @@ func (r *EventRouter) enqueueScopedResync( Scope: &scope, SourceCollection: sourceCollection, Heal: heal, - Result: resultCh, + // The worker records the scope clean itself once the resync applies, before the drain + // below hears of it, so a live event queued behind the resync meets the gate it produced. + RenderFidelityRevision: renderFidelityRevision, + Result: resultCh, }) return resultCh, enqueued, nil } diff --git a/internal/watch/event_router_test.go b/internal/watch/event_router_test.go index 5b500447e..1cdd04180 100644 --- a/internal/watch/event_router_test.go +++ b/internal/watch/event_router_test.go @@ -107,6 +107,7 @@ func TestEnqueueScopedResync_ReportsMissingWorker(t *testing.T) { nil, "12", false, + 0, ) require.Error(t, err) @@ -140,6 +141,7 @@ func TestEnqueueScopedResync_ReportsGoneGitTargetAsTerminal(t *testing.T) { nil, "12", false, + 0, ) require.Error(t, err) @@ -515,7 +517,7 @@ func TestWithdrawCommitRequest_ADeletedGitTargetsWorkerStillAnswers(t *testing.T // // enqueueReplayResync used to return on !enqueued BEFORE starting the drain, leaving that reply in // a buffered channel nobody read. With it went the only calls that mark acceptance, render -// fidelity and retention for the collection, so the render-fidelity scope owed a report under a revision +// fidelity and retention for the collection, so the render-fidelity scope waited for a report at a revision // no running stream would ever report again — which pins the GitTarget at Ready=False and, through // GitTargetReady, every WatchRule pointing at it. // diff --git a/internal/watch/git_path_acceptance_test.go b/internal/watch/git_path_acceptance_test.go index 8e0b5c6b3..6dd22e5b0 100644 --- a/internal/watch/git_path_acceptance_test.go +++ b/internal/watch/git_path_acceptance_test.go @@ -242,7 +242,7 @@ func requireGitPathEvent(t *testing.T, events <-chan event.GenericEvent) { // TestMarkRenderFidelityScopeClean_NamesAResultTheGateWouldNotTake covers the branch that made // Failure A undiagnosable: the gate answers applied=false for a stale revision, an unknown scope // or an unknown target, and the caller used to discard that answer without a word — so a scope -// could owe a report for ever with nothing anywhere saying why. +// could wait for a report for ever with nothing anywhere saying why. func TestMarkRenderFidelityScopeClean_NamesAResultTheGateWouldNotTake(t *testing.T) { workerManager := git.NewWorkerManager( nil, @@ -271,7 +271,7 @@ func TestMarkRenderFidelityScopeClean_NamesAResultTheGateWouldNotTake(t *testing // TestMarkRenderFidelityScopeClean_NamesAReportWithNoRevision covers the earlier return. A wired // gate always issues a non-zero revision, so a stream reporting under zero was started without -// one; its result is unusable and its scope keeps owing a report. +// one; its result is unusable and its scope stays waiting for a report. func TestMarkRenderFidelityScopeClean_NamesAReportWithNoRevision(t *testing.T) { workerManager := git.NewWorkerManager( nil, diff --git a/internal/watch/last_field_managers_test.go b/internal/watch/last_field_managers_test.go index 3211f017e..640f283d1 100644 --- a/internal/watch/last_field_managers_test.go +++ b/internal/watch/last_field_managers_test.go @@ -16,7 +16,7 @@ import ( // TestLastFieldManagers pins who counts as the last writer: the latest entry, status left out so a // controller's status write cannot hide the apply before it, scale kept because it writes replicas, -// and every manager of a same-second tie. +// and every manager of a same-second tie, with the time of those entries. func TestLastFieldManagers(t *testing.T) { at := func(seconds int) *metav1.Time { return &metav1.Time{Time: time.Date(2026, 10, 17, 12, 0, seconds, 0, time.UTC)} @@ -28,23 +28,24 @@ func TestLastFieldManagers(t *testing.T) { name string entries []metav1.ManagedFieldsEntry want []string + wantAt *metav1.Time }{ - {name: "none", want: nil}, + {name: "none", want: nil, wantAt: &metav1.Time{}}, {name: "the latest entry", entries: []metav1.ManagedFieldsEntry{ entry("kubectl-edit", "", at(1)), entry("kustomize-controller", "", at(2)), - }, want: []string{"kustomize-controller"}}, + }, want: []string{"kustomize-controller"}, wantAt: at(2)}, {name: "a later status write does not hide the apply", entries: []metav1.ManagedFieldsEntry{ entry("kustomize-controller", "", at(2)), entry("kube-controller-manager", "status", at(3)), - }, want: []string{"kustomize-controller"}}, + }, want: []string{"kustomize-controller"}, wantAt: at(2)}, {name: "a later scale write is a write", entries: []metav1.ManagedFieldsEntry{ entry("kustomize-controller", "", at(2)), entry("horizontal-pod-autoscaler", "scale", at(3)), - }, want: []string{"horizontal-pod-autoscaler"}}, + }, want: []string{"horizontal-pod-autoscaler"}, wantAt: at(3)}, {name: "a same-second tie names both", entries: []metav1.ManagedFieldsEntry{ entry("kubectl-edit", "", at(2)), entry("kustomize-controller", "", at(2)), entry("old", "", at(1)), - }, want: []string{"kubectl-edit", "kustomize-controller"}}, + }, want: []string{"kubectl-edit", "kustomize-controller"}, wantAt: at(2)}, {name: "an entry without a time says nothing", entries: []metav1.ManagedFieldsEntry{ entry("kubectl-edit", "", at(1)), entry("before-v1.14", "", nil), - }, want: []string{"kubectl-edit"}}, + }, want: []string{"kubectl-edit"}, wantAt: at(1)}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -53,7 +54,9 @@ func TestLastFieldManagers(t *testing.T) { "metadata": map[string]any{"name": "api", "namespace": "shop"}, }} u.SetManagedFields(tc.entries) - assert.Equal(t, tc.want, lastFieldManagers(u)) + managers, managedAt := lastFieldManagers(u) + assert.Equal(t, tc.want, managers) + assert.True(t, tc.wantAt.Time.Equal(managedAt), "written at %s, not %s", tc.wantAt, managedAt) event := targetWatchGitEvent( schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, @@ -61,6 +64,7 @@ func TestLastFieldManagers(t *testing.T) { string(types.OperationUpdate), ) assert.Equal(t, tc.want, event.LastFieldManagers) + assert.True(t, tc.wantAt.Time.Equal(event.LastFieldManagedAt)) assert.Empty(t, event.Object.GetManagedFields(), "managedFields never reach Git") }) } diff --git a/internal/watch/owner.go b/internal/watch/owner.go index cfb371092..69c4e61ca 100644 --- a/internal/watch/owner.go +++ b/internal/watch/owner.go @@ -99,7 +99,7 @@ type declareIntent struct { // pass has yet put in place. func (d declareIntent) forcePending() bool { return d.forceRequests != d.forceConsumed } -// dirtyTarget is one GitTarget the owner owes a pass. +// dirtyTarget is one GitTarget waiting for a plan pass from the owner. type dirtyTarget struct { ref types.ResourceReference // firstDirty is when this target went from clean to dirty, and is what maxSettleWait bounds. @@ -282,7 +282,7 @@ func (m *Manager) declareIntentFor( } // A GitTarget reconcile is LEVEL-triggered: the controller re-declares on its steady requeue // and on every event it watches, with the same values every time. Marking the target dirty for - // each of those would keep a healthy target permanently owed a pass, which is both a pass per + // each of those would keep a healthy target permanently waiting for a pass, which is both a pass per // reconcile and — because "the declaration has landed" is a readiness input — a target that // never reads converged. So a re-declare that changes nothing and has already landed is a // no-op, and the 30s periodic sweep remains the floor under it. @@ -340,7 +340,7 @@ func (m *Manager) forgetIntent(gitDest types.ResourceReference) { t.signal() } -// DeclareStatusForGitTarget reports whether the owner still owes this GitTarget a pass and how +// DeclareStatusForGitTarget reports whether a plan pass is still pending for this GitTarget and how // the last one ended. A target whose passes keep timing out must not read as idle. func (m *Manager) DeclareStatusForGitTarget(gitDest types.ResourceReference) DeclareStatus { t := m.triggers() @@ -441,7 +441,7 @@ func (m *Manager) ownerTurn(ctx context.Context, log logr.Logger) time.Duration // behind a queue of passes. entry, wait := m.nextReadyTarget() if entry == nil { - // Nothing to plan. Sleep no longer than a backed-off shared refresh is owed, or a failed + // Nothing to plan. Sleep no longer than until a backed-off shared refresh is due, or a failed // one would not be reattempted until something else happened to wake the loop. return minDuration(wait, m.sharedRefreshWait()) } @@ -450,7 +450,7 @@ func (m *Manager) ownerTurn(ctx context.Context, log logr.Logger) time.Duration } // sharedRefreshWait is how long until a backed-off shared refresh may be reattempted, or a long -// sleep when none is owed. +// sleep when none is due. func (m *Manager) sharedRefreshWait() time.Duration { t := m.triggers() t.mu.Lock() diff --git a/internal/watch/owner_test.go b/internal/watch/owner_test.go index c78886baf..9dc2823e4 100644 --- a/internal/watch/owner_test.go +++ b/internal/watch/owner_test.go @@ -317,7 +317,7 @@ func TestOwner_ARedeclareThatChangesNothingIsANoOp(t *testing.T) { m.DeclareForGitTarget(ref, configPlaneClusterID, "route", v1alpha3.PruneOnEvent) - assert.Empty(t, m.triggers().dirty, "an identical re-declare owes the owner nothing") + assert.Empty(t, m.triggers().dirty, "an identical re-declare leaves the owner no pass to run") assert.True(t, m.DeclareStatusForGitTarget(ref).Settled(), "and leaves the target converged rather than perpetually pending") @@ -386,9 +386,9 @@ func TestOwner_AStaleDeleteDoesNotDropTheSuccessorsPendingPass(t *testing.T) { m.settle(first) require.Equal(t, 1, m.runTurns(t, 4)) - // The successor declares, and is owed a pass... + // The successor declares, and waits for a pass... m.DeclareForGitTarget(second, "prod-eu-1", "", v1alpha3.PruneOnEvent) - require.Len(t, m.triggers().dirty, 1, "precondition: the recreation is owed a pass") + require.Len(t, m.triggers().dirty, 1, "precondition: the recreation waits for a pass") // ...and only then does the predecessor's delete arrive. m.ForgetGitTargetDeclaration(first) diff --git a/internal/watch/render_fidelity_gate.go b/internal/watch/render_fidelity_gate.go index 8a5a111e1..d43b72fba 100644 --- a/internal/watch/render_fidelity_gate.go +++ b/internal/watch/render_fidelity_gate.go @@ -69,7 +69,7 @@ func (m *Manager) MarkTargetRenderFidelityScopeClean( } if revision == 0 { // A wired gate always issues a non-zero revision, so a stream reporting under zero was - // started without one. Its result is unusable and its scope keeps owing a report. + // started without one. Its result is unusable and its scope stays waiting for a report. m.logUnappliedFidelityReport(target, collection, revision, "clean (stream carries no revision)") return } @@ -100,7 +100,7 @@ func (m *Manager) MarkTargetRenderFidelityScopeClean( // This is the last silent branch in the roll-up. RecordScope* answers applied=false for three // different reasons — the target is unknown, the scope is not in the current plan, or the result // carries a superseded revision — and every caller used to drop that answer on the floor. A scope -// then owes a report for ever with nothing to say why, which is Failure A's signature. +// then waits for a report for ever with nothing to say why, which is Failure A's signature. // // The retention roll-up already logs its refusals, and that asymmetry is exactly why B had // evidence and A had none. Info, because it is rare by construction: a healthy plan produces one @@ -113,7 +113,7 @@ func (m *Manager) logUnappliedFidelityReport( kind string, ) { m.Log.WithName("render-fidelity").Info( - "a render scope result was not applied; this scope still owes a report and cannot converge alone", + "a render scope result was not applied; this scope is still waiting for a report and cannot converge alone", "gitDest", target.String(), "collection", collection.String(), "reportedRevision", revision, "result", kind, "status", m.RenderFidelityForGitTarget(target).Message) } @@ -131,7 +131,7 @@ func (m *Manager) MarkTargetRenderFidelityScopeDiverged( } if revision == 0 { // A wired gate always issues a non-zero revision, so a stream reporting under zero was - // started without one. Its result is unusable and its scope keeps owing a report. + // started without one. Its result is unusable and its scope stays waiting for a report. m.logUnappliedFidelityReport(target, collection, revision, "diverged (stream carries no revision)") return } diff --git a/internal/watch/source_dynamic_client_test.go b/internal/watch/source_dynamic_client_test.go deleted file mode 100644 index 74c54112f..000000000 --- a/internal/watch/source_dynamic_client_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package watch - -import ( - "context" - "testing" - - "github.com/go-logr/logr" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "k8s.io/client-go/rest" -) - -// TestSourceDynamicClient_RefusesTheConfigPlane: the empty id is the operator's own cluster, and a -// caller asking for a source cluster by an empty name has a bug, not a cluster. -func TestSourceDynamicClient_RefusesTheConfigPlane(t *testing.T) { - m := &Manager{Log: logr.Discard()} - - _, err := m.SourceDynamicClient(context.Background(), configPlaneClusterID) - require.Error(t, err) -} - -// TestSourceDynamicClient_RefusesAnUndeclaredCluster: asking for a cluster no GitTarget declared -// fails, and creates no context, so nothing is left behind that only a GitTarget's deletion could -// ever tear down. -func TestSourceDynamicClient_RefusesAnUndeclaredCluster(t *testing.T) { - m := &Manager{ - Log: logr.Discard(), - SourceClusters: stubSourceClusterResolver{cfg: &rest.Config{Host: "https://prod.example"}}, - } - - _, err := m.SourceDynamicClient(context.Background(), "prod-eu-1") - require.Error(t, err) - assert.Contains(t, err.Error(), "not declared") - assert.Nil(t, m.clusterContextByID("prod-eu-1"), "an undeclared id creates no cluster context") -} - -// TestSourceDynamicClient_ServesADeclaredCluster: once a GitTarget declared the cluster, the client -// is the one its watches use, built once and cached. -func TestSourceDynamicClient_ServesADeclaredCluster(t *testing.T) { - m := &Manager{ - Log: logr.Discard(), - SourceClusters: stubSourceClusterResolver{cfg: &rest.Config{Host: "https://prod.example"}, version: "v1"}, - } - m.rememberGitTargetCluster(gd("checkout"), "prod-eu-1") - - first, err := m.SourceDynamicClient(context.Background(), "prod-eu-1") - require.NoError(t, err) - require.NotNil(t, first) - - second, err := m.SourceDynamicClient(context.Background(), "prod-eu-1") - require.NoError(t, err) - assert.Same(t, first, second, "the cluster's cached client is reused, not rebuilt per call") - - cc := m.clusterContextByID("prod-eu-1") - require.NotNil(t, cc) - assert.Same(t, first, cc.dynamicClient, "it is the client the cluster's watches run on") -} - -// TestApplierDynamicClient_ServesAnUndeclaredClusterWithoutKeepingIt: an applier may live in a -// cluster no GitTarget mirrors from. It is served, from the provider as it resolves now, and no -// cluster context is created, so nothing outlives the call. -func TestApplierDynamicClient_ServesAnUndeclaredClusterWithoutKeepingIt(t *testing.T) { - m := &Manager{ - Log: logr.Discard(), - SourceClusters: stubSourceClusterResolver{cfg: &rest.Config{Host: "https://hub.example"}, version: "v1"}, - } - - dc, err := m.ApplierDynamicClient(context.Background(), "hub") - require.NoError(t, err) - require.NotNil(t, dc) - assert.Nil(t, m.clusterContextByID("hub"), "an applier-only cluster creates no cluster context") -} - -// TestApplierDynamicClient_ReusesADeclaredClustersClient: a cluster some GitTarget mirrors from -// already has a client, with the same identity; the applier is asked through that one. -func TestApplierDynamicClient_ReusesADeclaredClustersClient(t *testing.T) { - m := &Manager{ - Log: logr.Discard(), - SourceClusters: stubSourceClusterResolver{cfg: &rest.Config{Host: "https://prod.example"}, version: "v1"}, - } - m.rememberGitTargetCluster(gd("checkout"), "prod-eu-1") - - source, err := m.SourceDynamicClient(context.Background(), "prod-eu-1") - require.NoError(t, err) - applier, err := m.ApplierDynamicClient(context.Background(), "prod-eu-1") - require.NoError(t, err) - assert.Same(t, source, applier) - - _, err = m.ApplierDynamicClient(context.Background(), configPlaneClusterID) - require.Error(t, err, "the config plane is not a ClusterProvider") -} diff --git a/internal/watch/source_namespace_test.go b/internal/watch/source_namespace_test.go index 9cd0fb43a..42b1fad4d 100644 --- a/internal/watch/source_namespace_test.go +++ b/internal/watch/source_namespace_test.go @@ -50,8 +50,8 @@ func snbClusterProvider(delegate bool) *configv1alpha3.ClusterProvider { return &configv1alpha3.ClusterProvider{ ObjectMeta: metav1.ObjectMeta{Name: snbProvider}, Spec: configv1alpha3.ClusterProviderSpec{ - AccessFrom: &configv1alpha3.NamespaceMatcher{Names: []string{snbTenantNS}}, - AllowAnySourceNamespace: delegate, + AccessFrom: &configv1alpha3.NamespaceMatcher{Names: []string{snbTenantNS}}, + AllowCrossNamespace: &configv1alpha3.ClusterProviderCrossNamespace{Sources: delegate}, }, } } @@ -184,7 +184,7 @@ func TestCompileWatchRule_WildcardIsRefusedWithoutDelegation(t *testing.T) { require.NoError(t, err) assert.False(t, resolved.Admitted()) - assert.Contains(t, resolved.Message, "allowAnySourceNamespace", + assert.Contains(t, resolved.Message, "spec.allowCrossNamespace.sources", "the refusal must name the flag a platform admin has to set") assert.Empty(t, m.RuleStore.SnapshotWatchRules()) } @@ -211,7 +211,7 @@ func TestCompileWatchRule_TerminalRefusalRemovesAnAlreadyCompiledRule(t *testing // The platform admin withdraws the delegation. var stored configv1alpha3.ClusterProvider require.NoError(t, m.Client.Get(ctx, k8stypes.NamespacedName{Name: snbProvider}, &stored)) - stored.Spec.AllowAnySourceNamespace = false + stored.Spec.AllowCrossNamespace = nil require.NoError(t, m.Client.Update(ctx, &stored)) resolved, err = CompileWatchRule(ctx, m.Client, m.RuleStore, rule, target, provider) diff --git a/internal/watch/target_watch.go b/internal/watch/target_watch.go index 679cb4c11..b846e287a 100644 --- a/internal/watch/target_watch.go +++ b/internal/watch/target_watch.go @@ -1038,7 +1038,7 @@ func (m *Manager) enqueueReplayResync( if !stream.gate.enqueue(ctx, func() { resultCh, enqueued, err = m.EventRouter.enqueueScopedResync( ctx, gitDest, resyncScopeForWatchKey(stream.key), stream.sourceCollection(), snapshot.desired, - resourceVersion, false) + resourceVersion, false, stream.revision) }) { return nil } @@ -1049,7 +1049,7 @@ func (m *Manager) enqueueReplayResync( // case: EnqueueResync has ALREADY delivered ErrFinalizeQueueFull on the reply channel for the // drain to record, so returning here instead left that reply in a buffered channel nobody ever // read -- and with it went the only calls that mark acceptance, render fidelity and retention - // for this collection. The render-fidelity scope then owed a report under a revision no running + // for this collection. The render-fidelity scope then waited for a report at a revision no running // stream would ever report again, which pins the GitTarget at Ready=False and, through it, // every WatchRule pointing at it. // @@ -1329,18 +1329,18 @@ func targetWatchGitEvent(gvr schema.GroupVersionResource, u *unstructured.Unstru Generation: u.GetGeneration(), } if op != string(types.OperationDelete) { - event.LastFieldManagers = lastFieldManagers(u) + event.LastFieldManagers, event.LastFieldManagedAt = lastFieldManagers(u) event.Object = sanitize.Sanitize(u) } return event } // lastFieldManagers returns the managers whose managedFields entries carry the latest time, in -// entry order: who wrote the object last, or several when the second-resolution times tie. Status +// entry order, and that time: who wrote the object last, and when, or several when the second-resolution times tie. Status // entries are left out, because a controller writing status after an apply would otherwise hide who // made the apply, and status is never mirrored. Other subresources stay: a write to scale is a // write to spec.replicas. Entries without a time are left out too. -func lastFieldManagers(u *unstructured.Unstructured) []string { +func lastFieldManagers(u *unstructured.Unstructured) ([]string, time.Time) { var latest metav1.Time var managers []string for _, entry := range u.GetManagedFields() { @@ -1354,7 +1354,7 @@ func lastFieldManagers(u *unstructured.Unstructured) []string { managers = append(managers, entry.Manager) } } - return managers + return managers, latest.Time } func operationForWatchEvent(eventType watch.EventType) string { diff --git a/internal/watch/watch_plane_state.go b/internal/watch/watch_plane_state.go index 1fa03b8ff..4b2a34f02 100644 --- a/internal/watch/watch_plane_state.go +++ b/internal/watch/watch_plane_state.go @@ -66,8 +66,8 @@ type targetPassStatus struct { Landed bool } -// DeclareStatus is the observable state of a GitTarget's declaration: whether the owner still -// owes it a plan pass, and how the last one ended. It is what makes a target whose passes keep +// DeclareStatus is the observable state of a GitTarget's declaration: whether a plan pass is still +// pending for it, and how the last one ended. It is what makes a target whose passes keep // timing out read as failing rather than as idle. type DeclareStatus struct { // Declared reports whether the GitTarget has ever reached DeclareForGitTarget. It is the @@ -78,7 +78,7 @@ type DeclareStatus struct { // ClusterID is the source cluster the declaration named — a ClusterProvider name, empty for // the cluster the operator runs in. Meaningful only when Declared. ClusterID string - // Pending reports whether a plan pass is owed — the target is dirty, or has never + // Pending reports whether a plan pass is pending — the target is dirty, or has never // successfully applied a plan. Pending bool // Failures counts consecutive failed passes; LastError carries the most recent message. @@ -86,7 +86,7 @@ type DeclareStatus struct { LastError string } -// Settled reports whether the target's declaration has landed: it declared, nothing is owed, +// Settled reports whether the target's declaration has landed: it declared, no pass is pending, // and the last pass succeeded. func (s DeclareStatus) Settled() bool { return s.Declared && !s.Pending && s.Failures == 0 diff --git a/internal/webhook/audit_fact_publish_test.go b/internal/webhook/audit_fact_publish_test.go index 703dfbba7..5156330dc 100644 --- a/internal/webhook/audit_fact_publish_test.go +++ b/internal/webhook/audit_fact_publish_test.go @@ -382,7 +382,7 @@ func aggregatedCreateEvent(auditID, user string) string { // objectRef carries no name at all — produces no fact, so nothing is appended for it and no watch // event can ever join it. // -// It used to be counted `queued`, which claimed an append that was never owed and buried the whole +// It used to be counted `queued`, which claimed an append that was never expected and buried the whole // population under the busiest value on the counter. It is Dropped rather than Error because // nothing failed, which also keeps the e2e invariant on category="error" intact. func TestAuditHandler_AnEventThatProducesNoFactIsCountedAsSuch(t *testing.T) { @@ -417,7 +417,7 @@ func TestAuditHandler_AnEventThatProducesNoFactIsCountedAsSuch(t *testing.T) { // The invariant the e2e suite gates on is untouched: nothing here is an error. _, anyError := telemetry.CollectInt64Sum(reader, auditEventsMetric, map[string]string{"category": "error"}) - assert.False(t, anyError, "an event that owes no append has not failed") + assert.False(t, anyError, "an event that expects no append has not failed") } // TestAuditHandler_RecordsTheRouteOfEveryAppendedBatch pins the publish-side signal the diff --git a/internal/webhook/audit_handler.go b/internal/webhook/audit_handler.go index 74cdcb72a..e364bb159 100644 --- a/internal/webhook/audit_handler.go +++ b/internal/webhook/audit_handler.go @@ -338,7 +338,7 @@ func (h *AuditHandler) processEvents(ctx context.Context, route auditRoute, even // deciding each from whether it produced a fact at all and then from whether its OWN stream // appended. // -// An event that produced no fact is not a failure — nothing was owed for it — but it is not queued +// An event that produced no fact is not a failure — nothing was expected for it — but it is not queued // either. It used to be counted queued, which claimed an append that never happened and hid the // whole population that can never be attributed behind the busiest value on the counter. This is // the only place it can be counted: no fact exists, so no fact-side counter can ever see it. diff --git a/test/e2e/Taskfile-e2e.yml b/test/e2e/Taskfile-e2e.yml index 0b13a8d29..d44e97f35 100644 --- a/test/e2e/Taskfile-e2e.yml +++ b/test/e2e/Taskfile-e2e.yml @@ -1319,11 +1319,13 @@ tasks: cmds: - mkdir -p "{{.CS}}" - '{{.KUSTOMIZE}} build config/crd | {{.KUBECTL}} --context "{{.CTX}}" apply -f -' + # Not a bare `kubectl wait`: on a CRD created by the apply above it can fail at once. - | - {{.KUBECTL}} --context "{{.CTX}}" wait --for=condition=Established --timeout=60s \ + CTX="{{.CTX}}" KUBECTL="{{.KUBECTL}}" bash hack/e2e/wait-crds-established.sh 60 \ crd/clusterproviders.configbutler.ai crd/clusterwatchrules.configbutler.ai \ crd/commitrequests.configbutler.ai crd/gitproviders.configbutler.ai \ - crd/gittargets.configbutler.ai crd/watchrules.configbutler.ai + crd/gittargets.configbutler.ai crd/watchrules.configbutler.ai \ + crd/fluxtriggers.configbutler.ai crd/argocdtriggers.configbutler.ai - touch "{{.CS}}/crds.installed" _install-config-dir: @@ -1545,11 +1547,10 @@ tasks: done kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/voter-gitops/crds/coffeeconfigs.yaml" kubectl --context "{{.CTX}}" apply -f "{{.DEMO_MANIFESTS_DIR}}/vote/crds" - kubectl --context "{{.CTX}}" wait --for=condition=Established \ + CTX="{{.CTX}}" bash hack/e2e/wait-crds-established.sh 180 \ crd/coffeeconfigs.examples.configbutler.ai \ crd/quizsessions.examples.configbutler.ai \ - crd/quizsubmissions.examples.configbutler.ai \ - --timeout=180s + crd/quizsubmissions.examples.configbutler.ai # Traefik and kro live in the demo path (not the core bring-up): the # demo podinfo flows below need kro's ResourceGraphDefinition CRD and # traefik's IngressRoutes. Install them here and wait before applying diff --git a/test/e2e/apply_trigger.go b/test/e2e/apply_trigger.go new file mode 100644 index 000000000..0a0780e00 --- /dev/null +++ b/test/e2e/apply_trigger.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The apply trigger kinds as kubectl names them. +const ( + fluxTriggerResource = "fluxtrigger" + argoCDTriggerResource = "argocdtrigger" +) + +// createFluxTrigger creates the FluxTrigger name in namespace, holding the Kustomization +// flux-system/ in the operator's own cluster (the "default" ClusterProvider). An +// empty quietPeriod takes the default. +func createFluxTrigger(namespace, name, kustomization, quietPeriod string) { + GinkgoHelper() + quiet := "" + if quietPeriod != "" { + quiet = fmt.Sprintf("\n quietPeriod: %q", quietPeriod) + } + manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: FluxTrigger +metadata: + name: %s + namespace: %s +spec: + clusterProviderRef: + name: default + kustomizationRef: + namespace: flux-system + name: %s%s +`, name, namespace, kustomization, quiet) + _, err := kubectlRunWithStdin(namespace, manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to create FluxTrigger %s/%s", namespace, name) +} + +// createArgoCDTrigger creates the ArgoCDTrigger name in namespace, holding the Application +// / in the operator's own cluster. +func createArgoCDTrigger(namespace, name, appNamespace, application string) { + GinkgoHelper() + manifest := fmt.Sprintf(`apiVersion: configbutler.ai/v1alpha3 +kind: ArgoCDTrigger +metadata: + name: %s + namespace: %s +spec: + clusterProviderRef: + name: default + applicationRef: + namespace: %s + name: %s +`, name, namespace, appNamespace, application) + _, err := kubectlRunWithStdin(namespace, manifest, "apply", "-f", "-") + Expect(err).NotTo(HaveOccurred(), "failed to create ArgoCDTrigger %s/%s", namespace, name) +} + +// allowCrossNamespaceAppliers lets the triggers that reach their applier through the default +// ClusterProvider name one in another namespace, as the e2e appliers are: in flux-system and +// argocd, while each spec's triggers live in its own namespace. Granting it is idempotent, and no +// spec relies on its absence on the default provider, so parallel specs may all ask for it. +func allowCrossNamespaceAppliers() { + GinkgoHelper() + _, err := kubectlRun("patch", "clusterprovider", "default", "--type=merge", "-p", + `{"spec":{"allowCrossNamespace":{"appliers":true}}}`) + Expect(err).NotTo(HaveOccurred(), "failed to let the default ClusterProvider's triggers reach other namespaces") +} + +// nameApplyTrigger points a GitTarget at the trigger of kind (FluxTrigger or ArgoCDTrigger) and +// name, and sets onRefusal when it is not empty. +func nameApplyTrigger(namespace, gitTarget, kind, trigger, onRefusal string) { + GinkgoHelper() + refusal := "" + if onRefusal != "" { + refusal = fmt.Sprintf(`,"onRefusal":%q`, onRefusal) + } + _, err := kubectlRunInNamespace(namespace, "patch", "gittarget", gitTarget, "--type=merge", "-p", + fmt.Sprintf(`{"spec":{"applyTriggerRef":{"kind":%q,"name":%q}%s}}`, kind, trigger, refusal)) + Expect(err).NotTo(HaveOccurred(), "failed to point GitTarget %s/%s at %s %s", namespace, gitTarget, kind, trigger) +} + +// requestTriggerReconcile asks a trigger's reconciler to read its applier now. It reads the +// applier on its own cadence and watches nothing in the applier's cluster, so a test that changes +// the applier asks for the read rather than wait for it. +func requestTriggerReconcile(namespace, resource, name string) { + GinkgoHelper() + _, err := kubectlRunInNamespace(namespace, "annotate", resource, name, "--overwrite", + "reconcile.configbutler.ai/requestedAt="+strconv.FormatInt(time.Now().UnixNano(), 10)) + Expect(err).NotTo(HaveOccurred(), "failed to request a %s reconcile", resource) +} + +// triggerOperation is a trigger's status.operation: its reason, commit, phase and token, as the +// branch worker writes them on every transition. +type triggerOperation struct { + Reason, Commit, Phase, Token string +} + +// readTriggerOperation reads a trigger's status.operation; the zero value when it has none. +func readTriggerOperation(g Gomega, namespace, resource, name string) triggerOperation { + out, err := kubectlRunInNamespace(namespace, "get", resource, name, "-o", + "jsonpath={.status.operation.reason}|{.status.operation.commit}|{.status.operation.phase}|"+ + "{.status.operation.token}") + g.Expect(err).NotTo(HaveOccurred()) + const fields = 4 + parts := strings.SplitN(strings.TrimSpace(out), "|", fields) + for len(parts) < fields { + parts = append(parts, "") + } + return triggerOperation{Reason: parts[0], Commit: parts[1], Phase: parts[2], Token: parts[3]} +} + +// fluxSourceStamp is the reconcile.fluxcd.io/requestedAt of a GitRepository in flux-system: set +// only when something asked it to fetch. +func fluxSourceStamp(name string) string { + GinkgoHelper() + out, err := kubectlRunInNamespace("flux-system", "get", "gitrepository", name, "-o", + `jsonpath={.metadata.annotations.reconcile\.fluxcd\.io/requestedAt}`) + Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) +} + +// kustomizationSuspended reads a flux-system Kustomization's spec.suspend. +func kustomizationSuspended(g Gomega, name string) bool { + out, err := kubectlRunInNamespace("flux-system", "get", "kustomization", name, "-o", "jsonpath={.spec.suspend}") + g.Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) == "true" +} + +// kustomizationManagers lists the field managers of a flux-system Kustomization's managedFields. +func kustomizationManagers(g Gomega, name string) string { + out, err := kubectlRunInNamespace("flux-system", "get", "kustomization", name, "-o", + "jsonpath={.metadata.managedFields[*].manager}", "--show-managed-fields") + g.Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(out) +} + +// fluxTriggerEvents is every message of the Events with reason recorded on the FluxTrigger name. +func fluxTriggerEvents(g Gomega, namespace, name, reason string) string { + out, err := kubectlRunInNamespace(namespace, "get", "events", "--field-selector", + "involvedObject.kind=FluxTrigger,involvedObject.name="+name+",reason="+reason, + "-o", "jsonpath={.items[*].message}") + g.Expect(err).NotTo(HaveOccurred()) + return out +} diff --git a/test/e2e/argocd_bi_directional_e2e_test.go b/test/e2e/argocd_bi_directional_e2e_test.go index 5e1447206..1fb45dc9b 100644 --- a/test/e2e/argocd_bi_directional_e2e_test.go +++ b/test/e2e/argocd_bi_directional_e2e_test.go @@ -21,8 +21,8 @@ import ( // Argo CD is only installed for this corner, by the `_argocd-installed` Taskfile // node. `task test-e2e` skips these specs; `task test-e2e-bi-directional` runs them. // -// Every spec runs the standard (docs/argocd.md): the Application has automated sync -// off and is named by the GitTarget's spec.applier.applicationRef, so every sync it +// Every spec runs the standard (docs/bi-directional-argocd.md): the Application has automated sync +// off and is held by the ArgoCDTrigger the GitTarget names, so every sync it // runs is one GitOps Reverser started, pinned to a commit it checked. Argo CD's own // Gitea webhook stays wired, and is shown to be harmless: with automated sync off it // only refreshes. The last spec turns automated sync on and proves GitOps Reverser @@ -109,6 +109,7 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), BeforeAll(func() { skipUnlessBiDirectionalEnabled() requireArgoCDInstalled() + allowCrossNamespaceAppliers() By("creating test namespace") testNs = testNamespaceFor("argo-bi-directional") @@ -198,13 +199,13 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), By("GitOps Reverser asks Argo CD to sync exactly that commit, as a ForeignCommit") run.waitForReverserSync(head, "ForeignCommit") run.waitForOrderContainer(run.syncedOrderName, "Cone") - // status.applier is published on the GitTarget's reconcile, which a delivered - // request does not trigger; a converged target reconciles every few minutes. - run.requestGitTargetReconcile() + // The branch worker writes the operation on the trigger at every transition, so nothing + // has to reconcile for it to show. Eventually(func(g Gomega) { - reason, commit := run.gitTargetApplierStatus(g) - g.Expect(reason).To(Equal("ForeignCommit")) - g.Expect(commit).To(Equal(head)) + operation := run.triggerOperation(g) + g.Expect(operation.Reason).To(Equal("ForeignCommit")) + g.Expect(operation.Commit).To(Equal(head)) + g.Expect(operation.Phase).To(Equal("Succeeded")) }, biEventuallyTimeout, biPollInterval).Should(Succeed()) By("verifying Argo CD stamped its bookkeeping onto the live object") @@ -264,7 +265,7 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), g.Expect(count).To(Equal(baseline+2), "a sync of the first edit's commit over the second edit would show as more commits") g.Expect(run.operationFinishedAt()).To(Equal(lastSync), - "GitOps Reverser's own commits owe Argo CD no sync: there is no catch-up") + "GitOps Reverser's own commits ask Argo CD for no sync: there is no catch-up") }, 30*time.Second, 2*time.Second).Should(Succeed()) By("and Argo CD, which only compared, finds Git and the cluster in agreement") @@ -282,15 +283,17 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), automated := run.baseAppConfig() automated.Automated = true run.applyArgoApp(automated) - // The applier is read on the GitTarget's reconcile, not watched; a converged - // target reconciles every few minutes, so ask for one now. - run.requestGitTargetReconcile() - verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplierConfigured", "False", + // The applier is read on the trigger's reconcile, not watched; a Ready trigger + // reconciles every few minutes, so ask for one now. + requestTriggerReconcile(testNs, argoCDTriggerResource, run.appName) + verifyResourceCondition(argoCDTriggerResource, run.appName, testNs, "Configured", "False", + "AutomatedSyncEnabled", "has automated sync on") + verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplyTriggerReady", "False", "AutomatedSyncEnabled", "has automated sync on") verifyResourceCondition("gittarget", run.gitTargetName, testNs, "Stalled", "True", "AutomatedSyncEnabled", "") - _, requestedBefore := run.gitTargetApplierStatus(Default) + requestedBefore := run.triggerOperation(Default) By("pushing a Git-side change, which GitOps Reverser hears of and refuses to sync") run.writeLiveOrder(iceCreamOrderFile{ @@ -317,8 +320,7 @@ var _ = Describe("Bi Directional (Argo CD)", Label("bi-directional", "argocd"), g.Expect(revision).NotTo(Equal(head), "GitOps Reverser must not sync an Application that syncs on its own") } - _, requested := run.gitTargetApplierStatus(g) - g.Expect(requested).To(Equal(requestedBefore), "no request was delivered") + g.Expect(run.triggerOperation(g)).To(Equal(requestedBefore), "no request was delivered") }, argoNoSyncWindow, biPollInterval).Should(Succeed()) }) }) @@ -340,6 +342,7 @@ var _ = Describe("Bi Directional (Argo CD) refusal", Label("bi-directional", "ar BeforeAll(func() { skipUnlessBiDirectionalEnabled() requireArgoCDInstalled() + allowCrossNamespaceAppliers() testNs = testNamespaceFor("argo-refusal") _, _ = kubectlRun("create", "namespace", testNs) @@ -392,11 +395,9 @@ var _ = Describe("Bi Directional (Argo CD) refusal", Label("bi-directional", "ar By("creating the GitTarget on the overlay, naming the Application and asking it to reapply") createGitTarget(run.gitTargetName, testNs, run.gitProviderName, overlayPath, "main") - _, err = kubectlRunInNamespace(testNs, "patch", "gittarget", run.gitTargetName, "--type=merge", "-p", - fmt.Sprintf(`{"spec":{"applier":{"applicationRef":{"namespace":%q,"name":%q}},"onRefusal":"Reapply"}}`, - run.argoNs, run.appName)) - Expect(err).NotTo(HaveOccurred(), "failed to set spec.applier and spec.onRefusal") - verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplierConfigured", "True", "Succeeded", "") + createArgoCDTrigger(testNs, run.appName, run.argoNs, run.appName) + nameApplyTrigger(testNs, run.gitTargetName, "ArgoCDTrigger", run.appName, "Reapply") + verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") Expect(applyFromTemplate("test/e2e/templates/manager/watchrule-resources.tmpl", struct { Name string Namespace string @@ -501,10 +502,9 @@ func (r argoBiDirectionalRun) startReverserPipeline() int64 { hookID := registerOperatorReceiver(r.testNs, r.gitProviderName, r.receiverSecretName, r.repoName) createGitTarget(r.gitTargetName, r.testNs, r.gitProviderName, r.livePath, "main") - _, err := kubectlRunInNamespace(r.testNs, "patch", "gittarget", r.gitTargetName, "--type=merge", "-p", - fmt.Sprintf(`{"spec":{"applier":{"applicationRef":{"namespace":%q,"name":%q}}}}`, r.argoNs, r.appName)) - Expect(err).NotTo(HaveOccurred(), "failed to set spec.applier") - verifyResourceCondition("gittarget", r.gitTargetName, r.testNs, "ApplierConfigured", "True", "Succeeded", "") + createArgoCDTrigger(r.testNs, r.appName, r.argoNs, r.appName) + nameApplyTrigger(r.testNs, r.gitTargetName, "ArgoCDTrigger", r.appName, "") + verifyResourceCondition("gittarget", r.gitTargetName, r.testNs, "ApplyTriggerReady", "True", "Succeeded", "") Expect(applyFromTemplate("test/e2e/templates/watchrule-crd.tmpl", struct { Name string @@ -689,24 +689,10 @@ func (r argoBiDirectionalRun) lastOperation(g Gomega) (string, string) { return username, revision } -// requestGitTargetReconcile asks the GitTarget controller to reconcile now, the way -// its reconcile-request annotation does: the applier check and status.applier are -// published on a reconcile, and nothing about an applier triggers one. -func (r argoBiDirectionalRun) requestGitTargetReconcile() { - GinkgoHelper() - _, err := kubectlRunInNamespace(r.testNs, "annotate", "gittarget", r.gitTargetName, "--overwrite", - "reconcile.configbutler.ai/requestedAt="+strconv.FormatInt(time.Now().UnixNano(), 10)) - Expect(err).NotTo(HaveOccurred(), "failed to request a GitTarget reconcile") -} - -// gitTargetApplierStatus is the GitTarget's status.applier: the reason and the -// commit of the last request delivered. -func (r argoBiDirectionalRun) gitTargetApplierStatus(g Gomega) (string, string) { - out, err := kubectlRunInNamespace(r.testNs, "get", "gittarget", r.gitTargetName, - "-o", "jsonpath={.status.applier.reason}|{.status.applier.requestedCommit}") - g.Expect(err).NotTo(HaveOccurred()) - reason, commit, _ := strings.Cut(strings.TrimSpace(out), "|") - return reason, commit +// triggerOperation is the ArgoCDTrigger's status.operation: the sync GitOps Reverser +// last started on the Application. +func (r argoBiDirectionalRun) triggerOperation(g Gomega) triggerOperation { + return readTriggerOperation(g, r.testNs, argoCDTriggerResource, r.appName) } // operationFinishedAt returns "" when the Application has never run an operation. diff --git a/test/e2e/audit_route_attribution_e2e_test.go b/test/e2e/audit_route_attribution_e2e_test.go index 7c90fad95..669114858 100644 --- a/test/e2e/audit_route_attribution_e2e_test.go +++ b/test/e2e/audit_route_attribution_e2e_test.go @@ -226,7 +226,8 @@ metadata: spec: accessFrom: names: [%s, %s] - allowAnySourceNamespace: %t + allowCrossNamespace: + sources: %t attribution: auditRoute: %s `, name, allowedNS, extraNS, delegate, auditRoute) @@ -250,7 +251,7 @@ spec: // applyGitTargetForClusterProvider applies a GitTarget that mirrors through a named // ClusterProvider. Which SOURCE namespaces may reach it is bounded by that provider's -// spec.allowAnySourceNamespace plus the source credential's own RBAC, not by any field here. +// spec.allowCrossNamespace.sources plus the source credential's own RBAC, not by any field here. func applyGitTargetForClusterProvider( ns, name, gitProvider, targetPath, clusterProvider string, ) (string, error) { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 96cf880b5..7e4526bf7 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -454,13 +454,14 @@ func prepareE2EClusterOnce() { fmt.Sprintf("NAMESPACE=%s", ns), "prepare-e2e", ) - output, err := utils.Run(cmd) + // Streamed: prepare is the longest command of the run, and the one a suite timeout has killed + // with nothing to show for it (CI run 39246816224 sat 28 minutes in it). + _, err = utils.RunStreaming(cmd) Expect(err).NotTo(HaveOccurred(), "failed to run task target for e2e prepare") - _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) // Some e2e shell scripts call `kubectl` without an explicit `--context` flag. Ensure `kubectl` is // pointed at the intended cluster context for the remainder of the test run. - output, err = kubectlRun("config", "use-context", ctx) + output, err := kubectlRun("config", "use-context", ctx) Expect(err).NotTo(HaveOccurred(), "failed to set kubectl context for e2e run") _, _ = fmt.Fprintf(GinkgoWriter, "%s", output) diff --git a/test/e2e/flux_applier_e2e_test.go b/test/e2e/flux_applier_e2e_test.go index 38021b607..ec695fb65 100644 --- a/test/e2e/flux_applier_e2e_test.go +++ b/test/e2e/flux_applier_e2e_test.go @@ -15,14 +15,14 @@ import ( . "github.com/onsi/gomega" ) -// GitTarget spec.applier against a real Flux, from what a request is allowed to do to how GitOps +// A FluxTrigger, named by a GitTarget's spec.applyTriggerRef, against a real Flux, from what a request is allowed to do to how GitOps // Reverser learns that it has to make one. See docs/design/apply-trigger.md. // // The Kustomization applies its own folder (fluxApplierFolder) into a namespace the WatchRule does // not watch, so nothing Flux applies is mirrored back and every commit on the branch is either this -// spec's API edit or this spec's Git-side push. Its GitRepository and Kustomization sit on a 24h -// interval and no Flux Receiver is wired to the repository, so any fetch or apply inside the spec -// is one GitOps Reverser asked for. +// spec's API edit or this spec's Git-side push. Once the trigger holds it, the Kustomization is +// suspended, and no Flux Receiver is wired to the repository, so any apply inside the spec is one +// GitOps Reverser started. var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Ordered, func() { const ( providerName = "flux-applier-provider" @@ -39,6 +39,7 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order ) BeforeAll(func() { + allowCrossNamespaceAppliers() testNs = testNamespaceFor("manager-flux-applier") appliedNs = testNs + "-applied" fluxName = "flux-applier-" + testNs @@ -66,6 +67,7 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order deleteRepoHook(repo.RepoName, receiverHookID) cleanupWatchRule(ruleName, testNs) cleanupGitTarget(destName, testNs) + cleanupNamespacedResource(testNs, fluxTriggerResource, fluxName) _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", providerName, "--ignore-not-found=true") cleanupNamespacedResource(testNs, "secret", receiverSecretName) cleanupNamespacedResource("flux-system", "kustomization", fluxName) @@ -75,22 +77,31 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order cleanupNamespace(testNs) }) - // A Kustomization whose source follows another branch is reported, and a request for it never - // stamps that source, however it is triggered; once the source is fixed, Flux reaches the tip. - It("reports the applier from the source cluster, and never stamps a source on another branch", func() { - By("naming a Kustomization that does not exist yet") + // A trigger acts only while an eligible GitTarget references it. One nothing names, or one whose + // Kustomization's source follows another branch than the GitTarget writes, holds nothing, and a + // request for it never stamps that source; once the source is fixed, the trigger adopts the + // Kustomization and Flux reaches the tip. Dropping the reference hands it back, and naming the + // trigger again adopts it anew. + It("holds the Kustomization only while an eligible GitTarget names its trigger", func() { + By("creating the Kustomization, with a GitRepository that follows the release branch") + applyFluxApplier(testNs, repo, fluxName, fluxSecretName, "release", "./"+fluxApplierFolder) + + By("creating a trigger no GitTarget names: it is Ready, and does nothing") createGitTarget(destName, testNs, providerName, gitPath, "main") - _, err := kubectlRunInNamespace(testNs, "patch", "gittarget", destName, "--type=merge", "-p", - fmt.Sprintf(`{"spec":{"applier":{"kustomizationRef":{"namespace":"flux-system","name":%q}}}}`, fluxName)) - Expect(err).NotTo(HaveOccurred(), "failed to set spec.applier") - verifyResourceCondition("gittarget", destName, testNs, "ApplierConfigured", "False", "ApplierNotFound", - "not found in the source cluster") + createFluxTrigger(testNs, fluxName, fluxName, "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Ready", "True", "NotReferenced", "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "False", "NotReferenced", "") + Expect(kustomizationSuspended(Default, fluxName)).To(BeFalse(), "Flux applies it on its own") - By("creating it, with a GitRepository that follows the release branch") - applyFluxApplier(testNs, repo, fluxName, fluxSecretName, "release", "./"+fluxApplierFolder) - verifyResourceCondition("gittarget", destName, testNs, "ApplierConfigured", "False", "SourceMismatch", - `fetches branch "release", not main`) - Expect(fluxRequestStamps(fluxName)).To(Equal("|"), "nothing has asked Flux for anything yet") + By("naming it from a GitTarget that writes main: the trigger is refused, and holds nothing") + nameApplyTrigger(testNs, destName, "FluxTrigger", fluxName, "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Configured", "False", "SourceMismatch", + `applies branch "release", not main`) + verifyResourceCondition("gittarget", destName, testNs, "ApplyTriggerReady", "False", "SourceMismatch", + `applies branch "release", not main`) + Expect(kustomizationSuspended(Default, fluxName)).To(BeFalse(), "a trigger that is not bound adopts nothing") + Expect(fluxTriggerEvents(Default, testNs, fluxName, "Adopted")).To(BeEmpty()) + Expect(fluxSourceStamp(fluxName)).To(BeEmpty(), "nothing has asked Flux to fetch yet") By("mirroring a first API edit, so the branch has a tip GitOps Reverser pushed") Expect(applyFromTemplate("test/e2e/templates/manager/watchrule-resources.tmpl", struct { @@ -116,25 +127,52 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order g.Expect(failed).To(ContainSubstring(`fetches branch "release", not main`)) }, 3*time.Minute, 2*time.Second).Should(Succeed()) Consistently(func(g Gomega) { - g.Expect(fluxRequestStamps(fluxName)).To(Equal("|"), + g.Expect(fluxSourceStamp(fluxName)).To(BeEmpty(), "a GitRepository that follows another branch must never be stamped for this one") }, 20*time.Second, 2*time.Second).Should(Succeed()) - // The branch worker keeps the ForeignCommit and decides it again after its backoff. Changing - // the GitRepository's branch also makes Flux fetch on its own, so which of the two brings the - // artifact to the tip is a race: when Flux wins, the worker's retry finds the artifact - // current and rightly stamps nothing. Either way Flux must end at the tip, and the worker's - // half (an obligation kept across a rejected attempt) is proven in apply_attempt_test.go. - By("pointing the GitRepository at main, after which Flux reaches the tip") + // Changing the GitRepository's branch binds the trigger, which adopts the Kustomization. The + // branch worker keeps the ForeignCommit across the rejected attempts and decides it again + // after its backoff; Flux may also apply the new artifact on its own before the adoption. + By("pointing the GitRepository at main: the trigger adopts the Kustomization, and Flux reaches the tip") head := remoteHeadSHA(Default, repo.CheckoutDir) - _, err = kubectlRunInNamespace("flux-system", "patch", "gitrepository", fluxName, "--type=merge", "-p", + _, err := kubectlRunInNamespace("flux-system", "patch", "gitrepository", fluxName, "--type=merge", "-p", `{"spec":{"ref":{"branch":"main"}}}`) Expect(err).NotTo(HaveOccurred(), "failed to point the GitRepository at main") - verifyResourceCondition("gittarget", destName, testNs, "ApplierConfigured", "True", "Succeeded", "") + requestTriggerReconcile(testNs, fluxTriggerResource, fluxName) + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Ready", "True", "Succeeded", "") + verifyResourceCondition("gittarget", destName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") + Expect(fluxTriggerEvents(Default, testNs, fluxName, "Adopted")). + To(ContainSubstring("suspended Kustomization flux-system/" + fluxName)) Eventually(func(g Gomega) { g.Expect(fluxLastAppliedRevision(g, fluxName)).To(ContainSubstring(head)) g.Expect(appliedColor(g, appliedNs)).To(Equal("green")) }, 5*time.Minute, 5*time.Second).Should(Succeed()) + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "True", "Suspended", "") + Expect(kustomizationSuspended(Default, fluxName)).To(BeTrue(), "the trigger holds its Kustomization") + prefetch, err := kubectlRunInNamespace(testNs, "get", fluxTriggerResource, fluxName, + "-o", "jsonpath={.status.source.prefetch}|{.status.source.otherConsumers}") + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(prefetch)).To(Equal("true|"), + "its Kustomization is the GitRepository's only consumer, so the source is prefetched") + + By("dropping the reference: the trigger hands the Kustomization back, and is idle again") + _, err = kubectlRunInNamespace(testNs, "patch", "gittarget", destName, "--type=merge", "-p", + `{"spec":{"applyTriggerRef":null}}`) + Expect(err).NotTo(HaveOccurred(), "failed to drop the GitTarget's applyTriggerRef") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "False", "NotReferenced", "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Ready", "True", "NotReferenced", "") + Eventually(func(g Gomega) { + g.Expect(kustomizationSuspended(g, fluxName)).To(BeFalse()) + g.Expect(kustomizationManagers(g, fluxName)).NotTo(ContainSubstring("gitops-reverser")) + g.Expect(fluxTriggerEvents(g, testNs, fluxName, "Released")). + To(ContainSubstring("handed Kustomization flux-system/" + fluxName + " back to Flux")) + }, time.Minute, 2*time.Second).Should(Succeed()) + + By("naming the trigger again: it adopts the Kustomization anew") + nameApplyTrigger(testNs, destName, "FluxTrigger", fluxName, "") + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "True", "Suspended", "") + Expect(kustomizationSuspended(Default, fluxName)).To(BeTrue()) }) // With no receiver and no API edit, the only thing that can find a Git-side push is the @@ -188,13 +226,78 @@ var _ = Describe("Manager Flux Applier", Label("manager", "flux-applier"), Order "(ForeignCommit)")).To(BeNumerically(">", foreignBefore), "the apply must come from a ForeignCommit request, not from Flux's own schedule") }) + + // The suspended lifecycle's two promises beyond an ordinary apply: a failed apply is suspended + // again, so Flux's own retryInterval retry cannot fire and the worker owns the retry; and a + // Kustomization resumed by hand (`flux resume`) is suspended again, with an Event. + It("holds the Kustomization suspended through a failed apply and a resume by hand", func() { + laterNs := appliedNs + "-later" + DeferCleanup(cleanupNamespace, laterNs) + + By("pushing a commit whose apply fails: a ConfigMap into a namespace that does not exist yet") + pushFileFromOutside(repo, fluxApplierFolder+"/later.yaml", fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: flux-applied-later + namespace: %s +data: + color: teal +`, laterNs), "e2e: another writer adds a ConfigMap in a namespace that does not exist") + + By("the operation fails, and the Kustomization is suspended again after it") + var failed triggerOperation + Eventually(func(g Gomega) { + failed = readTriggerOperation(g, testNs, fluxTriggerResource, fluxName) + g.Expect(failed.Phase).To(Equal("Failed")) + g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue()) + }, 3*time.Minute, 2*time.Second).Should(Succeed()) + + By("once the namespace exists, the worker's own retry applies it") + _, err := kubectlRun("create", "namespace", laterNs) + Expect(err).NotTo(HaveOccurred()) + Eventually(func(g Gomega) { + _, err := kubectlRunInNamespace(laterNs, "get", "configmap", "flux-applied-later") + g.Expect(err).NotTo(HaveOccurred()) + }, 5*time.Minute, 2*time.Second).Should(Succeed()) + Eventually(func(g Gomega) { + done := readTriggerOperation(g, testNs, fluxTriggerResource, fluxName) + g.Expect(done.Phase).To(Equal("Succeeded")) + g.Expect(done.Token).NotTo(Equal(failed.Token), + "applied by an operation the worker decided, not by Flux's retryInterval retry") + g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue()) + }, time.Minute, 2*time.Second).Should(Succeed()) + + By("resuming the Kustomization by hand, as `flux resume` does") + _, err = kubectlRunInNamespace("flux-system", "patch", "kustomization", fluxName, "--type=merge", + "--field-manager=flux", "-p", `{"spec":{"suspend":false}}`) + Expect(err).NotTo(HaveOccurred()) + requestTriggerReconcile(testNs, fluxTriggerResource, fluxName) + Eventually(func(g Gomega) { + g.Expect(fluxTriggerEvents(g, testNs, fluxName, "ResumedElsewhere")). + To(ContainSubstring("was resumed outside GitOps Reverser")) + g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue(), "suspended again") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + verifyResourceCondition(fluxTriggerResource, fluxName, testNs, "Held", "True", "Suspended", "") + }) + + // Deleting the trigger hands the Kustomization back: its finalizer waits until nothing is + // running and releases spec.suspend, so Flux applies on its own again. + It("hands the Kustomization back to Flux when the trigger is deleted", func() { + _, err := kubectlRunInNamespace(testNs, "delete", fluxTriggerResource, fluxName, "--timeout=2m") + Expect(err).NotTo(HaveOccurred(), "the finalizer must let the trigger go") + Eventually(func(g Gomega) { + g.Expect(kustomizationSuspended(g, fluxName)).To(BeFalse()) + g.Expect(kustomizationManagers(g, fluxName)).NotTo(ContainSubstring("gitops-reverser")) + }, time.Minute, 2*time.Second).Should(Succeed()) + }) }) // fluxApplierFolder is the folder the Manager Flux Applier spec's Kustomization applies. const fluxApplierFolder = "flux/applier" // applyFluxApplier creates a GitRepository following branch and a Kustomization applying path from -// it, both on a 24h interval, and waits for the Kustomization's first apply. The Flux specs share it. +// it, both on a 24h interval, and waits for the Kustomization's first apply, made before any trigger +// holds it. The Flux specs share it. func applyFluxApplier(testNs string, repo *RepoArtifacts, name, secretName, branch, path string) { GinkgoHelper() // The in-cluster Gitea test credential, read from the cluster at runtime, base64-encoded as the @@ -224,9 +327,7 @@ func applyFluxApplier(testNs string, repo *RepoArtifacts, name, secretName, bran } // pushAppliedColorFromOutside writes the ConfigMap the Kustomization applies, with the given -// color, and pushes it to main the way another writer would, leaving it checked out, and returns -// the commit it pushed. It retries, because GitOps Reverser may move the tip at any moment and losing that race says nothing -// about the behavior under test. +// color, and pushes it to main the way another writer would (pushFileFromOutside). func pushAppliedColorFromOutside(repo *RepoArtifacts, appliedNs, color string) string { GinkgoHelper() manifest := fmt.Sprintf(`apiVersion: v1 @@ -237,6 +338,15 @@ metadata: data: color: %s `, appliedNs, color) + return pushFileFromOutside(repo, fluxApplierFolder+"/configmap.yaml", manifest, + "e2e: another writer sets the applied color to "+color) +} + +// pushFileFromOutside writes one file and pushes it to main the way another writer would, leaving +// it checked out, and returns the commit it pushed. It retries, because GitOps Reverser may move +// the tip at any moment and losing that race says nothing about the behavior under test. +func pushFileFromOutside(repo *RepoArtifacts, path, content, message string) string { + GinkgoHelper() var head string Eventually(func() error { // The first push seeds an empty repository, which has no main to build on. @@ -249,16 +359,16 @@ data: return fmt.Errorf("git %v: %w: %s", args, err, out) } } - dest := filepath.Join(repo.CheckoutDir, fluxApplierFolder) - if err := os.MkdirAll(dest, 0o750); err != nil { + dest := filepath.Join(repo.CheckoutDir, path) + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { return err } - if err := os.WriteFile(filepath.Join(dest, "configmap.yaml"), []byte(manifest), 0o600); err != nil { + if err := os.WriteFile(dest, []byte(content), 0o600); err != nil { return err } for _, args := range [][]string{ - {"add", fluxApplierFolder}, - {"commit", "-m", "e2e: another writer sets the applied color to " + color}, + {"add", path}, + {"commit", "-m", message}, {"push", "origin", "HEAD:main"}, } { if out, err := gitRun(repo.CheckoutDir, args...); err != nil { diff --git a/test/e2e/flux_bi_directional_e2e_test.go b/test/e2e/flux_bi_directional_e2e_test.go index e9ea16e6f..4b0c71473 100644 --- a/test/e2e/flux_bi_directional_e2e_test.go +++ b/test/e2e/flux_bi_directional_e2e_test.go @@ -77,8 +77,8 @@ var biDirectionalRepo *RepoArtifacts // only place Argo CD is installed, and the two specs share that one cluster. // // Exact commit counts are safe here because Flux is driven as a *manually -// triggered applier* — every reconcile in this spec is explicit. The Argo CD -// spec cannot assert counts the same way; see the note on its selfHeal phase. +// triggered applier* — every reconcile in this spec is explicit, until its last +// phase hands the Kustomization to a FluxTrigger. // // Not Serial: a whole-cluster bidirectional behavioral test (Flux applies from // git while gitops-reverser mirrors live state back) that asserts on *exact* @@ -96,6 +96,7 @@ var _ = Describe("Bi Directional (Flux)", Label("bi-directional", "flux"), Order BeforeAll(func() { skipUnlessBiDirectionalEnabled() + allowCrossNamespaceAppliers() By("creating test namespace") testNs = testNamespaceFor("bi-directional") @@ -427,19 +428,17 @@ var _ = Describe("Bi Directional (Flux)", Label("bi-directional", "flux"), Order // only once the branch has been quiet, so the later value survives and Flux still ends on the // tip, without anybody reconciling it by hand. It("asks Flux to apply only on a quiet branch, so the later of two quick API edits survives", func() { - By("naming the live Kustomization as the GitTarget's applier") - _, err := kubectlRunInNamespace(testNs, "patch", "gittarget", run.gitTargetName, "--type=merge", "-p", - fmt.Sprintf(`{"spec":{"applier":{"kustomizationRef":{"namespace":"flux-system","name":%q},`+ - `"quietPeriod":"5s"}}}`, run.fluxLiveName)) - Expect(err).NotTo(HaveOccurred(), "failed to set spec.applier") - verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplierConfigured", "True", "Succeeded", "") + By("naming a trigger that holds the live Kustomization") + createFluxTrigger(testNs, run.fluxLiveName, run.fluxLiveName, "5s") + nameApplyTrigger(testNs, run.gitTargetName, "FluxTrigger", run.fluxLiveName, "") + verifyResourceCondition("gittarget", run.gitTargetName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") run.waitForGitTargetReady() Expect(run.gitPull()).To(Succeed()) baseline := run.waitForStableRemoteCommitCount(biStableCountMediumWait) By("editing the second order through the API, and again seconds after that edit is pushed") - _, err = kubectlRunInNamespace(testNs, "patch", iceCreamCRDName(crdGroupBiDirectional), + _, err := kubectlRunInNamespace(testNs, "patch", iceCreamCRDName(crdGroupBiDirectional), run.secondOrderName, "--type=merge", "-p", `{"spec":{"container":"Cone"}}`) Expect(err).NotTo(HaveOccurred()) run.waitForCommittedOrderSpec(run.secondOrderName, "Cone", "Strawberry", "WhippedCream") diff --git a/test/e2e/flux_restart_recovery_e2e_test.go b/test/e2e/flux_restart_recovery_e2e_test.go new file mode 100644 index 000000000..b8f9874fa --- /dev/null +++ b/test/e2e/flux_restart_recovery_e2e_test.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Restart recovery for a FluxTrigger (docs/design/apply-trigger.md, "Restart recovery"). The +// operation an apply runs as is recorded on the trigger before the Kustomization is resumed, so a +// manager that restarts while the apply runs follows that same operation to its end, and asks for +// nothing new until it has. The stopping manager suspends the Kustomization it had resumed (a +// graceful stop), so Flux applies nothing on its own while no process follows the apply; the apply +// already running still finishes, and the new process judges it. +// +// The apply is made slow on purpose: the Kustomization waits for its Deployment to be ready, and +// the Deployment's pod is pinned to a node label no node has, so the reconcile that read the +// resume's token keeps running across the restart. The spec then labels a node, the health check +// passes, and that reconcile ends. +// +// Serial: rolls the controller deployment and labels a node, both shared by every concurrent spec. +// See docs/spec/e2e-serial-registry.md. +var _ = Describe("Flux Restart Recovery", Label("flux-restart-recovery"), Serial, Ordered, func() { + const ( + providerName = "flux-recovery-provider" + destName = "flux-recovery-dest" + ruleName = "flux-recovery-rule" + gitPath = "apps/recovery" + folder = "flux/recovery" + ) + var ( + testNs, appliedNs, fluxName, nodeLabel, node string + repo *RepoArtifacts + ) + + BeforeAll(func() { + allowCrossNamespaceAppliers() + testNs = testNamespaceFor("flux-restart-recovery") + appliedNs = testNs + "-applied" + fluxName = "flux-recovery-" + testNs + nodeLabel = "e2e-recovery-" + testNs + for _, ns := range []string{testNs, appliedNs} { + _, _ = kubectlRun("create", "namespace", ns) + } + out, err := kubectlRun("get", "nodes", "-o", "jsonpath={.items[0].metadata.name}") + Expect(err).NotTo(HaveOccurred()) + node = strings.TrimSpace(out) + + By("setting up the Gitea repo, its credentials and the GitProvider") + repo = SetupRepo(resolveE2EContext(), testNs, fmt.Sprintf("e2e-flux-recovery-%d", GinkgoRandomSeed())) + _, err = kubectlRunInNamespace(testNs, "apply", "-f", repo.SecretsYAML) + Expect(err).NotTo(HaveOccurred(), "failed to apply git secrets to test namespace") + applySOPSAgeKeyToNamespace(testNs) + createGitProviderWithURLInNamespace(providerName, testNs, repo.GitSecretHTTP, repo.RepoURLHTTP) + verifyResourceStatus("gitprovider", providerName, testNs, "True", "Succeeded", "") + configureRepoOriginWithCredentials(repo, testNs) + + By("seeding the folder Flux applies with a ConfigMap, and holding its Kustomization") + pushFileFromOutside(repo, folder+"/configmap.yaml", recoveryConfigMap(appliedNs), "e2e: seed the folder") + applyFluxApplier(testNs, repo, fluxName, fluxName+"-git", "main", "./"+folder) + _, err = kubectlRunInNamespace("flux-system", "patch", "kustomization", fluxName, "--type=merge", "-p", + `{"spec":{"wait":true,"timeout":"5m"}}`) + Expect(err).NotTo(HaveOccurred(), "the Kustomization waits for what it applies to be ready") + createGitTarget(destName, testNs, providerName, gitPath, "main") + createFluxTrigger(testNs, fluxName, fluxName, "") + nameApplyTrigger(testNs, destName, "FluxTrigger", fluxName, "") + verifyResourceCondition("gittarget", destName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") + Expect(applyFromTemplate("test/e2e/templates/manager/watchrule-resources.tmpl", struct { + Name, Namespace, DestinationName, Resources string + }{Name: ruleName, Namespace: testNs, DestinationName: destName, Resources: `"configmaps"`}, + testNs)).To(Succeed(), "failed to apply the WatchRule") + }) + + AfterAll(func() { + _, _ = kubectlRun("label", "node", node, nodeLabel+"-") + cleanupWatchRule(ruleName, testNs) + cleanupGitTarget(destName, testNs) + cleanupNamespacedResource(testNs, fluxTriggerResource, fluxName) + _, _ = kubectlRunInNamespace(testNs, "delete", "gitprovider", providerName, "--ignore-not-found=true") + cleanupNamespacedResource("flux-system", "kustomization", fluxName) + cleanupNamespacedResource("flux-system", "gitrepository", fluxName) + cleanupNamespacedResource("flux-system", "secret", fluxName+"-git") + cleanupNamespace(appliedNs) + cleanupNamespace(testNs) + }) + + It("follows the operation an apply runs as across a restart, and suspends the Kustomization after it", func() { + By("pushing a Deployment that cannot be ready yet, then an API edit that finds the push") + pushFileFromOutside(repo, folder+"/deployment.yaml", recoveryDeployment(appliedNs, nodeLabel), + "e2e: another writer adds a Deployment pinned to a node label nobody has yet") + applyConfigMap(testNs, "recovery-edit") + + By("the ForeignCommit's apply starts, and waits for the Deployment") + var before triggerOperation + Eventually(func(g Gomega) { + before = readTriggerOperation(g, testNs, fluxTriggerResource, fluxName) + g.Expect(before.Reason).To(Equal("ForeignCommit")) + g.Expect(before.Phase).To(BeElementOf("Requested", "Running")) + g.Expect(kustomizationSuspended(g, fluxName)).To(BeFalse(), "resumed for the apply") + }, 3*time.Minute, 2*time.Second).Should(Succeed()) + + By("restarting the manager while that apply runs: the stopping manager suspends the Kustomization") + restartControllerDeployment() + Eventually(func(g Gomega) { + g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue(), "a graceful stop suspends what it resumed") + }, 30*time.Second, time.Second).Should(Succeed()) + + By("letting the Deployment become ready, which ends the reconcile that read the resume's token") + _, err := kubectlRun("label", "node", node, nodeLabel+"=true", "--overwrite") + Expect(err).NotTo(HaveOccurred()) + + By("the restarted manager completes that same operation, and the Kustomization stays suspended") + Eventually(func(g Gomega) { + after := readTriggerOperation(g, testNs, fluxTriggerResource, fluxName) + g.Expect(after.Token).To(Equal(before.Token), "the operation recorded before the restart, not a new one") + g.Expect(after.Phase).To(Equal("Succeeded")) + g.Expect(kustomizationSuspended(g, fluxName)).To(BeTrue()) + }, 4*time.Minute, 2*time.Second).Should(Succeed()) + }) +}) + +// recoveryConfigMap is the ConfigMap the restart-recovery Kustomization applies from the start. +func recoveryConfigMap(namespace string) string { + return fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: flux-recovery-seed + namespace: %s +data: + color: blue +`, namespace) +} + +// recoveryDeployment is a Deployment whose pod schedules only on a node carrying nodeLabel, so it is +// not ready until the spec labels one. +func recoveryDeployment(namespace, nodeLabel string) string { + return fmt.Sprintf(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: flux-recovery-wait + namespace: %s +spec: + replicas: 1 + selector: + matchLabels: {app: flux-recovery-wait} + template: + metadata: + labels: {app: flux-recovery-wait} + spec: + nodeSelector: + %s: "true" + containers: + - name: pause + image: registry.k8s.io/pause:3.10 +`, namespace, nodeLabel) +} diff --git a/test/e2e/refusal_reapply_e2e_test.go b/test/e2e/refusal_reapply_e2e_test.go index eebf7052f..65aae2dc2 100644 --- a/test/e2e/refusal_reapply_e2e_test.go +++ b/test/e2e/refusal_reapply_e2e_test.go @@ -29,8 +29,9 @@ import ( // var, which no kustomize declaration in the overlay can express. Read scope reaches the base; // write scope never does. // -// Flux applies the overlay when its Kustomization is created and then waits out a 24h interval, so -// a revert of the drifted env var inside this spec is GitOps Reverser's request and nothing else. +// Flux applies the overlay when its Kustomization is created, and the trigger then holds it +// suspended, so a revert of the drifted env var inside this spec is an apply GitOps Reverser started, +// and nothing else: a resume of the unchanged commit, which is all a Refusal needs. var _ = Describe("Manager Refusal Reapply", Label("manager", "refusal-reapply"), Ordered, func() { var ( testNs string @@ -44,6 +45,7 @@ var _ = Describe("Manager Refusal Reapply", Label("manager", "refusal-reapply"), ) BeforeAll(func() { + allowCrossNamespaceAppliers() By("creating the test namespace") testNs = testNamespaceFor("manager-refusal-reapply") _, _ = kubectlRun("create", "namespace", testNs) @@ -78,7 +80,7 @@ var _ = Describe("Manager Refusal Reapply", Label("manager", "refusal-reapply"), DeferCleanup(func() { _ = os.RemoveAll(seedFolder) }) seedRenderedFolderIntoRepo(repo, testNs, seedFolder, "apps/checkout") - By("letting Flux apply the overlay once, then leaving it on a 24h interval") + By("letting Flux apply the overlay once, before any trigger holds it") applyFluxApplier(testNs, repo, fluxName, fluxSecretName, "main", "./"+gitPath) Eventually(func(g Gomega) { g.Expect(checkoutLogLevel(g, testNs)).To(Equal("info")) @@ -94,11 +96,9 @@ var _ = Describe("Manager Refusal Reapply", Label("manager", "refusal-reapply"), By("creating the GitTarget on the OVERLAY, naming the Kustomization and asking it to reapply") createGitTarget(destName, testNs, providerName, gitPath, "main") - _, err = kubectlRunInNamespace(testNs, "patch", "gittarget", destName, "--type=merge", "-p", - fmt.Sprintf(`{"spec":{"applier":{"kustomizationRef":{"namespace":"flux-system","name":%q}},`+ - `"onRefusal":"Reapply"}}`, fluxName)) - Expect(err).NotTo(HaveOccurred(), "failed to set spec.applier and spec.onRefusal") - verifyResourceCondition("gittarget", destName, testNs, "ApplierConfigured", "True", "Succeeded", "") + createFluxTrigger(testNs, fluxName, fluxName, "") + nameApplyTrigger(testNs, destName, "FluxTrigger", fluxName, "Reapply") + verifyResourceCondition("gittarget", destName, testNs, "ApplyTriggerReady", "True", "Succeeded", "") // The overlay renders into THIS namespace, so the rule watches its own and needs no // sourceNamespace override. ConfigMaps ride along so this target has an ACCEPTED watched @@ -134,12 +134,13 @@ var _ = Describe("Manager Refusal Reapply", Label("manager", "refusal-reapply"), // (the env var changed back), so one more request may follow it. What must never grow is // the count after that: a forced recheck re-observes the standing refusal every minute. By("and it settles instead of asking on every recheck") - seen := map[string]struct{}{fluxRequestStamps(fluxName): {}} + seen := map[string]struct{}{readTriggerOperation(Default, testNs, fluxTriggerResource, fluxName).Token: {}} Consistently(func(g Gomega) { - seen[fluxRequestStamps(fluxName)] = struct{}{} + seen[readTriggerOperation(g, testNs, fluxTriggerResource, fluxName).Token] = struct{}{} g.Expect(len(seen)).To(BeNumerically("<=", 2), "a standing refusal must not be requested once per recheck") }, 150*time.Second, 10*time.Second).Should(Succeed()) + Expect(kustomizationSuspended(Default, fluxName)).To(BeTrue(), "every operation suspends it again") By("a ConfigMap on the same target is mirrored normally") Expect(applyFromTemplate("test/e2e/templates/manager/configmap.tmpl", struct { @@ -183,20 +184,6 @@ func checkoutLogLevel(g Gomega, testNs string) string { return strings.TrimSpace(out) } -// fluxRequestStamps is the reconcile.fluxcd.io/requestedAt annotation of the GitRepository and the -// Kustomization together: one apply request changes one of them. -func fluxRequestStamps(name string) string { - GinkgoHelper() - stamps := make([]string, 0, 2) - for _, kind := range []string{"gitrepository", "kustomization"} { - out, err := kubectlRunInNamespace("flux-system", "get", kind, name, "-o", - `jsonpath={.metadata.annotations.reconcile\.fluxcd\.io/requestedAt}`) - Expect(err).NotTo(HaveOccurred()) - stamps = append(stamps, strings.TrimSpace(out)) - } - return strings.Join(stamps, "|") -} - // writeBaseOwnedFieldFolder builds corpus shape 8 with the overlay pointed at the caller's // namespace, so the rendered object lands where a plain WatchRule already watches. // diff --git a/test/e2e/setup/argocd/README.md b/test/e2e/setup/argocd/README.md index 85d6188d5..d70614aba 100644 --- a/test/e2e/setup/argocd/README.md +++ b/test/e2e/setup/argocd/README.md @@ -49,7 +49,7 @@ ServiceMonitors (it gates them on `monitoring.coreos.com` being present). The specs in `test/e2e/argocd_bi_directional_e2e_test.go` assert Argo CD's *defaults* — the `annotation` resource-tracking method, the exact `tracking-id` value format, client-side apply. A floating version would let an upstream default -change turn a real behavioural regression into a silent pass. +change turn a real behavioral regression into a silent pass. `ARGOCD_CHART_VERSION` lives in [`test/e2e/Taskfile-e2e.yml`](../../Taskfile-e2e.yml) and pins the chart, which carries a specific Argo CD release (its appVersion; e.g. @@ -65,14 +65,15 @@ chart `10.1.3` ships `v3.4.5`). When bumping, re-verify against the upstream sou `:/:/`? 4. `reposerver/repository/repository.go` — does the repo-server still call `SetAppInstance` on every non-CRD target? -5. `cmd/argocd-application-controller/commands/` — is the self-heal backoff still - 2s initial (vs. the 120s reconciliation default)? The `selfHeal` spec depends - on that gap. Also re-check that the **chart** did not start pinning `configs.cm.application.resourceTrackingMethod` or enabling server-side apply by default — `values.yaml` relies on both staying unset. +`values.yaml` also pins the redis image to `mirror.gcr.io`, by digest, because ECR Public +rate-limits the chart's default pull from shared CI runners. On a bump, set its tag and +digest to the chart's new `redis.image.tag`. + If any of those move, the specs fail loudly. That is intended. ## Config diff --git a/test/e2e/setup/argocd/values.yaml b/test/e2e/setup/argocd/values.yaml index fdf7089a1..af8166199 100644 --- a/test/e2e/setup/argocd/values.yaml +++ b/test/e2e/setup/argocd/values.yaml @@ -19,11 +19,10 @@ # real behavioural regression into a silent pass. configs: cm: - # Make Argo's *spontaneous* refresh slow, not fast. Two clocks drive Argo and - # the gap between them is what the specs measure: self-heal reacts to live - # drift sub-second, while a NEW GIT REVISION is only noticed on this timed - # refresh. The specs drive refresh/sync explicitly, so they never wait on this; - # shortening it would make the selfHeal spec flaky. Longer than the 120s default. + # Make Argo's *spontaneous* refresh slow, not fast: a NEW GIT REVISION is only + # noticed on this timed refresh. The specs drive refresh/sync explicitly (an + # ArgoCDTrigger's sync, or the spec itself), so they never wait on it. Longer + # than the 120s default. timeout.reconciliation: 180s params: # Plain-HTTP UI behind the port-forward (task argocd-ui); no TLS gymnastics. @@ -37,6 +36,17 @@ dex: notifications: enabled: false +# The chart pulls redis from ECR Public (ecr-public.aws.com/docker/library/redis), which throttles +# anonymous pulls by source IP. GitHub's shared runner IPs hit that limit often enough that the +# bi-directional leg failed in `_argocd-installed` on 8 CI runs between 2026-09-30 and 2026-10-19, +# every one with argocd-redis in ImagePullBackOff on "429 Too Many Requests". mirror.gcr.io is +# Google's Docker Hub cache and serves the same image: the digest pinned here is the one ECR +# refused (sha256:08ad0b1d…, the chart's 8.2.3-alpine). Re-pin both on an Argo CD chart bump. +redis: + image: + repository: mirror.gcr.io/library/redis + tag: 8.2.3-alpine@sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba + # Prometheus metrics + ServiceMonitors for every running component. The corner's # cluster runs the Prometheus operator; its shared instance selects ServiceMonitors # by `monitoring.configbutler.ai/instance: gitops-reverser` diff --git a/test/e2e/source_namespace_e2e_test.go b/test/e2e/source_namespace_e2e_test.go index 94983b78f..1038b0e59 100644 --- a/test/e2e/source_namespace_e2e_test.go +++ b/test/e2e/source_namespace_e2e_test.go @@ -271,7 +271,7 @@ var _ = Describe("WatchRule source namespace", Label("manager"), Ordered, func() By("asserting the terminal refusal is published with a fix-naming message") verifyResourceCondition("watchrule", refusedRule, configNS, "SourceNamespaceAuthorized", "False", "SourceNamespaceNotAllowed", - "allowAnySourceNamespace") + "spec.allowCrossNamespace.sources") verifyResourceCondition("watchrule", refusedRule, configNS, "Stalled", "True", "SourceNamespaceNotAllowed", "") verifyResourceCondition("watchrule", refusedRule, configNS, @@ -307,7 +307,8 @@ metadata: spec: accessFrom: names: [%s] - allowAnySourceNamespace: %t + allowCrossNamespace: + sources: %t `, name, allowedNS, delegate) return kubectlRunWithStdin("", manifest, "apply", "-f", "-") } diff --git a/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/clusterprovider.yaml b/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/clusterprovider.yaml index e76b4d22a..11eed5156 100644 --- a/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/clusterprovider.yaml +++ b/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/clusterprovider.yaml @@ -10,4 +10,5 @@ metadata: spec: accessFrom: names: [homelab-config] - allowAnySourceNamespace: true + allowCrossNamespace: + sources: true diff --git a/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/gittarget.yaml b/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/gittarget.yaml index 37350cd99..00241d047 100644 --- a/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/gittarget.yaml +++ b/test/fixtures/layout-corpus/shapes/1-flat-serialized/config/gittarget.yaml @@ -11,7 +11,7 @@ spec: branch: main path: mirror/prod # The WatchRule beside this file names source namespaces other than its own. The - # ClusterProvider's allowAnySourceNamespace is the only grant that needs, and the + # ClusterProvider's allowCrossNamespace.sources is the only grant that needs, and the # source credential's RBAC bounds what it can actually read. # Flat is a DECLARED shape. Without a template the built-in ladder ends at the # built-in identity path, which is a tree. One file per object, directly in the diff --git a/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/clusterprovider.yaml b/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/clusterprovider.yaml index c010ab313..55974d237 100644 --- a/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/clusterprovider.yaml +++ b/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/clusterprovider.yaml @@ -10,4 +10,5 @@ metadata: spec: accessFrom: names: [homelab-config] - allowAnySourceNamespace: true + allowCrossNamespace: + sources: true diff --git a/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/gittarget.yaml b/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/gittarget.yaml index 6e2ebbcb7..f71024eb6 100644 --- a/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/gittarget.yaml +++ b/test/fixtures/layout-corpus/shapes/3-tree-serialized/config/gittarget.yaml @@ -11,7 +11,7 @@ spec: branch: main path: clusters/home # The WatchRule beside this file names source namespaces other than its own. The - # ClusterProvider's allowAnySourceNamespace is the only grant that needs, and the + # ClusterProvider's allowCrossNamespace.sources is the only grant that needs, and the # source credential's RBAC bounds what it can actually read. # No placement declared and no kustomization anywhere in the subtree, so the # ladder falls through to the built-in identity path: diff --git a/test/fixtures/layout-corpus/specific-examples/prerequisites/README.md b/test/fixtures/layout-corpus/specific-examples/prerequisites/README.md index b0d1ea82b..d3ae968fa 100644 --- a/test/fixtures/layout-corpus/specific-examples/prerequisites/README.md +++ b/test/fixtures/layout-corpus/specific-examples/prerequisites/README.md @@ -23,13 +23,13 @@ provider in another namespace. The examples here use the operator's in-cluster source. A target that captures objects from a different source namespace needs one grant, and it belongs to the platform admin: the `ClusterProvider` delegates the *ability* to choose a source namespace -(`allowAnySourceNamespace`). What may actually be read is bounded by that provider credential's own +(`allowCrossNamespace.sources`). What may actually be read is bounded by that provider credential's own RBAC in the source cluster. Concrete specimens are in [`../../shapes/1-flat-serialized/config/`](../../shapes/1-flat-serialized/config/clusterprovider.yaml) and [`../../shapes/3-tree-serialized/config/`](../../shapes/3-tree-serialized/config/clusterprovider.yaml). The source-cluster rule and source-namespace authorization are separate checks. `accessFrom` decides -which configuration namespaces may reference the provider at all; `allowAnySourceNamespace` decides +which configuration namespaces may reference the provider at all; `allowCrossNamespace.sources` decides whether those namespaces' `GitTarget`s may look outside their own. ## Credentials stay out of this design diff --git a/test/utils/utils.go b/test/utils/utils.go index 90ee22d51..a01068c7a 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "fmt" + "io" "os" "os/exec" "strings" @@ -34,6 +35,31 @@ func Run(cmd *exec.Cmd) (string, error) { return string(output), nil } +// RunStreaming is Run for a long command: it also copies the output to GinkgoWriter as it comes. +// Run collects the output and hands it over when the command ends, so a command the suite timeout +// kills leaves nothing behind; a timed-out node's report carries what GinkgoWriter received, which +// then shows where the command stopped. +func RunStreaming(cmd *exec.Cmd) (string, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + if err := os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) + var output bytes.Buffer + // One writer for both streams, so os/exec copies them on one goroutine. + out := io.MultiWriter(&output, GinkgoWriter) + cmd.Stdout, cmd.Stderr = out, out + if err := cmd.Run(); err != nil { + return output.String(), fmt.Errorf("%q failed: %w (its output is above)", command, err) + } + return output.String(), nil +} + // GetNonEmptyLines converts given command output string into individual objects // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string {