Repository navigation
Expand file tree
/
Copy pathbranch_worker.go
More file actions
3229 lines (2953 loc) · 135 KB
/
Copy pathbranch_worker.go
File metadata and controls
3229 lines (2953 loc) · 135 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
package git
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
gogit "github.com/go-git/go-git/v6"
"github.com/go-git/go-git/v6/plumbing"
gitclient "github.com/go-git/go-git/v6/plumbing/client"
"github.com/go-logr/logr"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3"
"github.com/ConfigButler/gitops-reverser/internal/sanitize"
"github.com/ConfigButler/gitops-reverser/internal/telemetry"
itypes "github.com/ConfigButler/gitops-reverser/internal/types"
"github.com/ConfigButler/gitops-reverser/internal/typeset"
)
const (
// DefaultCommitWindow is the default idle timeout that closes a commit window, used when
// GitTarget.spec.commit.window.idleTimeout is unset or the target cannot be read.
DefaultCommitWindow = 5 * time.Second
// DefaultCommitWindowMaxDuration is the default maximum duration of a commit window, used when
// GitTarget.spec.commit.window.maxDuration is unset or the target cannot be read. It keeps
// continuous activity from postponing a commit until the memory limit forces one.
DefaultCommitWindowMaxDuration = time.Minute
// PushCooldown is the minimum interval between successful pushes. The cooldown
// is intentionally fixed: commit cadence is a user concern (spec.commit.window on
// the CRD); push cadence is an implementation/politeness concern.
PushCooldown = 5 * time.Second
)
var (
// These function vars are narrow test seams for classifying push failures
// without duplicating the repository layer in tests.
//nolint:gochecknoglobals
pushAtomicFn = PushAtomic
//nolint:gochecknoglobals
fetchRemoteBranchHashFn = fetchRemoteBranchHash
//nolint:gochecknoglobals
syncToRemoteFn = syncToRemoteFrom
// beforePushAdmission runs right before a push cycle compares its root's parent generation
// with the current one. Nil in production; a test changes the parent here to land a change
// just before admission.
//nolint:gochecknoglobals
beforePushAdmission func()
//nolint:gochecknoglobals
advertiseRemoteBranchFn = advertiseRemoteBranch
)
// BranchWorker processes events for a single (GitProvider, Branch) combination.
// It can serve multiple GitTargets that write to different paths in the same branch.
// This design ensures serialized commits per branch, preventing merge conflicts.
type BranchWorker struct {
// Identity (immutable after creation)
GitProviderRef string
GitProviderNamespace string
Branch string
// repo is the repository this worker is ABOUT: its clone, its base trust, its observations
// and its retained writes all describe that one repository and no other.
//
// It is immutable for the same reason the three fields above are. The worker does not follow
// its GitProvider to a new repository — meeting one is not a state change to be undone field
// by field, it is a different subject — so the WorkerManager replaces the worker instead.
// See RepoIdentity and WorkerManager.EnsureWorker.
repo RepoIdentity
// Dependencies
Client client.Client
Log logr.Logger
contentWriter *contentWriter
// mapper resolves manifest GVKs into resource identities while building the
// GitTarget inventory, for the LOCAL cluster and as the fallback when no source
// cluster is named. A nil mapper keeps the writer structure-only, so object-less
// deletes have no resource index to target.
mapper typeset.Lookup
// clusterMapper resolves the GVK->GVR lookup for a NAMED source cluster: a folder that
// mirrors a remote resolves its documents against that remote's type registry, never a
// union of all clusters (two clusters can serve one GVK under different GVRs/scopes). Nil
// in the CLI and in tests, which fall back to `mapper`; wired at startup by the
// WorkerManager from the watch manager's ClusterTypeLookup.
clusterMapper func(clusterID string) typeset.Lookup
// sshHostKeys configures SSH host-key resolution (install-level default ConfigMap and the
// dev-only missing-key opt-out) for this worker's credential reads. Set by the WorkerManager
// before Start, on the same goroutine the event loop reads it from.
sshHostKeys SSHHostKeyConfig
// credentialPolicy controls explicit insecure opt-ins for Git credential transports.
credentialPolicy CredentialTransportPolicy
// pathRefusal surfaces a refused write plan as GitTarget GitPathAccepted=False. The
// live-event paths have no result channel to carry the refusal back, so without it a
// refused live write would abort the commit and leave the GitTarget looking healthy. Set
// by the WorkerManager before Start; a nil reporter only drops the status transition.
pathRefusal PathRefusalReporter
// renderFidelityGate closes normal live writes while a target's current render-vs-live
// epoch is pending or divergent. Resync remains allowed so it can measure and repair Git.
// Set by WorkerManager before Start, alongside pathRefusal.
renderFidelityGate *RenderFidelityGate
// scanAcceptance publishes what a read-only scan of the folder found, so a refusal that
// arrived in Git is visible before anything tries to write. Set by the WorkerManager before
// Start, alongside pathRefusal; a nil reporter only drops the projection.
scanAcceptance ScanAcceptanceReporter
// layoutReporter publishes what each scan resolved about a target's folder, projected as
// status.placement and the LayoutResolved condition. Set by the WorkerManager before Start,
// alongside pathRefusal; a nil reporter only drops the projection.
layoutReporter LayoutReporter
// remoteReporter delivers each confirmed observation of the branch's remote state to the
// worker manager, which keeps it for every GitTarget on the branch. Set by the WorkerManager
// before Start, alongside pathRefusal; nil in the CLI and in tests, where an observation is
// simply not shared.
remoteReporter RemoteReporter
// Event processing
eventQueue chan WorkItem
ctx context.Context
cancelFunc context.CancelFunc
wg sync.WaitGroup
started bool
mu sync.Mutex
// stoppingState closes the queue to new work from the moment Stop is called.
//
// Shutdown drains whatever is still on the FIFO without processing it, so an item accepted
// after that point is silently thrown away, while Enqueue's "true" has already told a watch
// loop it may advance its durable cursor past the event. That cost nothing while Stop was
// only ever called at process exit. A worker REPLACED because its GitProvider names a
// different repository is stopped with the process still running, and the sibling GitTargets
// on its branch go on holding streams that point at it until their own next reconcile, which
// is minutes of accepted-and-discarded events.
//
// Refusing them turns a silent loss into the drop path every producer already handles: the
// watch leaves its cursor where it is and redelivers to the replacement. Items already ON the
// queue when Stop is called are still drained, which is the behaviour shutdown always had.
//
// It is written and read under pendingResyncsMu, the lock every enqueue already holds across
// its send. Checking it outside that lock is not enough: an enqueue could pass the check,
// block on the lock, and send after Stop had finished draining, into a queue with no loop left
// to read it — and still answer "accepted".
stoppingState bool
// exitedState is set once the event loop has returned, for whatever reason, and the worker
// has settled every CommitRequest it knew. From then on nothing can act on a request, so a
// withdraw is answered at once instead of being queued for a loop that is gone. Guarded by
// pendingResyncsMu, like stoppingState.
exitedState bool
// lastObservation is what this worker last PROVED about the target branch on the remote, and
// when. See RemoteObservation: it replaces a fetch-only trio (branchExists, lastCommitSHA,
// lastFetchTime) that went arbitrarily stale on exactly the busiest targets, because a push
// never wrote to it.
lastObservation atomic.Pointer[RemoteObservation]
// repoMu serializes repository/worktree operations within this worker.
repoMu sync.Mutex
// baseTrustedState carries the invariant "the checkout's root sits at the remote tip of the
// target branch, or the worker knows it does not". It is read through baseTrusted() and written
// through setBaseTrusted and invalidateBase.
//
// It is an atomic rather than a repoMu-protected field because the loss points are spread
// across every error return on the write path, and a lock there would put a second ordering
// constraint on code whose only job is to record that something went wrong.
//
// See the package's design note in docs/design/push-notification-and-reconcile-trigger.md §1.5.
baseTrustedState atomic.Bool
// checkoutApplied is how many of the loop's retained writes the checkout holds: the checkout is
// the projection of the pending writes, its root plus one commit per write in
// pendingWrites[:applied].
// checkoutUnknown means a write failed part-way and the worktree may hold anything.
//
// It is the one answer to "may the loop commit on, or push, what the checkout holds?", which
// is why it is a count and not a flag. A reset to the remote tip sets it to zero, which with
// writes retained says their commits are gone and must be replayed: pushing then would find the
// branch already at the remote tip, report success without sending anything, and settle work
// that exists nowhere. A commit raises it, a replay sets it to every pending write, and a push
// that publishes them returns it to zero. A failed write sets checkoutUnknown, and only a reset
// clears that: a successful push says where the remote is, and nothing about leftovers in the
// worktree.
//
// See docs/design/gittarget-branch-worker-pending-writes.md.
checkoutApplied atomic.Int64
// lastRefusalTouch records when this worker last pushed an empty commit for a GitTarget, keyed
// by "namespace/name", so refusalTouchInterval can floor the rate. It is guarded by its own
// mutex rather than repoMu: the decision is taken before any repository work starts, and
// borrowing the repository lock for it would order the two for no reason.
refusalTouchMu sync.Mutex
lastRefusalTouch map[string]time.Time
// coveredRefusal records, per GitTarget AND WATCHED COLLECTION, the refusal observation the last
// empty commit was made for: a digest of the objects that were refused together with the
// issues raised about them. A recheck that observes exactly the same thing is already covered
// by that commit, so it is not another trigger. It is dropped when a resync for that collection is
// ACCEPTED — see refusalRecovered, which is what makes a live edit re-made after a revert
// count as new, and what keeps one watched type's success from speaking for another's.
// Guarded by refusalTouchMu with the rate limit, because the two are read together.
coveredRefusal map[refusalKey]string
// branchBufferMaxBytes caps the retained in-memory event data; tripped on
// event arrival, an immediate finalize bypasses the commit window.
branchBufferMaxBytes int64
// pushCycleRootHash is the hash the target branch was at on the remote
// before the first commit of the current push cycle. pushCycleRootBranch is
// the remote branch/ref that hash came from. For an existing worker branch
// this is usually the same branch; for a brand new branch it is the default
// branch we based the new branch on.
//
// PushAtomic uses this pair to detect concurrent updates: if the remote has
// moved since, we hit the conflict path and replay from retained pending
// writes.
pushCycleRootBranch plumbing.ReferenceName
// Cleared after a successful push. Protected by repoMu.
pushCycleRootHash plumbing.Hash
// newBranchParent and newBranchParentHash are the parent the local write branch was created
// from, while the remote does not carry the write branch. They let a cycle that finds HEAD
// already on the local write branch still root itself on the parent: a no-op cycle (a resync
// or a live write with nothing to commit) creates the local branch and leaves HEAD on it, and
// local HEAD alone cannot say that the branch is absent on the remote.
//
// Cleared as soon as the remote is known to carry the branch: a fetch that found it, or a push
// that created or confirmed it. Protected by repoMu.
newBranchParent plumbing.ReferenceName
newBranchParentHash plumbing.Hash
// parentState is the GitTarget's spec.parentBranch, with a generation that SetParentBranch
// raises on every change. It is set by the reconcile and read by the event loop, hence atomic.
// See parentConfig.
parentState atomic.Pointer[parentConfig]
// baseParentGen is the generation of the parent configuration the checkout was last proven
// against: the snapshot whose fetch produced it, or the current one once the checkout is known
// not to depend on the parent (the write branch exists). Trust is scoped to it, so a push or a
// fetch that finishes after a change can never vouch for the new configuration.
baseParentGen atomic.Uint64
// pushCycleRootGen is the parent generation pushCycleRootBranch was chosen under. The push
// admission check compares it with the current one. Protected by repoMu.
pushCycleRootGen uint64
// firsts surfaces the first successful commit and push at default verbosity.
firsts branchWorkerLogFirsts
// retentionLoggedAt throttles the default-verbosity "prune policy retained documents"
// line to one per GitTarget subtree per retentionLogInterval. Retaining is a steady
// state, so the log must not scale with the reconcile rate. Owned by the event loop
// goroutine, like firsts, so it carries no lock.
retentionLoggedAt map[string]time.Time
// hasUnpushedWork mirrors whether the event loop is currently holding a live
// open window or any committed-but-not-yet-pushed pending writes. The event
// loop is the only writer and, via syncUnpushedWorkFlag, the only reader; the
// atomic guards the store/load and lets the depth gauge account for retained
// work the channel length alone cannot see.
hasUnpushedWork atomic.Bool
// inflightItems counts work items accepted onto eventQueue but not yet fully
// handled. It is incremented before the channel send (so it can never lag the
// loop's receive) and decremented only after handleQueueItem returns. Using
// len(eventQueue) instead would read 0 in the window between the loop
// receiving an item and finishing it — a scrape there could satisfy a drain
// gate while a commit/push is still in flight. The depth gauge sums this with
// hasUnpushedWork, so it reads 0 only once every accepted item has been
// handled and nothing is retained.
inflightItems atomic.Int64
// pendingResyncs coalesces queued resyncs by (GitTarget, scope). A resync is
// state-based and idempotent — "make Git match this desired set for this
// scope" — so a newer one wholly supersedes an older one still waiting. The
// map holds the open marker per key, and the marker the current request; the
// FIFO holds the marker, which is what preserves ordering against live events. Queue depth is
// therefore bounded by the number of distinct scopes, not by the request
// rate, so a storm of resyncs for one target can no longer fill the queue
// and starve every other GitTarget on this branch.
//
// Coalescing reuses the marker's FIFO POSITION, so it is sound only while nothing for that
// scope was queued behind it: running a newer snapshot at the older position would apply it
// before writes it already contains, which would then overwrite it. The entry's tail flag
// records that boundary.
pendingResyncsMu sync.Mutex
pendingResyncs map[resyncKey]*resyncMarker
// crOutcomes holds resolved CommitRequest outcomes for the controller to poll
// via LookupCommitRequestOutcome. The event loop is the only writer (on its
// goroutine), the controller the only reader (on a reconcile goroutine), so the
// mutex guards the cross-goroutine handoff. Entries are GC'd by age. The
// accessors live in commit_request_attach_loop.go alongside the attach logic.
crOutcomesMu sync.Mutex
crOutcomes map[commitRequestID]commitRequestOutcomeEntry
// crPhases holds where each unresolved CommitRequest stands, for the controller to report
// while it waits. Guarded by crOutcomesMu; the event loop is the only writer.
crPhases map[commitRequestID]CommitRequestPhase
// crQueuedAttaches counts, per request, the attaches on the FIFO that the loop has not handled
// yet. An outcome is not collected while any are queued: the outcome is what turns a late attach
// into a no-op, so collecting it first would let that attach register a withdrawn request
// again, however long the outcome had been kept. Guarded by crOutcomesMu.
crQueuedAttaches map[commitRequestID]int
// crOwners is the manager's table of which worker holds each request. See commitRequestOwners.
// Set by the WorkerManager before Start; nil for a worker built on its own.
crOwners *commitRequestOwners
// clock is the time source of the retry schedule; nil is time.Now. See now().
clock func() time.Time
// parentRecoveryOpen, parentRecoveryFound and parentProbeHold mirror the event loop's
// recovery latch for readers outside it. See parent_recovery.go.
parentRecoveryOpen atomic.Bool
parentRecoveryFound atomic.Bool
parentProbeHold atomic.Pointer[parentProbeHold]
// intake decides whether new writes, saves and resyncs may enter the FIFO. See intake.go.
intake intakeGate
// publicationReporter is told when the worker's publication report changes. Set by the
// WorkerManager before Start; nil in tests that do not assert on it. See publication.go.
publicationReporter func()
// publication is the loop's latest publication report, and backlog what it keeps, for readers
// outside the loop. See publication.go.
publication atomic.Pointer[PublicationStatus]
backlog publicationBacklog
}
// branchWorkerLogFirsts logs the first successful commit and push of a worker's
// lifetime at default verbosity, so an operator can confirm the git write path
// works end to end, without logging every subsequent commit/push cycle.
type branchWorkerLogFirsts struct {
commit sync.Once
push sync.Once
}
// healKey identifies a deferred heal by the (GitTarget, scope) it corrects, so re-stashing a heal
// for the same target+type replaces the parked one rather than queuing a duplicate.
type healKey struct {
name string
namespace string
scope string
}
type windowFinalizeReason string
const (
windowFinalizeReasonUnspecified windowFinalizeReason = "unspecified"
windowFinalizeReasonIdleTimeout windowFinalizeReason = "idle-timeout"
windowFinalizeReasonMaxDuration windowFinalizeReason = "max-duration"
windowFinalizeReasonAttachNext windowFinalizeReason = "attach-next"
windowFinalizeReasonResyncBeforeApply windowFinalizeReason = "resync-before-apply"
windowFinalizeReasonAtomicBeforeApply windowFinalizeReason = "atomic-before-apply"
windowFinalizeReasonIdentityChange windowFinalizeReason = "author-or-target-change"
windowFinalizeReasonBufferLimit windowFinalizeReason = "buffer-limit"
windowFinalizeReasonShutdown windowFinalizeReason = "shutdown"
)
// NewBranchWorker creates a worker for a (provider, branch) combination. A zero
// BranchWorkerLimits gets the defaults for both knobs; see BranchWorkerLimits.
func NewBranchWorker(
client client.Client,
log logr.Logger,
providerName, providerNamespace string,
branch string,
repo RepoIdentity,
writer *contentWriter,
limits BranchWorkerLimits,
) *BranchWorker {
if writer == nil {
writer = newContentWriter(itypes.SensitiveResourcePolicy{})
}
limits = limits.withDefaults()
// Both flags start at their zero values, which is the state §3 requires of a new worker:
// nothing has looked at the remote, so the base is untrusted, and nothing has written, so
// the worktree is clean.
return &BranchWorker{
GitProviderRef: providerName,
GitProviderNamespace: providerNamespace,
Branch: branch,
repo: repo,
Client: client,
Log: log.WithValues(
"provider", providerName,
"namespace", providerNamespace,
"branch", branch,
),
contentWriter: writer,
eventQueue: make(chan WorkItem, limits.QueueDepth),
pendingResyncs: make(map[resyncKey]*resyncMarker),
branchBufferMaxBytes: limits.MaxBufferBytes,
}
}
// workerStateRoot is where every worker's on-disk state lives. It is a variable rather than a
// constant so a test can point it at its own directory: the tests that exercise reclaiming a
// retired worker's checkout delete directories, and doing that under the shared root would take
// out the state of anything else using it, including a developer's running operator.
//
//nolint:gochecknoglobals // a test seam, like pushAtomicFn above
var workerStateRoot = filepath.Join("/tmp", "gitops-reverser-workers")
// branchPathComponent renders a branch name as ONE directory component.
//
// Git branch names contain slashes — `release/v1` is ordinary — and joining one into a path makes
// its worker's directory a CHILD of the `release` worker's. That is harmless while nothing deletes
// anything, and it is a data-loss bug the moment something does: retiring `release` would take the
// live `release/v1` worker's checkout with it.
//
// The readable prefix is for whoever is looking at the directory; the hash of the raw name is what
// makes it unambiguous, including for the names sanitising would otherwise collapse together
// (`release/v1` and `release-v1`).
func branchPathComponent(branch string) string {
sum := sha256.Sum256([]byte(branch))
digest := hex.EncodeToString(sum[:8])
readable := make([]rune, 0, len(branch))
for _, r := range branch {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_':
readable = append(readable, r)
default:
readable = append(readable, '-')
}
}
if len(readable) > branchComponentReadableMax {
readable = readable[:branchComponentReadableMax]
}
return string(readable) + "-" + digest
}
// branchComponentReadableMax caps the readable half so a long branch name cannot push the path
// past what a filesystem accepts; the digest that follows is what carries the identity.
const branchComponentReadableMax = 40
// repoRootPath is where this worker keeps its checkout: the GitProvider's UID, then the branch.
//
// The UID is the whole of the first component, and it replaces three things that were there
// before: the provider's namespace, its name, and a digest of the remote URL. A UID is globally
// unique and is minted fresh by the API server for every object, so it says everything those
// three said and one thing they could not — WHICH INCARNATION of the provider this is. A
// repointed provider is a recreated one, so its worker cannot inherit, share, or delete the
// directory its predecessor was using, and no digest has to stand in for that.
func (w *BranchWorker) repoRootPath() string {
return filepath.Join(workerStateRoot, w.stateComponent(), branchPathComponent(w.Branch))
}
// stateComponent names this worker's state directory.
//
// A worker built with no GitProvider behind it — the CLI, and tests — has no UID to be unique by,
// and every one of them would otherwise share one directory per branch. Those workers fall back to
// a digest of the remote they were built for, which is the only identity they have.
func (w *BranchWorker) stateComponent() string {
if w.repo.ProviderUID != "" {
return string(w.repo.ProviderUID)
}
sum := sha256.Sum256([]byte(strings.TrimSpace(w.repo.URL)))
return "url-" + hex.EncodeToString(sum[:16])
}
// repoPath is the checkout of the repository this worker is ABOUT.
//
// The URL comes from the worker's own identity and never from the GitProvider as it reads NOW. The
// provider is still read on every cycle — for credentials, commit identity and signing — and it
// can name a different repository by then, because it is repointed by being recreated while the
// worker is keyed by (provider, branch). Following it there is what produced the outage in #382:
// the cycle planned against trust earned on the old repository and every write failed at
// `open repository: repository does not exist`. A worker that cannot be pointed anywhere cannot
// have that happen to it; the WorkerManager replaces it instead.
func (w *BranchWorker) repoPath() string { return w.repoRootPath() }
// removeLocalState deletes everything this worker kept on disk: the clone of every remote it
// worked with, under its own (provider UID, branch) directory.
//
// It is called only when a worker is being retired because nothing needs it any more, never on
// shutdown — a restart wants the clones it left behind, and re-cloning every branch on every
// restart is exactly the cost the on-disk cache exists to avoid.
//
// The identity guard is not defensive programming. repoRootPath joins the UID and the branch into
// a fixed prefix, so a worker with neither — which tests build — resolves to the ROOT of every
// worker's state, and deleting that would take out the live workers next to it.
func (w *BranchWorker) removeLocalState() {
if w.repo.IsZero() || w.Branch == "" {
w.Log.V(1).Info("Not removing worker state: the worker has no complete identity")
return
}
path := w.repoRootPath()
if err := os.RemoveAll(path); err != nil {
// Worth knowing and not worth failing for: the worker is gone either way, and what is
// left is disk rather than anything that can produce a wrong answer.
w.Log.Error(err, "Could not remove the retired worker's checkout", "path", path)
return
}
w.Log.Info("Removed the retired worker's checkout", "path", path)
}
// Start begins processing events.
func (w *BranchWorker) Start(parentCtx context.Context) error {
w.mu.Lock()
if w.started {
w.mu.Unlock()
return errors.New("worker already started")
}
w.ctx, w.cancelFunc = context.WithCancel(parentCtx)
w.started = true
w.mu.Unlock()
w.Log.Info("Starting branch worker")
w.wg.Add(1)
go func() {
defer w.wg.Done()
w.processEvents()
w.settleCommitRequestsAfterExit()
}()
return nil
}
// Stop gracefully shuts down the worker.
func (w *BranchWorker) Stop() {
w.mu.Lock()
if !w.started {
w.mu.Unlock()
return
}
w.mu.Unlock()
// Before the cancel, and under the lock every enqueue holds across its send, so admission
// closes at one instant: an enqueue either completed before this point or is refused.
w.pendingResyncsMu.Lock()
w.stoppingState = true
w.pendingResyncsMu.Unlock()
w.Log.Info("Stopping branch worker")
w.cancelFunc()
w.wg.Wait()
w.Log.Info("Branch worker stopped")
}
// Enqueue adds a single live event to this worker's queue. It reports whether the
// event entered the FIFO; a false return means the queue was full and the event was
// dropped, so a caller advancing a durable watch cursor past this event must not treat
// the drop as success (see reconcile.GitTargetEventStream.OnWatchEvent).
func (w *BranchWorker) Enqueue(event Event) bool {
return w.enqueueRequest(&WriteRequest{
Events: []Event{event},
CommitMode: CommitModePerEvent,
CommitMessage: "",
})
}
// EnqueueWithdraw asks the loop to cancel a CommitRequest it has not acted on. It never blocks: a
// full queue or a stopping worker drops it, and the controller sends it again on its next poll. A
// worker whose loop has exited answers at once, because nothing is left that could act on it.
func (w *BranchWorker) EnqueueWithdraw(req *AttachCommitRequest) {
if req == nil {
return
}
w.pendingResyncsMu.Lock()
if w.exitedState {
w.pendingResyncsMu.Unlock()
w.answerWithdrawAfterExit(req.id())
return
}
defer w.pendingResyncsMu.Unlock()
if w.stoppingLocked() {
// Still draining: its last push may yet carry the request, so only the exited worker can
// answer. The controller asks again on its next poll.
return
}
w.inflightItems.Add(1)
select {
case w.eventQueue <- WorkItem{Withdraw: req}:
default:
w.inflightItems.Add(-1)
w.Log.V(1).Info("Queue full, CommitRequest withdraw dropped (the controller re-sends)",
"request", req.Namespace+"/"+req.Name)
}
}
// EnqueueAttach adds a CommitRequest attach to this worker's queue. Riding the
// same queue as resource events is what makes it process in audit order, after
// every earlier write. The attach is fire-and-forget: the controller polls the
// outcome via LookupCommitRequestOutcome and re-sends idempotently, so a queue-
// full drop is recovered by the next poll rather than a synchronous reply.
func (w *BranchWorker) EnqueueAttach(req *AttachCommitRequest) {
if req == nil {
return
}
charge := attachCharge(req.Message)
// Increment before the send so inflightItems can never lag the loop's
// receive; roll back if the queue is full and the item is dropped.
w.inflightItems.Add(1)
// An attach decides which commit window subsequent work joins, so a resync
// must not be moved across one either. Attaches are controller-driven and
// rare, so blocking a coalesce on them costs no storm protection. The mark
// and the send are ONE critical section: see enqueueRequest.
w.pendingResyncsMu.Lock()
if w.stoppingLocked() {
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.recordQueueRefusal(queueRefusalAttach)
w.Log.V(1).Info("Worker is stopping, CommitRequest attach refused (the controller re-sends)")
return
}
if !w.admitPayload(charge) {
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.recordQueueRefusal(queueRefusalAttach)
w.Log.V(1).Info("CommitRequest attach refused while the branch has paused intake "+
"(the controller re-sends)", "request", req.Namespace+"/"+req.Name)
return
}
id := req.id()
if !w.crOwners.claim(id, w) {
// The router routes to the owner, so only a request that moved workers between two polls
// gets here. The owner answers for it; registering it twice could commit it twice.
w.intake.release(charge)
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.Log.Info("CommitRequest attach refused: another branch worker holds the request",
"request", req.Namespace+"/"+req.Name)
return
}
w.markResyncTailForTargetLocked(req.GitTargetNamespace, req.GitTargetName)
w.countQueuedAttach(id)
select {
case w.eventQueue <- WorkItem{Attach: req, charge: charge}:
w.pendingResyncsMu.Unlock()
w.Log.Info("CommitRequest attach enqueued",
"request", req.Namespace+"/"+req.Name,
"author", req.Author,
"target", req.GitTargetNamespace+"/"+req.GitTargetName,
"attach", string(req.Attach),
"messageOverride", req.Message != "")
// Nothing publishes depth here: the gauge reads inflightItems at scrape
// time, so an enqueue is visible to the next scrape whether or not the
// loop has woken to notice it.
default:
w.uncountQueuedAttach(id)
w.intake.release(charge)
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.recordQueueRefusal(queueRefusalAttach)
w.Log.Error(nil, "Event queue full, CommitRequest attach refused (controller will re-send)")
}
}
// EnqueueResync adds a resync request to this worker's queue. Like a finalize
// signal it rides the same queue as resource events, so it is applied in order with
// live events: a resync enqueued during the snapshot window lands before the buffered
// live events that follow it. If the queue is full the request is refused and its
// caller is told immediately via the result channel, so it can gather the scope again.
//
// It reports whether the request actually entered the FIFO. A caller gating downstream state on
// the resync's ordering must not treat a refusal as success: it would mark the target
// reconciled-through-Hc with no reconcile ever queued.
func (w *BranchWorker) EnqueueResync(request *ResyncRequest) bool {
if request == nil {
return false
}
key := resyncKeyFor(request)
charge := request.charge() // sized here, off the loop and outside the lock
w.pendingResyncsMu.Lock()
if w.stoppingLocked() {
w.pendingResyncsMu.Unlock()
w.recordQueueRefusal(queueRefusalResync)
w.Log.V(1).Info("Worker is stopping, resync request refused",
"gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName)
// Answered rather than dropped in silence: the caller is waiting on this channel, and the
// shutdown drain does not answer what it throws away.
request.reply(ResyncResult{Err: ErrFinalizeQueueFull})
return false
}
if w.pendingResyncs == nil {
w.pendingResyncs = make(map[resyncKey]*resyncMarker)
}
w.markResyncTailForResyncLocked(request, key)
if pending, queued := w.pendingResyncs[key]; queued && !pending.tailPassed {
// A marker for this key is already in the FIFO and nothing for this scope
// was queued behind it, so the marker's position is still the right place
// for a newer snapshot. Swap in the newer request; the loop reads whatever
// is current when the marker comes up. Only the difference in size is new.
if !w.admitPayload(charge - pending.request.charge()) {
w.pendingResyncsMu.Unlock()
return w.refuseResyncAtIntake(request, charge)
}
superseded := pending.request
pending.request, pending.charge = request, charge
w.pendingResyncsMu.Unlock()
// The superseded request's caller is waiting on its reply channel. Answer
// it so its drain ends, with a sentinel that says "a newer resync for the
// same scope is queued", not "this failed".
superseded.reply(ResyncResult{Err: ErrResyncSuperseded})
w.Log.V(1).Info("Resync request coalesced into the one already queued",
"resources", len(request.Desired),
"gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName,
"sourceCollection", sourceCollectionForLog(request.SourceCollection))
return true
} else if queued {
// A write inside this scope was queued behind the marker. Coalescing here
// would run this snapshot ahead of those writes and let them overwrite it
// with older state. Release the key instead: the queued marker keeps the
// request it holds, and this request takes a fresh marker at the tail below.
delete(w.pendingResyncs, key)
w.Log.V(1).Info("Resync request not coalesced: writes are queued behind the pending marker",
"scope", request.Scope.String(),
"gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName)
}
// Insert the entry and queue its marker in ONE critical section. Releasing the
// lock between them would let a concurrent enqueue for the same key coalesce
// into an entry whose marker does not exist yet -- and which this call is
// about to remove on a full queue. That caller would be told enqueued=true
// while its request never ran and its drain waited for a reply that never
// came. The send is non-blocking, so holding the lock across it cannot
// deadlock against the loop's takePendingResync.
if !w.admitPayload(charge) {
w.pendingResyncsMu.Unlock()
return w.refuseResyncAtIntake(request, charge)
}
marker := &resyncMarker{request: request, charge: charge}
w.pendingResyncs[key] = marker
w.inflightItems.Add(1)
select {
case w.eventQueue <- WorkItem{Resync: marker}:
w.pendingResyncsMu.Unlock()
w.Log.V(1).Info("Resync request enqueued",
"resources", len(request.Desired),
"gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName,
"sourceCollection", sourceCollectionForLog(request.SourceCollection))
return true
default:
w.inflightItems.Add(-1)
delete(w.pendingResyncs, key)
w.intake.release(charge)
w.pendingResyncsMu.Unlock()
w.recordQueueRefusal(queueRefusalResync)
w.Log.Error(nil, "Event queue full, resync request refused; its caller gathers it again",
"gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName,
"sourceCollection", sourceCollectionForLog(request.SourceCollection))
request.reply(ResyncResult{Err: ErrFinalizeQueueFull})
return false
}
}
// refuseResyncAtIntake answers a resync the intake gate refused. It reports false: the request did
// not enter the FIFO.
func (w *BranchWorker) refuseResyncAtIntake(request *ResyncRequest, charge int64) bool {
w.recordQueueRefusal(queueRefusalResync)
err := w.intakeRefusal(charge)
w.Log.V(1).Info("Resync refused while the branch has paused intake",
"gitTarget", request.GitTargetNamespace+"/"+request.GitTargetName, "reason", err.Error())
request.reply(ResyncResult{Err: err})
return false
}
// markResyncTailForWriteLocked records, on every pending resync whose GitTarget and
// scope contain one of this write's events, that work for that scope is now queued
// behind its marker. A pending resync with a nil scope covers the whole GitTarget, so
// any write for that target marks it. The caller must hold pendingResyncsMu.
//
// The target is read from the EVENT, not the request: the live path leaves the request-level
// fields empty, so reading the request alone would silently never match the only path this fence
// exists for.
//
// The scope match is by object identity, not producing stream, because the collection that produced an
// event cannot be recovered from it. Over-matching is deliberate: it can forgo a coalesce, never
// wrongly permit one.
func (w *BranchWorker) markResyncTailForWriteLocked(request *WriteRequest) {
for key, pending := range w.pendingResyncs {
if pending.tailPassed {
continue
}
for i := range request.Events {
namespace, name := writeEventTarget(request, i)
if key.namespace != namespace || key.name != name {
continue
}
if pending.request.Scope.Matches(request.Events[i].Identifier) {
pending.tailPassed = true
break
}
}
}
}
// markResyncTailForResyncLocked records, on every OTHER pending resync of the same GitTarget whose
// sweep boundary overlaps this one's, that a snapshot is now queued behind its marker. A snapshot
// is a write to its whole boundary, so it fences an earlier snapshot exactly as a live write does.
//
// Without it a selector changed A -> B -> A queued snapshots A1, B, A2 and coalesced A2 into A1's
// position: A2 ran first and the retired B snapshot then ran after it, undoing the current
// selection. With it A2 takes the tail, so the order stays A1, B, A2. The caller must hold
// pendingResyncsMu.
func (w *BranchWorker) markResyncTailForResyncLocked(request *ResyncRequest, key resyncKey) {
for pendingKey, pending := range w.pendingResyncs {
if pending.tailPassed || pendingKey == key {
continue
}
if pendingKey.namespace == request.GitTargetNamespace && pendingKey.name == request.GitTargetName &&
pending.request.Scope.Overlaps(request.Scope) {
pending.tailPassed = true
}
}
}
// writeEventTarget resolves the GitTarget one event of a write belongs to: the event's
// own fields when set, falling back to the request's for the atomic paths that carry the
// target once for the whole request.
func writeEventTarget(request *WriteRequest, i int) (string, string) {
if request.Events[i].GitTargetName != "" {
return request.Events[i].GitTargetNamespace, request.Events[i].GitTargetName
}
return request.GitTargetNamespace, request.GitTargetName
}
// markResyncTailForTargetLocked records the same boundary for every pending resync of one
// GitTarget, regardless of scope. It is used for queued items that carry no resource
// identity of their own. The caller must hold pendingResyncsMu.
func (w *BranchWorker) markResyncTailForTargetLocked(namespace, name string) {
for key, pending := range w.pendingResyncs {
if key.namespace == namespace && key.name == name {
pending.tailPassed = true
}
}
}
// takePendingResync returns the request a marker runs, which may be newer than the one it was queued
// with, and the charge it holds, and closes its key if it is still open, so the next enqueue queues a
// fresh marker. A key already released belongs to a later marker, which is left alone.
func (w *BranchWorker) takePendingResync(marker *resyncMarker) (*ResyncRequest, int64) {
w.pendingResyncsMu.Lock()
defer w.pendingResyncsMu.Unlock()
key := resyncKeyFor(marker.request)
if w.pendingResyncs[key] == marker {
delete(w.pendingResyncs, key)
}
return marker.request, marker.charge
}
// enqueueRequest places a write request on the FIFO and reports whether it was
// accepted. A false return means the item was refused (a full queue, paused intake, or a
// stopping worker), so a caller that gates durable state on the write (a watch cursor) must not
// treat the refusal as success, and offers the write again.
func (w *BranchWorker) enqueueRequest(request *WriteRequest) bool {
if request == nil {
return false
}
item := WorkItem{Request: request, charge: eventCharge(request)} // sized off the loop
// Increment before the send so inflightItems can never lag the loop's
// receive; roll back if the queue is full and the item is dropped.
w.inflightItems.Add(1)
// Mark the pending resyncs this write falls inside, and send, in ONE critical
// section. Marking before an unlocked send would leave a window in which a
// concurrent EnqueueResync sees the mark, declines to coalesce, and takes its
// own tail position BEFORE this write reaches the FIFO -- the same inversion
// the fence exists to prevent, one step removed. EnqueueResync holds the same
// lock across its own send, so the two orderings agree. Both sends are
// non-blocking, so holding the lock across them cannot deadlock. Marking a
// write the full queue then drops only forgoes a coalesce, the safe direction.
w.pendingResyncsMu.Lock()
if w.stoppingLocked() {
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.recordQueueRefusal(queueRefusalWrite)
w.Log.V(1).Info("Worker is stopping, request refused; the producer keeps it to redeliver",
"events", len(request.Events),
"gitTarget", request.GitTargetName)
return false
}
if !w.admitPayload(item.charge) {
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.recordQueueRefusal(queueRefusalWrite)
w.Log.V(1).Info("Write refused while the branch has paused intake; "+
"the producer keeps it to redeliver", "events", len(request.Events), "gitTarget", request.GitTargetName)
return false
}
w.markResyncTailForWriteLocked(request)
select {
case w.eventQueue <- item:
w.pendingResyncsMu.Unlock()
w.Log.V(1).Info("Write request enqueued",
"events", len(request.Events),
"mode", request.CommitMode,
"gitTarget", request.GitTargetName,
"sourceCollection", sourceCollectionForLog(request.sourceCollection()))
// Nothing publishes depth here: the gauge reads inflightItems at scrape
// time, so an enqueue is visible to the next scrape whether or not the
// loop has woken to notice it.
return true
default:
w.intake.release(item.charge)
w.pendingResyncsMu.Unlock()
w.inflightItems.Add(-1)
w.recordQueueRefusal(queueRefusalWrite)
// Name the producing collection on a refusal. A saturated queue is diagnosed from what was
// refused and by whom: the 595-in-16-seconds storm was one GitTarget, and the next
// one may be one COLLECTION of one GitTarget.
w.Log.Error(nil, "Event queue full, request refused; its producer offers it again",
"events", len(request.Events),
"mode", request.CommitMode,
"gitTarget", request.GitTargetName,
"sourceCollection", sourceCollectionForLog(request.sourceCollection()))
return false
}
}
// queueDepth reports this worker's pending work: accepted-but-unhandled items plus one when the
// loop is holding retained unpushed work. It reads 0 only on a full drain, so a drain gate cannot
// be satisfied mid-push.
//
// Read by the git_queue_depth observable gauge at scrape time. Both fields are atomics, so the
// callback takes no lock the loop could be holding across its slow work.
func (w *BranchWorker) queueDepth() int64 {
depth := w.inflightItems.Load()
if w.hasUnpushedWork.Load() {
depth++
}
return depth
}
// EnsurePathBootstrapped prepares bootstrap templates locally for a path.
// Existing files are preserved, and only missing template files are added.
// The files are staged in the local worktree but never committed or pushed here.
func (w *BranchWorker) EnsurePathBootstrapped(path, targetName, targetNamespace string) error {
w.repoMu.Lock()
defer w.repoMu.Unlock()
ctx := w.ctx
if ctx == nil {
ctx = context.Background()
}
normalizedPath, err := normalizeBootstrapPath(path)
if err != nil {
return err
}
if err := validateBootstrapTarget(targetName, targetNamespace); err != nil {
return err
}
bootstrapOptions, err := w.resolveBootstrapOptions(ctx, targetName, targetNamespace)
if err != nil {
return err
}
repoPath, err := w.prepareBootstrapRepository(ctx)
if err != nil {
return err
}
if err := w.bootstrapPathIfNeeded(repoPath, normalizedPath, bootstrapOptions); err != nil {
return err
}
w.Log.Info("Prepared bootstrap files in worktree", "path", normalizedPath)
return nil
}
func normalizeBootstrapPath(path string) (string, error) {
normalizedPath := sanitizePath(path)
if strings.TrimSpace(path) != "" && normalizedPath == "" {
return "", fmt.Errorf("invalid path %q", path)
}
return normalizedPath, nil
}
func validateBootstrapTarget(targetName, targetNamespace string) error {
if strings.TrimSpace(targetName) == "" || strings.TrimSpace(targetNamespace) == "" {
return errors.New("target name and namespace must be set for path bootstrap")
}
return nil
}
func (w *BranchWorker) resolveBootstrapOptions(
ctx context.Context,
targetName string,
targetNamespace string,
) (pathBootstrapOptions, error) {
targetKey := types.NamespacedName{Name: targetName, Namespace: targetNamespace}
var target configv1alpha3.GitTarget
if err := w.Client.Get(ctx, targetKey, &target); err != nil {