From 4abecc8d963e0179dee2278adfb26c7fe6479a5d Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:42:21 -0700 Subject: [PATCH 1/6] Return dropped search-attribute registration error in ebb_and_flow spawnWorkflowWithActivities ignored the error from RegisterDefaultSearchAttributes, so a registration failure was silently lost and surfaced later as an obscure workflow-start failure. --- scenarios/ebb_and_flow.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scenarios/ebb_and_flow.go b/scenarios/ebb_and_flow.go index 3ef39bef..640643f1 100644 --- a/scenarios/ebb_and_flow.go +++ b/scenarios/ebb_and_flow.go @@ -281,7 +281,9 @@ func (e *ebbAndFlowExecutor) spawnWorkflowWithActivities( // Start workflow. run := e.NewRun(int(iteration)) - e.RegisterDefaultSearchAttributes(ctx) + if err := e.RegisterDefaultSearchAttributes(ctx); err != nil { + return fmt.Errorf("failed to register search attributes for iteration %d: %w", iteration, err) + } options := run.DefaultStartWorkflowOptions() options.ID = fmt.Sprintf("%s-track-%d", e.id, iteration) options.WorkflowExecutionErrorWhenAlreadyStarted = false From 3275fe052fcf4b2b2a4b5ac148a8a2750f7932e9 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:44:06 -0700 Subject: [PATCH 2/6] Surface failed ebbAndFlowTrack workflows instead of swallowing them spawnWorkflowWithActivities logged wf.Get failures and returned nil, so they never reached the MaxConsecutiveErrors guard and a failed workflow was still counted as completed (inflating TotalCompletedWorkflows, which post-scenario verification relies on). Return the error and gate the completion count on success; the backlog still drains either way so the controller's behavior is unchanged. --- scenarios/ebb_and_flow.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scenarios/ebb_and_flow.go b/scenarios/ebb_and_flow.go index 640643f1..e1297dec 100644 --- a/scenarios/ebb_and_flow.go +++ b/scenarios/ebb_and_flow.go @@ -305,10 +305,12 @@ func (e *ebbAndFlowExecutor) spawnWorkflowWithActivities( // Wait for workflow completion var result ebbandflow.WorkflowOutput err = wf.Get(ctx, &result) + // The batch has settled either way, so drain it from the backlog the + // controller tracks; only a successful workflow counts as completed. + e.completedActivities.Add(activities) if err != nil { - e.Logger.Errorf("ebbAndFlowTrack workflow failed for iteration %d: %v", iteration, err) + return fmt.Errorf("ebbAndFlowTrack workflow failed for iteration %d: %w", iteration, err) } - e.completedActivities.Add(activities) e.incrementTotalCompletedWorkflow() return nil From faa9609a2c9c3f292bf011a9abf2eddec8ecb764 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:44:40 -0700 Subject: [PATCH 3/6] Fix dropped terminate error and pointer log in ExecuteKitchenSinkWorkflow The client-actions goroutine logged the atomic.Pointer rather than the error value, and silently dropped the result of the cleanup TerminateWorkflow call. Log the actual error, and log the terminate failure (the goroutine has no return path for it). --- loadgen/scenario.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/loadgen/scenario.go b/loadgen/scenario.go index 26599332..ac5733fe 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -428,14 +428,13 @@ func (r *Run) ExecuteKitchenSinkWorkflow(ctx context.Context, options *KitchenSi err := executor.ExecuteClientSequence(cancelCtx, clientSeq) if err != nil { clientActionsErrPtr.Store(&err) - r.Logger.Error("Client actions failed: ", clientActionsErrPtr) + r.Logger.Error("Client actions failed: ", err) cancel() // TODO: Remove or change to "always terminate when exiting early" flag - err := r.Client.TerminateWorkflow( - ctx, options.StartOptions.ID, "", "client actions failed", nil) - if err != nil { - return + if terminateErr := r.Client.TerminateWorkflow( + ctx, options.StartOptions.ID, "", "client actions failed", nil); terminateErr != nil { + r.Logger.Errorf("Failed to terminate workflow after client actions failure: %v", terminateErr) } } }() From 4e7ea36c85450cc2c588d0278c7da5b2e31a4fc2 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:46:31 -0700 Subject: [PATCH 4/6] Stop double-handling terminal iteration errors in generic_executor The per-iteration goroutine logged the terminal failure and also sent it on doneCh, where Run returns it wrapped for its caller to report. Drop the goroutine log for the terminal branch (propagate only); keep the retry-branch log, whose transient error is superseded and never returned. --- loadgen/generic_executor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loadgen/generic_executor.go b/loadgen/generic_executor.go index df28e3c8..3f653992 100644 --- a/loadgen/generic_executor.go +++ b/loadgen/generic_executor.go @@ -151,11 +151,14 @@ func (g *genericRun) Run(ctx context.Context) error { backoff, retry := run.ShouldRetry(err) if retry { + // Transient: this attempt is superseded by the retry and + // never propagated, so log it here as the only record. err = fmt.Errorf("iteration %v encountered error: %w", run.Iteration, err) g.logger.Error(err) } else { + // Terminal: propagated via doneCh and reported by Run's + // caller, so don't also log it here. err = fmt.Errorf("iteration %v failed: %w", run.Iteration, err) - g.logger.Error(err) break retryLoop } From 3f265c34e8db2d0033477392497b9f148e993e0e Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:47:09 -0700 Subject: [PATCH 5/6] Branch on gRPC status code, not error string, in RegisterDefaultSearchAttributes Replace strings.Contains(err.Error(), ...) with status.Code(err) == codes.AlreadyExists, matching ensureNexusEndpoint in the same file. The second match string was a typo ("unavailble") that never matched, so no working behavior is lost. --- loadgen/scenario.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/loadgen/scenario.go b/loadgen/scenario.go index ac5733fe..accd3dd3 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -306,22 +306,10 @@ func (s *ScenarioInfo) RegisterDefaultSearchAttributes(ctx context.Context) erro }, Namespace: s.Namespace, }) - // Throw an error if the attributes could not be registered, but ignore already exists errs - alreadyExistsStrings := []string{ - "already exists", - "attributes mapping unavailble", - } - if err != nil { - isAlreadyExistsErr := false - for _, s := range alreadyExistsStrings { - if strings.Contains(err.Error(), s) { - isAlreadyExistsErr = true - break - } - } - if !isAlreadyExistsErr { - return fmt.Errorf("failed to register search attributes: %w", err) - } + // Throw an error if the attributes could not be registered, but ignore the + // case where they already exist (re-registration on a reused namespace). + if err != nil && status.Code(err) != codes.AlreadyExists { + return fmt.Errorf("failed to register search attributes: %w", err) } return nil } From e940ea43e9f4366bd7d47f4aa0f36224c905c922 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:47:23 -0700 Subject: [PATCH 6/6] Wrap cert-load error with %w in loadTLSConfig Preserve the error chain at this library boundary with %w, matching the sibling wraps in the same function (lines 93, 118). --- clioptions/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clioptions/client.go b/clioptions/client.go index 8aa4d289..959e24c4 100644 --- a/clioptions/client.go +++ b/clioptions/client.go @@ -64,7 +64,7 @@ func (c *ClientOptions) loadTLSConfig() (*tls.Config, error) { } cert, err := tls.LoadX509KeyPair(c.ClientCertPath, c.ClientKeyPath) if err != nil { - return nil, fmt.Errorf("failed to load certs: %s", err) + return nil, fmt.Errorf("failed to load certs: %w", err) } tlsConfig.Certificates = []tls.Certificate{cert} return tlsConfig, nil