From ca762fea56b3993dd853e549259668d948971fc6 Mon Sep 17 00:00:00 2001 From: Anjani Mallampati Date: Mon, 21 Sep 2026 22:56:48 -0700 Subject: [PATCH 1/3] SCIM performance testing --- benchmarktests/reports_test.go | 86 +++ benchmarktests/scim_helper.go | 503 ++++++++++++++++++ benchmarktests/target_scim_groups.go | 412 ++++++++++++++ benchmarktests/target_scim_users.go | 303 +++++++++++ test-fixtures/configs/scim.hcl | 160 ++++++ test-fixtures/configs/scim_membership.hcl | 101 ++++ test-fixtures/configs/scim_scale.hcl | 137 +++++ .../configs/scim_spike_user_adopt.hcl | 28 + .../configs/scim_spike_user_create.hcl | 28 + 9 files changed, 1758 insertions(+) create mode 100644 benchmarktests/scim_helper.go create mode 100644 benchmarktests/target_scim_groups.go create mode 100644 benchmarktests/target_scim_users.go create mode 100644 test-fixtures/configs/scim.hcl create mode 100644 test-fixtures/configs/scim_membership.hcl create mode 100644 test-fixtures/configs/scim_scale.hcl create mode 100644 test-fixtures/configs/scim_spike_user_adopt.hcl create mode 100644 test-fixtures/configs/scim_spike_user_create.hcl diff --git a/benchmarktests/reports_test.go b/benchmarktests/reports_test.go index 90b65c31..6aa523c7 100644 --- a/benchmarktests/reports_test.go +++ b/benchmarktests/reports_test.go @@ -8,6 +8,8 @@ import ( "reflect" "strings" "testing" + + vegeta "github.com/tsenart/vegeta/v12/lib" ) func TestReportJSONRoundTrip(t *testing.T) { @@ -52,3 +54,87 @@ func TestReportJSONRoundTrip(t *testing.T) { t.Fatalf("expected reports to be unchanged after round trip: %v", reports2) } } + +// TestSCIMSentinelAttribution verifies that the reporter's HasPrefix-based +// attribution correctly routes SCIM results to distinct buckets when sentinel +// query parameters are used to disambiguate same-method, same-path targets. +// +// The four cases exercised: +// +// user_create → URL .../Users?_w=create → must land in scim_user_create, not scim_user_adopt +// user_adopt → URL .../Users → must land in scim_user_adopt, not scim_user_create +// group_create → URL .../Groups?_w=create → must land in scim_group_create, not scim_group_create_empty +// group_empty → URL .../Groups?_w=empty → must land in scim_group_create_empty, not scim_group_create +func TestSCIMSentinelAttribution(t *testing.T) { + const addr = "http://127.0.0.1:8200" + const base = "/v1/identity/scim/v2" + + cases := []struct { + name string + resultURL string + targets []BenchmarkTarget // highest-weight first, matching BuildTargets sort order + expectBucket string + expectZero []string // buckets that must remain at 0 + }{ + { + name: "user_create sentinel matches create, not adopt", + resultURL: addr + base + "/Users?_w=create", + targets: []BenchmarkTarget{ + {Name: "scim_user_create", Method: "POST", PathPrefix: base + "/Users?_w=create", Weight: 21}, + {Name: "scim_user_adopt", Method: "POST", PathPrefix: base + "/Users", Weight: 19}, + }, + expectBucket: "scim_user_create", + expectZero: []string{"scim_user_adopt"}, + }, + { + name: "user_adopt bare URL matches adopt, not create", + resultURL: addr + base + "/Users", + targets: []BenchmarkTarget{ + {Name: "scim_user_create", Method: "POST", PathPrefix: base + "/Users?_w=create", Weight: 21}, + {Name: "scim_user_adopt", Method: "POST", PathPrefix: base + "/Users", Weight: 19}, + }, + expectBucket: "scim_user_adopt", + expectZero: []string{"scim_user_create"}, + }, + { + name: "group_create sentinel matches create, not empty", + resultURL: addr + base + "/Groups?_w=create", + targets: []BenchmarkTarget{ + {Name: "scim_group_create", Method: "POST", PathPrefix: base + "/Groups?_w=create", Weight: 20}, + {Name: "scim_group_create_empty", Method: "POST", PathPrefix: base + "/Groups?_w=empty", Weight: 10}, + }, + expectBucket: "scim_group_create", + expectZero: []string{"scim_group_create_empty"}, + }, + { + name: "group_create_empty sentinel matches empty, not create", + resultURL: addr + base + "/Groups?_w=empty", + targets: []BenchmarkTarget{ + {Name: "scim_group_create", Method: "POST", PathPrefix: base + "/Groups?_w=create", Weight: 20}, + {Name: "scim_group_create_empty", Method: "POST", PathPrefix: base + "/Groups?_w=empty", Weight: 10}, + }, + expectBucket: "scim_group_create_empty", + expectZero: []string{"scim_group_create"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tm := &TargetMulti{targets: tc.targets} + r := newReporter(tm, nil) + r.clientAddr = addr + + r.Add(&vegeta.Result{Method: "POST", URL: tc.resultURL}) + + if r.metrics[tc.expectBucket].Requests != 1 { + t.Errorf("bucket %q: want 1 request, got %d", tc.expectBucket, r.metrics[tc.expectBucket].Requests) + } + for _, other := range tc.expectZero { + if r.metrics[other].Requests != 0 { + t.Errorf("bucket %q: want 0 requests (no attribution bleed), got %d", + other, r.metrics[other].Requests) + } + } + }) + } +} diff --git a/benchmarktests/scim_helper.go b/benchmarktests/scim_helper.go new file mode 100644 index 00000000..12693d22 --- /dev/null +++ b/benchmarktests/scim_helper.go @@ -0,0 +1,503 @@ +// Copyright IBM Corp. 2022, 2026 +// SPDX-License-Identifier: MPL-2.0 + +package benchmarktests + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "strconv" + "sync" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/api" +) + +// generateUUID is a thin wrapper so callers don't need to import go-uuid directly. +func generateUUID() (string, error) { + return uuid.GenerateUUID() +} + +const ( + scimPolicyName = "scim-bench-policy" + scimMountBase = "userpass" + scimUserPassword = "scim-bench-pw" + + // scimUserSchemaCore is the SCIM 2.0 User schema ID. + scimUserSchemaCore = "urn:ietf:params:scim:schemas:core:2.0:User" + scimGroupSchemaCore = "urn:ietf:params:scim:schemas:core:2.0:Group" +) + +// scimClientPolicy is the minimum ACL policy a SCIM client token needs. +const scimClientPolicy = ` +path "identity/scim/v2/Users" { + capabilities = ["create","read","update","patch","delete","list"] +} +path "identity/scim/v2/Users/*" { + capabilities = ["create","read","update","patch","delete","list"] +} +path "identity/scim/v2/Groups" { + capabilities = ["create","read","update","patch","delete","list"] +} +path "identity/scim/v2/Groups/*" { + capabilities = ["create","read","update","patch","delete","list"] +} +path "identity/scim/v2/ResourceTypes" { capabilities = ["read"] } +path "identity/scim/v2/ResourceTypes/*" { capabilities = ["read"] } +path "identity/scim/v2/Schemas" { capabilities = ["read"] } +path "identity/scim/v2/Schemas/*" { capabilities = ["read"] } +path "identity/scim/v2/ServiceProviderConfig" { capabilities = ["read"] } +path "identity/scim/client" { + capabilities = ["create","read","update","delete","list","patch"] +} +path "identity/scim/client/*" { + capabilities = ["create","read","update","delete","list","patch"] +} +` + +// ── Shared SCIM client singleton ───────────────────────────────────────────── +// +// All scim_users and scim_groups test blocks in a single benchmark run share +// one SCIM client. This mirrors a real IdP: a single Entra/Okta tenant has one +// SCIM provisioning credential, not one per operation type. +// +// Sharing also fixes a correctness problem: when each block owns its own client, +// concurrent cleanup calls race — one block's delete-linked-resources=true can +// interfere with another block's per-resource SCIM DELETE, producing +// "managed by a different SCIM client" 403s. +// +// Implementation: a package-level map protected by a mutex. The first Setup() +// call creates the client; all subsequent ones reuse it. A reference counter +// tracks how many test blocks are using the shared instance. The last Cleanup() +// call (refCount reaching zero) performs the actual Vault teardown. + +// scimSharedClient holds the state shared across all test blocks in one run. +type scimSharedClient struct { + runID string + scimToken string + aliasMountAccessor string + namespace string + refCount int // number of test blocks currently holding this instance +} + +var ( + scimSharedMu sync.Mutex + scimSharedClients = map[string]*scimSharedClient{} // key = sharedKey +) + +// scimSharedKey returns the map key used to look up the shared client. +// We derive it from the admin client's address + namespace so that different +// clusters in the same process (e.g. parallel test suites) get distinct clients. +func scimSharedKey(client *api.Client) string { + ns := client.Headers().Get("X-Vault-Namespace") + return client.Address() + "|" + ns +} + +// scimAcquireSharedClient returns the shared SCIM client for this cluster, +// creating it if this is the first caller. The caller must eventually call +// scimReleaseSharedClient to decrement the reference count. +// +// The singleton is keyed by address+namespace (one client per cluster per +// process), but uses a fresh UUID as the runID each time a new singleton is +// created. The UUID scopes all Vault object names (entities, groups, mounts, +// policies) so that: +// - Different workload blocks within the same run share one client and one +// runID — their name pools are disjoint because each block uses a +// workload-specific prefix (scimCreateGroupDisplayName vs scimAdoptGroupName +// vs scimCreateUserName, etc.). +// - A new run always gets a new runID, so there are no name collisions with +// any state a previous run may have left behind on a persistent cluster. +func scimAcquireSharedClient(client *api.Client) (*scimSharedClient, error) { + key := scimSharedKey(client) + + scimSharedMu.Lock() + defer scimSharedMu.Unlock() + + if sc, ok := scimSharedClients[key]; ok { + sc.refCount++ + return sc, nil + } + + // First caller for this cluster in this process — generate a fresh runID + // and create the Vault objects. + runID, err := generateUUID() + if err != nil { + return nil, fmt.Errorf("scim shared client: generating run ID: %w", err) + } + ns := client.Headers().Get("X-Vault-Namespace") + + scimToken, accessor, err := SCIMClientSetup(client, runID, true, true) + if err != nil { + return nil, fmt.Errorf("scim shared client setup: %w", err) + } + + sc := &scimSharedClient{ + runID: runID, + scimToken: scimToken, + aliasMountAccessor: accessor, + namespace: ns, + refCount: 1, + } + scimSharedClients[key] = sc + return sc, nil +} + +// scimReleaseSharedClient decrements the reference count for the shared client. +// When the count reaches zero it tears down all Vault objects created by +// scimAcquireSharedClient. +func scimReleaseSharedClient(client *api.Client) error { + key := scimSharedKey(client) + + scimSharedMu.Lock() + defer scimSharedMu.Unlock() + + sc, ok := scimSharedClients[key] + if !ok { + // Already cleaned up or never created — nothing to do. + return nil + } + + sc.refCount-- + if sc.refCount > 0 { + return nil + } + + // Last holder — tear down. + delete(scimSharedClients, key) + return SCIMCleanup(client, sc.runID) +} + +// ── Name helpers ────────────────────────────────────────────────────────────── + +// scimMountPath returns the userpass mount path for a given runID. +func scimMountPath(runID string) string { + return scimMountBase + "-scim-" + runID +} + +// scimClientName returns the SCIM client config name for a given runID. +func scimClientName(runID string) string { + return "bench-scim-client-" + runID +} + +// scimClientEntityName returns the deterministic Vault entity name for the SCIM +// client itself (the entity that holds the access_grant_principal). +func scimClientEntityName(runID string) string { + return "bench-scim-entity-" + runID +} + +// scimClientAliasName returns the username used for the userpass alias of the SCIM +// client — this is the identity that logs in to get the SCIM token. +func scimClientAliasName(runID string) string { + return "bench-scim-alias-" + runID +} + +// scimAdoptUserAliasName returns the alias name (== SCIM userName) for the +// i-th pre-seeded adoption target entity. Must be distinct from the entity +// name (scimAdoptEntityName), mirroring how real IdP-created logins work: +// +// entity name = "bench-adopt-entity--" (internal Vault name) +// alias name = "bench-adopt-user--@example.com" (IdP identity) +func scimAdoptUserAliasName(runID string, i int) string { + return "bench-adopt-user-" + runID + "-" + strconv.Itoa(i) + "@example.com" +} + +// scimAdoptEntityName returns the internal Vault entity name for the i-th +// pre-seeded adoption target. Intentionally differs from the alias name. +func scimAdoptEntityName(runID string, i int) string { + return "bench-adopt-entity-" + runID + "-" + strconv.Itoa(i) +} + +// scimAdoptGroupName returns the Vault group name for the i-th pre-seeded +// adoption target group. Group adoption matches on displayName == Vault group +// name, so the name IS also used as the SCIM displayName in the PUT body. +func scimAdoptGroupName(runID string, i int) string { + return "bench-adopt-group-" + runID + "-" + strconv.Itoa(i) +} + +// scimCreateUserName returns the SCIM userName for the i-th freshly-created +// user in the given scope. scope differentiates pools that share the same runID +// but must not collide — e.g. "uc" for user_create vs "gm" for group members +// seeded by group_create. +func scimCreateUserName(runID, scope string, i int) string { + return "bench-create-user-" + scope + "-" + runID + "-" + strconv.Itoa(i) + "@example.com" +} + +// scimCreateGroupDisplayName returns the SCIM displayName for the i-th +// freshly-created group in the given scope. scope differentiates pools that +// share the same runID — e.g. "gc" for group_create vs "ge" for group_create_empty. +func scimCreateGroupDisplayName(runID, scope string, i int) string { + return "bench-create-group-" + scope + "-" + runID + "-" + strconv.Itoa(i) +} + +// ── Vault provisioning ──────────────────────────────────────────────────────── + +// SCIMClientSetup provisions all the Vault objects the SCIM client needs: +// +// 1. ACL policy (scimClientPolicy) +// 2. Userpass auth mount scoped to this run +// 3. A Vault entity (access_grant_principal) with the policy attached +// 4. A userpass alias + credential so the entity can log in +// 5. The identity/scim/client config +// +// Returns the SCIM client token (already authenticated as the SCIM entity) +// and the userpass mount accessor (needed to create adoption-target aliases). +// Must be called with an admin/root token. +func SCIMClientSetup(client *api.Client, runID string, allowUserAdoption, allowGroupAdoption bool) (scimToken, aliasMountAccessor string, err error) { + policyName := scimPolicyName + "-" + runID + if err = client.Sys().PutPolicy(policyName, scimClientPolicy); err != nil { + return "", "", fmt.Errorf("scim setup: writing policy %q: %w", policyName, err) + } + + mountPath := scimMountPath(runID) + if err = client.Sys().EnableAuthWithOptions(mountPath, &api.EnableAuthOptions{ + Type: "userpass", + Config: api.AuthConfigInput{ + TokenType: "service", + }, + }); err != nil { + return "", "", fmt.Errorf("scim setup: enabling userpass mount %q: %w", mountPath, err) + } + + mounts, err := client.Sys().ListAuth() + if err != nil { + return "", "", fmt.Errorf("scim setup: listing auth mounts: %w", err) + } + mount, ok := mounts[mountPath+"/"] + if !ok { + return "", "", fmt.Errorf("scim setup: auth mount %q not found after enable", mountPath) + } + aliasMountAccessor = mount.Accessor + + // Create the SCIM client entity. + entityResp, err := client.Logical().Write("identity/entity", map[string]any{ + "name": scimClientEntityName(runID), + "policies": []string{policyName}, + }) + if err != nil { + return "", "", fmt.Errorf("scim setup: creating SCIM client entity: %w", err) + } + entityID, err := idFromResponse(entityResp) + if err != nil { + return "", "", fmt.Errorf("scim setup: reading SCIM client entity id: %w", err) + } + + // Create userpass alias + credential so the entity can authenticate. + aliasName := scimClientAliasName(runID) + if _, err = client.Logical().Write("identity/entity-alias", map[string]any{ + "name": aliasName, + "canonical_id": entityID, + "mount_accessor": aliasMountAccessor, + }); err != nil { + return "", "", fmt.Errorf("scim setup: creating SCIM client entity-alias: %w", err) + } + userPath := filepath.ToSlash(filepath.Join("auth", mountPath, "users", aliasName)) + if _, err = client.Logical().Write(userPath, map[string]any{ + "password": scimUserPassword, + }); err != nil { + return "", "", fmt.Errorf("scim setup: creating userpass user %q: %w", aliasName, err) + } + + // Register the SCIM client config. + clientCfg := map[string]any{ + "access_grant_principal": entityID, + "allow_user_adoption": allowUserAdoption, + "allow_group_adoption": allowGroupAdoption, + "alias_mount_accessor": aliasMountAccessor, + } + if _, err = client.Logical().Write("identity/scim/client/"+scimClientName(runID), clientCfg); err != nil { + return "", "", fmt.Errorf("scim setup: writing scim/client config: %w", err) + } + + // Log in as the SCIM client and return its token. + loginPath := filepath.ToSlash(filepath.Join("auth", mountPath, "login", aliasName)) + loginResp, err := client.Logical().Write(loginPath, map[string]any{ + "password": scimUserPassword, + }) + if err != nil { + return "", "", fmt.Errorf("scim setup: logging in as SCIM client: %w", err) + } + if loginResp == nil || loginResp.Auth == nil || loginResp.Auth.ClientToken == "" { + return "", "", fmt.Errorf("scim setup: login returned no token") + } + + return loginResp.Auth.ClientToken, aliasMountAccessor, nil +} + +// SCIMCleanup removes all objects created by SCIMClientSetup (policy, userpass +// mount, entity, scim client config). Call with admin/root token. +// +// The SCIM client is deleted with delete-linked-resources=true so that any +// resources still owned by it (e.g. users/groups created during the run but not +// individually deleted beforehand) are removed atomically by Vault. +func SCIMCleanup(client *api.Client, runID string) error { + var errs []string + + // delete-linked-resources=true instructs Vault to remove all SCIM-owned + // entities and groups before deleting the client record. A plain DELETE + // returns HTTP 400 "SCIM client has linked resources" whenever the benchmark + // has created any users/groups during the run. + // The Vault Go SDK does not expose query-param DELETE, so we use a raw HTTP + // request — the same pattern used in DebugInfo and seedSCIMUsers. + if err := scimClientDeleteWithLinkedResources(client, scimClientName(runID)); err != nil { + errs = append(errs, fmt.Sprintf("delete scim client: %v", err)) + } + if _, err := client.Logical().Delete("identity/entity/name/" + scimClientEntityName(runID)); err != nil { + errs = append(errs, fmt.Sprintf("delete scim entity: %v", err)) + } + if err := client.Sys().DisableAuth(scimMountPath(runID)); err != nil { + errs = append(errs, fmt.Sprintf("disable userpass mount: %v", err)) + } + if err := client.Sys().DeletePolicy(scimPolicyName + "-" + runID); err != nil { + errs = append(errs, fmt.Sprintf("delete policy: %v", err)) + } + + if len(errs) > 0 { + return fmt.Errorf("scim cleanup errors: %v", errs) + } + return nil +} + +// scimClientDeleteWithLinkedResources sends: +// +// DELETE /v1/identity/scim/client/?delete-linked-resources=true +// +// using the admin token on client. The SDK's Logical().Delete() does not +// support query parameters, so we build a raw HTTP request instead. +func scimClientDeleteWithLinkedResources(client *api.Client, clientName string) error { + url := client.Address() + "/v1/identity/scim/client/" + clientName + "?delete-linked-resources=true" + req, err := http.NewRequest(http.MethodDelete, url, bytes.NewReader(nil)) + if err != nil { + return fmt.Errorf("building DELETE request for scim client %q: %w", clientName, err) + } + req.Header.Set("X-Vault-Token", client.Token()) + ns := client.Headers().Get("X-Vault-Namespace") + if ns != "" { + req.Header.Set("X-Vault-Namespace", ns) + } + resp, err := client.CloneConfig().HttpClient.Do(req) + if err != nil { + return fmt.Errorf("DELETE scim client %q: %w", clientName, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("DELETE scim client %q: unexpected status %d: %s", clientName, resp.StatusCode, body) + } + return nil +} + +// scimDeleteViaSCIMAPI deletes a single SCIM resource (User or Group) by ID +// using the SCIM token. This is required for resources that are already +// SCIM-managed: the admin identity API returns 403/500 for those. +// resourceType must be "Users" or "Groups". +func scimDeleteViaSCIMAPI(client *api.Client, scimToken, namespace, resourceType, resourceID string) error { + url := client.Address() + "/v1/identity/scim/v2/" + resourceType + "/" + resourceID + req, err := http.NewRequest(http.MethodDelete, url, bytes.NewReader(nil)) + if err != nil { + return fmt.Errorf("building DELETE request for %s/%s: %w", resourceType, resourceID, err) + } + for k, vals := range scimHeader(scimToken, namespace) { + for _, v := range vals { + req.Header.Set(k, v) + } + } + resp, err := client.CloneConfig().HttpClient.Do(req) + if err != nil { + return fmt.Errorf("DELETE %s/%s: %w", resourceType, resourceID, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("DELETE %s/%s: unexpected status %d: %s", resourceType, resourceID, resp.StatusCode, body) + } + return nil +} + +// scimDeletePhase bulk-deletes SCIM resources using the SCIM token (required +// for SCIM-owned entities/groups that the admin identity API refuses to delete). +func scimDeletePhase(logger hclog.Logger, phase, resourceType string, client *api.Client, scimToken, namespace string, ids []string) error { + return runPhase(logger, phase, identityConcurrency, len(ids), func(i int) error { + return scimDeleteViaSCIMAPI(client, scimToken, namespace, resourceType, ids[i]) + }) +} + +// scimHeader returns the HTTP headers for SCIM requests: Vault token + namespace. +func scimHeader(token, namespace string) http.Header { + h := http.Header{ + "X-Vault-Token": []string{token}, + "Content-Type": []string{"application/json"}, + } + if namespace != "" { + h["X-Vault-Namespace"] = []string{namespace} + } + return h +} + +// debugLogSCIMResponse reads the body of a raw *api.Response, logs it at DEBUG +// level, and returns the status code and raw body for the caller to assert on. +// The body is consumed and the response is closed. +func debugLogSCIMResponse(resp *http.Response) (statusCode int, body []byte) { + if resp == nil { + return 0, nil + } + defer resp.Body.Close() + body, _ = io.ReadAll(resp.Body) + return resp.StatusCode, body +} + +// ── SCIM request body builders ──────────────────────────────────────────────── + +// scimUserCreateBody returns the JSON body for a SCIM POST /Users (creation). +func scimUserCreateBody(userName, externalID string) []byte { + b, _ := json.Marshal(map[string]any{ + "schemas": []string{scimUserSchemaCore}, + "userName": userName, + "externalId": externalID, + "active": true, + }) + return b +} + +// scimUserAdoptBody returns the JSON body for a SCIM POST /Users (adoption). +// userName must match an existing unmanaged alias name on the SCIM client's +// alias_mount_accessor mount. +func scimUserAdoptBody(userName, externalID string) []byte { + b, _ := json.Marshal(map[string]any{ + "schemas": []string{scimUserSchemaCore}, + "userName": userName, + "externalId": externalID, + "active": true, + }) + return b +} + +// scimGroupCreateBody returns the JSON body for a SCIM POST /Groups (creation). +// members is a slice of Vault entity IDs that are already SCIM-owned. +func scimGroupCreateBody(displayName string, memberIDs []string) []byte { + members := make([]map[string]string, len(memberIDs)) + for i, id := range memberIDs { + members[i] = map[string]string{"value": id} + } + b, _ := json.Marshal(map[string]any{ + "schemas": []string{scimGroupSchemaCore}, + "displayName": displayName, + "members": members, + }) + return b +} + +// scimGroupAdoptBody returns the JSON body for a SCIM PUT /Groups/ (adoption). +// displayName must match the Vault group's name exactly. +func scimGroupAdoptBody(displayName string) []byte { + b, _ := json.Marshal(map[string]any{ + "schemas": []string{scimGroupSchemaCore}, + "displayName": displayName, + }) + return b +} + diff --git a/benchmarktests/target_scim_groups.go b/benchmarktests/target_scim_groups.go new file mode 100644 index 00000000..1050f57e --- /dev/null +++ b/benchmarktests/target_scim_groups.go @@ -0,0 +1,412 @@ +// Copyright IBM Corp. 2022, 2026 +// SPDX-License-Identifier: MPL-2.0 + +package benchmarktests + +// target_scim_groups.go implements four SCIM group workloads: +// +// - group_create: POST /scim/v2/Groups with a fresh unique displayName each time. +// Optional members_per_group controls how many SCIM-owned entity IDs appear in +// each POST body (tests the "groups with many memberships" case). +// +// - group_create_empty: POST /scim/v2/Groups with no members. Baseline for +// comparing membership overhead against group_create. +// +// - group_adopt: PUT /scim/v2/Groups/ with displayName matching a pre-seeded +// unmanaged group. Setup seeds group_count groups via the admin identity API. +// Note: group adoption uses PUT (not POST), targeting the group's Vault ID. +// +// - group_list: GET /scim/v2/Groups with a round-robin selection of filter +// expressions (including the bare, no-filter case). +// +// All four workloads share a single SCIM client per benchmark run +// (see scim_helper.go — scimAcquireSharedClient). This mirrors a real IdP: +// one Entra/Okta tenant has one credential, not one per operation type. + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/http" + "net/url" + "sync/atomic" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/vault/api" + vegeta "github.com/tsenart/vegeta/v12/lib" +) + +const ( + SCIMGroupsTestType = "scim_groups" + + scimGroupWorkloadCreate = "group_create" + scimGroupWorkloadCreateEmpty = "group_create_empty" + scimGroupWorkloadAdopt = "group_adopt" + scimGroupWorkloadList = "group_list" +) + +func init() { + TestList[SCIMGroupsTestType] = func() BenchmarkBuilder { return &SCIMGroups{} } +} + +// SCIMGroups implements BenchmarkBuilder for SCIM group workloads. +type SCIMGroups struct { + pathPrefix string + header http.Header + logger hclog.Logger + + // sc is the shared SCIM client; all workloads in a run share one client. + sc *scimSharedClient + namespace string + + // group_create / group_create_empty: pool of unique displayNames. + createGroupNames []string + // group_create: pool of SCIM-owned entity IDs for membership bodies. + scimEntityIDs []string + + // group_adopt: (vaultGroupID, displayName) pairs for the PUT request. + adoptGroupIDs []string + adoptGroupNames []string + + // group_list: pre-built URL strings. + listURLs []string + + atomicIdx int64 + + config *SCIMGroupsConfig +} + +// SCIMGroupsConfig holds all HCL-configurable parameters for the SCIM groups target. +type SCIMGroupsConfig struct { + // Workload selects which group workload to run. + // One of: "group_create" | "group_create_empty" | "group_adopt" | "group_list" + Workload string `hcl:"workload,optional"` + + // GroupCount is: + // group_create / group_create_empty: size of the unique-name pool + // group_adopt: number of pre-seeded unmanaged groups to create + // group_list: ignored + GroupCount int `hcl:"group_count,optional"` + + // MembersPerGroup is the number of SCIM-owned entity IDs to include in each + // group_create POST body. 0 = no members (same as group_create_empty). + MembersPerGroup int `hcl:"members_per_group,optional"` + + // UserSeedCount is the number of SCIM-owned users to create during setup for + // use as group members. Only meaningful when MembersPerGroup > 0. + // Defaults to MembersPerGroup when not set. + UserSeedCount int `hcl:"user_seed_count,optional"` + + // Filters is the list of SCIM filter query-strings cycled for group_list. + // An empty string means "no filter". Defaults to [""] when omitted. + Filters []string `hcl:"filters,optional"` +} + +func (s *SCIMGroups) ParseConfig(body hcl.Body) error { + testConfig := &struct { + Config *SCIMGroupsConfig `hcl:"config,block"` + }{ + Config: &SCIMGroupsConfig{ + Workload: scimGroupWorkloadCreate, + GroupCount: 100, + MembersPerGroup: 0, + Filters: []string{""}, + }, + } + + diags := gohcl.DecodeBody(body, nil, testConfig) + if diags.HasErrors() { + return fmt.Errorf("error decoding scim_groups config: %v", diags) + } + + c := testConfig.Config + s.config = c + + switch c.Workload { + case scimGroupWorkloadCreate, scimGroupWorkloadCreateEmpty, scimGroupWorkloadAdopt: + if c.GroupCount <= 0 { + return fmt.Errorf("scim_groups: group_count must be > 0 for workload %q", c.Workload) + } + case scimGroupWorkloadList: + if len(c.Filters) == 0 { + c.Filters = []string{""} + } + default: + return fmt.Errorf("scim_groups: invalid workload %q; must be one of %q, %q, %q, %q", + c.Workload, scimGroupWorkloadCreate, scimGroupWorkloadCreateEmpty, scimGroupWorkloadAdopt, scimGroupWorkloadList) + } + + if c.MembersPerGroup < 0 { + return fmt.Errorf("scim_groups: members_per_group must be >= 0") + } + if c.UserSeedCount == 0 && c.MembersPerGroup > 0 { + c.UserSeedCount = c.MembersPerGroup + } + + return nil +} + +func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig *TopLevelTargetConfig) (BenchmarkBuilder, error) { + s.logger = targetLogger.Named(SCIMGroupsTestType) + + // Acquire the shared SCIM client. + s.logger.Info("scim_groups setup: acquiring shared SCIM client", "workload", s.config.Workload) + sc, err := scimAcquireSharedClient(client) + if err != nil { + return nil, fmt.Errorf("scim_groups setup: %w", err) + } + + result := &SCIMGroups{ + sc: sc, + config: s.config, + logger: s.logger, + namespace: sc.namespace, + pathPrefix: "/v1/identity/scim/v2", + } + + switch s.config.Workload { + + case scimGroupWorkloadCreate, scimGroupWorkloadCreateEmpty: + // Scope "gc" vs "ge" keeps group_create and group_create_empty pools + // distinct — both share the same runID but must not create the same name. + scope := "gc" + if s.config.Workload == scimGroupWorkloadCreateEmpty { + scope = "ge" + } + result.createGroupNames = make([]string, s.config.GroupCount) + for i := range s.config.GroupCount { + result.createGroupNames[i] = scimCreateGroupDisplayName(sc.runID, scope, i) + } + s.logger.Info("scim_groups setup: group_create pool ready", + "pool_size", s.config.GroupCount, "members_per_group", s.config.MembersPerGroup, + ) + + // group_create_empty has MembersPerGroup=0 so this block is skipped. + if s.config.MembersPerGroup > 0 { + seedCount := s.config.UserSeedCount + if seedCount < s.config.MembersPerGroup { + seedCount = s.config.MembersPerGroup + } + s.logger.Info("scim_groups setup: seeding SCIM-owned users for group membership", "count", seedCount) + entityIDs, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, sc.namespace, seedCount) + if err != nil { + return nil, fmt.Errorf("scim_groups setup: seeding member users: %w", err) + } + result.scimEntityIDs = entityIDs + s.logger.Info("scim_groups setup: member users seeded", "count", len(entityIDs)) + } + + case scimGroupWorkloadAdopt: + s.logger.Info("scim_groups setup: seeding unmanaged adoption groups", "count", s.config.GroupCount) + groupIDs := make([]string, s.config.GroupCount) + groupNames := make([]string, s.config.GroupCount) + + err = runPhase(s.logger, "seed adoption groups", identityConcurrency, s.config.GroupCount, func(i int) error { + name := scimAdoptGroupName(sc.runID, i) + resp, err := client.Logical().Write("identity/group", map[string]any{ + "name": name, + "type": "internal", + }) + if err != nil { + return fmt.Errorf("creating adopt group %d (%q): %w", i, name, err) + } + id, err := idFromResponse(resp) + if err != nil { + return fmt.Errorf("reading adopt group id %d: %w", i, err) + } + groupIDs[i] = id + groupNames[i] = name + s.logger.Debug("scim_groups setup: seeded adoption group", + "index", i, "group_name", name, "group_id", id, + ) + return nil + }, "total", s.config.GroupCount) + if err != nil { + return nil, fmt.Errorf("scim_groups setup: seeding adoption groups: %w", err) + } + result.adoptGroupIDs = groupIDs + result.adoptGroupNames = groupNames + s.logger.Info("scim_groups setup: adoption groups seeded", "count", s.config.GroupCount) + + case scimGroupWorkloadList: + // Nothing to seed. + } + + result.header = scimHeader(sc.scimToken, sc.namespace) + + if s.config.Workload == scimGroupWorkloadList { + result.listURLs = make([]string, len(s.config.Filters)) + for i, f := range s.config.Filters { + u := "/v1/identity/scim/v2/Groups" + if f != "" { + u += "?filter=" + url.QueryEscape(f) + } + result.listURLs[i] = u + } + s.logger.Info("scim_groups setup: list URL variants built", "count", len(result.listURLs)) + } + + return result, nil +} + +func (s *SCIMGroups) Target(client *api.Client) vegeta.Target { + switch s.config.Workload { + + case scimGroupWorkloadCreate: + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + poolIdx := int(idx % int64(len(s.createGroupNames))) + displayName := s.createGroupNames[poolIdx] + + var memberIDs []string + if s.config.MembersPerGroup > 0 && len(s.scimEntityIDs) > 0 { + n := len(s.scimEntityIDs) + memberIDs = make([]string, s.config.MembersPerGroup) + for m := range s.config.MembersPerGroup { + memberIDs[m] = s.scimEntityIDs[(int(poolIdx)*s.config.MembersPerGroup+m)%n] + } + } + + s.logger.Debug("scim_groups target: group_create", + "pool_idx", poolIdx, "display_name", displayName, "member_count", len(memberIDs), + ) + + // Sentinel ?_w=create distinguishes this target from group_create_empty + // in the reporter. group_create (w=20) sorts before group_create_empty + // (w=10), so the reporter checks this longer prefix first. + return vegeta.Target{ + Method: http.MethodPost, + URL: client.Address() + s.pathPrefix + "/Groups?_w=create", + Header: s.header, + Body: scimGroupCreateBody(displayName, memberIDs), + } + + case scimGroupWorkloadCreateEmpty: + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + poolIdx := int(idx % int64(len(s.createGroupNames))) + displayName := s.createGroupNames[poolIdx] + + s.logger.Debug("scim_groups target: group_create_empty", + "pool_idx", poolIdx, "display_name", displayName, + ) + + // ?_w=empty is the sentinel for this target. It must NOT be a string + // prefix of ?_w=create — and it isn't. The reporter checks group_create's + // prefix first (higher weight), fails to match, then reaches this prefix. + return vegeta.Target{ + Method: http.MethodPost, + URL: client.Address() + s.pathPrefix + "/Groups?_w=empty", + Header: s.header, + Body: scimGroupCreateBody(displayName, nil), + } + + case scimGroupWorkloadAdopt: + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + poolIdx := int(idx % int64(len(s.adoptGroupIDs))) + groupID := s.adoptGroupIDs[poolIdx] + displayName := s.adoptGroupNames[poolIdx] + + s.logger.Debug("scim_groups target: group_adopt", + "pool_idx", poolIdx, "group_id", groupID, "display_name", displayName, + ) + + return vegeta.Target{ + Method: http.MethodPut, + URL: client.Address() + s.pathPrefix + "/Groups/" + groupID, + Header: s.header, + Body: scimGroupAdoptBody(displayName), + } + + default: // scimGroupWorkloadList + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + urlIdx := int(idx % int64(len(s.listURLs))) + + s.logger.Debug("scim_groups target: group_list", "url_idx", urlIdx, "url", s.listURLs[urlIdx]) + + return vegeta.Target{ + Method: http.MethodGet, + URL: client.Address() + s.listURLs[urlIdx], + Header: s.header, + } + } +} + +func (s *SCIMGroups) Cleanup(client *api.Client) error { + s.logger.Info("scim_groups cleanup: releasing shared SCIM client reference") + // All SCIM-owned resources (created users, created groups, adopted users, + // adopted groups) belong to the single shared client. The last block to + // release calls SCIMCleanup with delete-linked-resources=true, which removes + // everything in one atomic Vault operation. No per-resource delete loops + // are needed and concurrent cleanup races are impossible. + return scimReleaseSharedClient(client) +} + +func (s *SCIMGroups) GetTargetInfo() TargetInfo { + switch s.config.Workload { + case scimGroupWorkloadCreate: + return TargetInfo{method: http.MethodPost, pathPrefix: s.pathPrefix + "/Groups?_w=create"} + case scimGroupWorkloadCreateEmpty: + return TargetInfo{method: http.MethodPost, pathPrefix: s.pathPrefix + "/Groups?_w=empty"} + case scimGroupWorkloadAdopt: + return TargetInfo{method: http.MethodPut, pathPrefix: s.pathPrefix + "/Groups"} + default: // scimGroupWorkloadList + return TargetInfo{method: http.MethodGet, pathPrefix: s.pathPrefix + "/Groups"} + } +} + +func (s *SCIMGroups) Flags(fs *flag.FlagSet) {} + +// seedSCIMUsers creates SCIM-owned users via the SCIM API and returns their +// Vault entity IDs. Used as group members in group_create. +func seedSCIMUsers(logger hclog.Logger, adminClient *api.Client, scimToken, runID, namespace string, count int) ([]string, error) { + entityIDs := make([]string, count) + header := scimHeader(scimToken, namespace) + + err := runPhase(logger, "seed scim users for members", identityConcurrency, count, func(i int) error { + // Scope "gm" (group members) avoids collision with user_create's "uc" pool — + // both share the same runID within a benchmark run. + userName := scimCreateUserName(runID, "gm", i) + externalID := "member-ext-" + userName + body := scimUserCreateBody(userName, externalID) + + req, err := http.NewRequest(http.MethodPost, + adminClient.Address()+"/v1/identity/scim/v2/Users", + bytes.NewReader(body), + ) + if err != nil { + return fmt.Errorf("building seed user request %d: %w", i, err) + } + for k, vals := range header { + for _, v := range vals { + req.Header.Set(k, v) + } + } + + resp, err := adminClient.CloneConfig().HttpClient.Do(req) + if err != nil { + return fmt.Errorf("posting seed user %d: %w", i, err) + } + statusCode, respBody := debugLogSCIMResponse(resp) + logger.Debug("seed scim user response", + "index", i, "user_name", userName, "status", statusCode, "body", string(respBody), + ) + if statusCode != http.StatusCreated { + return fmt.Errorf("seed user %d (%q): unexpected status %d, body: %s", i, userName, statusCode, respBody) + } + + var parsed map[string]any + if err := json.Unmarshal(respBody, &parsed); err != nil { + return fmt.Errorf("parsing seed user response %d: %w", i, err) + } + id, ok := parsed["id"].(string) + if !ok || id == "" { + return fmt.Errorf("seed user response %d missing id field", i) + } + entityIDs[i] = id + return nil + }, "total", count) + + return entityIDs, err +} diff --git a/benchmarktests/target_scim_users.go b/benchmarktests/target_scim_users.go new file mode 100644 index 00000000..86d73fe7 --- /dev/null +++ b/benchmarktests/target_scim_users.go @@ -0,0 +1,303 @@ +// Copyright IBM Corp. 2022, 2026 +// SPDX-License-Identifier: MPL-2.0 + +package benchmarktests + +// target_scim_users.go implements three SCIM user workloads: +// +// - user_create: POST /scim/v2/Users with a fresh unique userName each time. +// Each request sends a name from a pre-generated pool so Vault always +// creates a new entity. No two requests in one pass share a name. +// +// - user_adopt: POST /scim/v2/Users with a userName matching a pre-seeded +// unmanaged alias. Setup seeds user_count entities (entity name ≠ alias +// name, matching real IdP login behaviour). The attack phase adopts each +// one exactly once per cycle through the pool. +// +// - user_list: GET /scim/v2/Users with a round-robin selection of filter +// expressions (including the bare, no-filter case). +// +// All three workloads share a single SCIM client per benchmark run +// (see scim_helper.go — scimAcquireSharedClient). This mirrors a real IdP: +// one Entra/Okta tenant has one credential, not one per operation type. + +import ( + "flag" + "fmt" + "net/http" + "net/url" + "sync/atomic" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/vault/api" + vegeta "github.com/tsenart/vegeta/v12/lib" +) + +const ( + SCIMUsersTestType = "scim_users" + + scimUserWorkloadCreate = "user_create" + scimUserWorkloadAdopt = "user_adopt" + scimUserWorkloadList = "user_list" +) + +func init() { + TestList[SCIMUsersTestType] = func() BenchmarkBuilder { return &SCIMUsers{} } +} + +// SCIMUsers implements BenchmarkBuilder for SCIM user workloads. +type SCIMUsers struct { + pathPrefix string + header http.Header + logger hclog.Logger + + // sc is the shared SCIM client; all workloads in a run share one client. + sc *scimSharedClient + + // namespace is the Vault namespace set on the admin client (may be empty). + namespace string + + // user_create: pool of unique userNames. atomicIdx cycles through the pool. + createUserNames []string + // user_adopt: pool of alias names for pre-seeded unmanaged entities. + adoptUserNames []string + // user_list: pre-built URL strings (one per filter variant). + listURLs []string + + atomicIdx int64 // shared counter; incremented on each Target() call + + config *SCIMUsersConfig +} + +// SCIMUsersConfig holds all HCL-configurable parameters for the SCIM users target. +type SCIMUsersConfig struct { + // Workload selects which of the three user workloads to run. + // One of: "user_create" | "user_adopt" | "user_list" + Workload string `hcl:"workload,optional"` + + // UserCount is: + // user_create: size of the unique-name pool (set ≥ rps × duration_seconds) + // user_adopt: number of pre-seeded unmanaged entities to create + // user_list: ignored (no pool needed) + UserCount int `hcl:"user_count,optional"` + + // Filters is the list of SCIM filter query-strings cycled for user_list. + // An empty string means "no filter" (returns all users). + // Example: ["", "userName eq \"alice@example.com\"", "meta.lastModified gt \"2020-01-01T00:00:00Z\""] + // Defaults to [""] (bare list, no filter) when omitted. + Filters []string `hcl:"filters,optional"` +} + +func (s *SCIMUsers) ParseConfig(body hcl.Body) error { + testConfig := &struct { + Config *SCIMUsersConfig `hcl:"config,block"` + }{ + Config: &SCIMUsersConfig{ + Workload: scimUserWorkloadCreate, + UserCount: 200, + Filters: []string{""}, + }, + } + + diags := gohcl.DecodeBody(body, nil, testConfig) + if diags.HasErrors() { + return fmt.Errorf("error decoding scim_users config: %v", diags) + } + + c := testConfig.Config + s.config = c + + switch c.Workload { + case scimUserWorkloadCreate, scimUserWorkloadAdopt: + if c.UserCount <= 0 { + return fmt.Errorf("scim_users: user_count must be > 0 for workload %q", c.Workload) + } + case scimUserWorkloadList: + if len(c.Filters) == 0 { + c.Filters = []string{""} + } + default: + return fmt.Errorf("scim_users: invalid workload %q; must be one of %q, %q, %q", + c.Workload, scimUserWorkloadCreate, scimUserWorkloadAdopt, scimUserWorkloadList) + } + + return nil +} + +func (s *SCIMUsers) Setup(client *api.Client, mountName string, topLevelConfig *TopLevelTargetConfig) (BenchmarkBuilder, error) { + s.logger = targetLogger.Named(SCIMUsersTestType) + + // Acquire the shared SCIM client — created once for the whole run, + // reused by every scim_users and scim_groups test block. + s.logger.Info("scim_users setup: acquiring shared SCIM client", "workload", s.config.Workload) + sc, err := scimAcquireSharedClient(client) + if err != nil { + return nil, fmt.Errorf("scim_users setup: %w", err) + } + + result := &SCIMUsers{ + sc: sc, + config: s.config, + logger: s.logger, + namespace: sc.namespace, + pathPrefix: "/v1/identity/scim/v2", + } + + switch s.config.Workload { + + case scimUserWorkloadCreate: + // Pre-generate a pool of unique userNames. No Vault state is created + // here; the names are just unique strings that Vault has never seen. + // Scope "uc" (user_create) keeps these distinct from "gm" (group member) + // users seeded by group_create, which share the same runID. + result.createUserNames = make([]string, s.config.UserCount) + for i := range s.config.UserCount { + result.createUserNames[i] = scimCreateUserName(sc.runID, "uc", i) + } + s.logger.Info("scim_users setup: user_create pool ready", "pool_size", s.config.UserCount) + + case scimUserWorkloadAdopt: + // Seed N unmanaged entities. Each entity has: + // - entity name = scimAdoptEntityName(runID, i) + // - alias name = scimAdoptUserAliasName(runID, i) + // - mount = the shared SCIM client's alias_mount_accessor + // The alias name becomes the SCIM userName in POST /Users, triggering + // adoption. The entity name is intentionally different — mirroring real + // login-created entities (e.g. Entra SAML login sets the entity name to + // the internal Vault ID, the alias to the UPN). + s.logger.Info("scim_users setup: seeding unmanaged adoption targets", "count", s.config.UserCount) + aliasNames := make([]string, s.config.UserCount) + + err = runPhase(s.logger, "seed adoption entities", identityConcurrency, s.config.UserCount, func(i int) error { + entityName := scimAdoptEntityName(sc.runID, i) + aliasName := scimAdoptUserAliasName(sc.runID, i) + + resp, err := client.Logical().Write("identity/entity", map[string]any{ + "name": entityName, + }) + if err != nil { + return fmt.Errorf("creating adopt entity %d (%q): %w", i, entityName, err) + } + entityID, err := idFromResponse(resp) + if err != nil { + return fmt.Errorf("reading adopt entity id %d: %w", i, err) + } + + if _, err = client.Logical().Write("identity/entity-alias", map[string]any{ + "name": aliasName, + "canonical_id": entityID, + "mount_accessor": sc.aliasMountAccessor, + }); err != nil { + return fmt.Errorf("creating alias %q for adopt entity %d: %w", aliasName, i, err) + } + + aliasNames[i] = aliasName + s.logger.Debug("scim_users setup: seeded adoption target", + "index", i, "entity_name", entityName, "alias_name", aliasName, "entity_id", entityID, + ) + return nil + }, "total", s.config.UserCount) + if err != nil { + return nil, fmt.Errorf("scim_users setup: seeding adoption entities: %w", err) + } + result.adoptUserNames = aliasNames + s.logger.Info("scim_users setup: adoption targets seeded", "count", s.config.UserCount) + + case scimUserWorkloadList: + // Nothing to seed; list URLs built below. + } + + result.header = scimHeader(sc.scimToken, sc.namespace) + + if s.config.Workload == scimUserWorkloadList { + result.listURLs = make([]string, len(s.config.Filters)) + for i, f := range s.config.Filters { + u := "/v1/identity/scim/v2/Users" + if f != "" { + u += "?filter=" + url.QueryEscape(f) + } + result.listURLs[i] = u + } + s.logger.Info("scim_users setup: list URL variants built", "count", len(result.listURLs)) + } + + return result, nil +} + +func (s *SCIMUsers) Target(client *api.Client) vegeta.Target { + switch s.config.Workload { + + case scimUserWorkloadCreate: + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + poolIdx := int(idx % int64(len(s.createUserNames))) + userName := s.createUserNames[poolIdx] + externalID := "ext-" + userName + + s.logger.Debug("scim_users target: user_create", "pool_idx", poolIdx, "user_name", userName) + + // The ?_w=create sentinel gives user_create a longer, more specific URL + // than user_adopt. The reporter's HasPrefix check runs in weight-sorted + // order and stops at the first match. Because HasPrefix(bare_url, + // sentinel_url) is FALSE and HasPrefix(sentinel_url, sentinel_url) is + // TRUE, create results match only create and adopt results fall through + // to match adopt's bare prefix. Without the sentinel the bare create + // prefix would swallow all adopt results too. + return vegeta.Target{ + Method: http.MethodPost, + URL: client.Address() + s.pathPrefix + "/Users?_w=create", + Header: s.header, + Body: scimUserCreateBody(userName, externalID), + } + + case scimUserWorkloadAdopt: + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + poolIdx := int(idx % int64(len(s.adoptUserNames))) + userName := s.adoptUserNames[poolIdx] + externalID := "ext-" + userName + + s.logger.Debug("scim_users target: user_adopt", "pool_idx", poolIdx, "user_name", userName) + + return vegeta.Target{ + Method: http.MethodPost, + URL: client.Address() + s.pathPrefix + "/Users", + Header: s.header, + Body: scimUserAdoptBody(userName, externalID), + } + + default: // scimUserWorkloadList + idx := atomic.AddInt64(&s.atomicIdx, 1) - 1 + urlIdx := int(idx % int64(len(s.listURLs))) + + s.logger.Debug("scim_users target: user_list", "url_idx", urlIdx, "url", s.listURLs[urlIdx]) + + return vegeta.Target{ + Method: http.MethodGet, + URL: client.Address() + s.listURLs[urlIdx], + Header: s.header, + } + } +} + +func (s *SCIMUsers) Cleanup(client *api.Client) error { + s.logger.Info("scim_users cleanup: releasing shared SCIM client reference") + // scimReleaseSharedClient decrements the shared refcount. The last block + // to release triggers delete-linked-resources=true which removes all + // SCIM-owned entities and groups in one atomic call — no per-resource + // delete loop needed, and no racing with other blocks' cleanup. + return scimReleaseSharedClient(client) +} + +func (s *SCIMUsers) GetTargetInfo() TargetInfo { + switch s.config.Workload { + case scimUserWorkloadCreate: + return TargetInfo{method: http.MethodPost, pathPrefix: s.pathPrefix + "/Users?_w=create"} + case scimUserWorkloadAdopt: + return TargetInfo{method: http.MethodPost, pathPrefix: s.pathPrefix + "/Users"} + default: + return TargetInfo{method: http.MethodGet, pathPrefix: s.pathPrefix + "/Users"} + } +} + +func (s *SCIMUsers) Flags(fs *flag.FlagSet) {} diff --git a/test-fixtures/configs/scim.hcl b/test-fixtures/configs/scim.hcl new file mode 100644 index 00000000..00ab9346 --- /dev/null +++ b/test-fixtures/configs/scim.hcl @@ -0,0 +1,160 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# SCIM performance benchmark — local smoke-test config. +# +# This config exercises all 6 SCIM workloads at small scale so you can verify +# the benchmark does the right thing on a local dev cluster before scaling up. +# +# Prerequisites (run once against your cluster): +# export VAULT_ADDR="http://127.0.0.1:8200" +# export VAULT_TOKEN= +# # Optionally: export VAULT_NAMESPACE=admin +# +# Run: +# vault-benchmark run -config=test-fixtures/configs/scim.hcl +# +# What each test block does: +# +# scim_user_create — POST /scim/v2/Users N times, each with a unique +# userName. Measures raw user-creation throughput. +# +# scim_user_adopt — POST /scim/v2/Users targeting 50 pre-seeded unmanaged +# entities (entity name ≠ alias name, matching real IdP +# login behaviour). Each request adopts a different entity. +# +# scim_user_list — GET /scim/v2/Users cycling through three filter shapes: +# no-filter, userName eq, and meta.lastModified gt. +# +# scim_group_create — POST /scim/v2/Groups with 5 SCIM-owned member IDs per +# group. Measures group-creation + membership write path. +# +# scim_group_adopt — PUT /scim/v2/Groups/ adopting 30 pre-seeded +# unmanaged groups one at a time. +# +# scim_group_list — GET /scim/v2/Groups cycling through two filter shapes. +# +# Weights must sum to 100. + +# Sizing: 50 rps × 30 s = 1500 requests total. +# Expected per target = 1500 × (weight / 100): +# weight=21 → ~315 requests weight=19 → ~285 requests weight=10 → ~150 requests +# +# Pool sizes are set to 2× the expected request count so adoption/create pools +# never cycle even with random weight variance. +# +# Formula when scaling further: +# user_count / group_count >= 2 × rps × duration_seconds × (weight / 100) +# +# NOTE: user_create intentionally has weight=21 (one more than user_adopt=19). +# The reporter attributes results by matching the URL against each target's +# pathPrefix in weight-sorted order (highest weight first), stopping at the first +# match. user_create uses a sentinel suffix (?_w=create) on its URL so the bare +# /Users prefix does not accidentally swallow create results. For this sentinel +# to work reliably, user_create MUST sort before user_adopt — which requires it +# to have a strictly higher weight. The 1-point difference has no measurable +# effect on load distribution at these scales. +duration = "30s" +rps = 50 +report_mode = "terse" +# vault-benchmark requires random_mounts = true when cleanup = true. +# SCIM targets ignore random_mounts (they scope everything by runID internally), +# so this flag has no effect on SCIM setup — it just satisfies the framework check. +random_mounts = true +cleanup = true + +# ── User workloads ──────────────────────────────────────────────────────────── + +# user_create: weight=21 → ~315 requests. Pool of 630 unique userNames (2× headroom). +# Every request creates a genuinely new entity — no name is reused within the run. +# weight=21 (not 20) guarantees user_create sorts before user_adopt in the reporter, +# so the ?_w=create sentinel URL is checked first and does not fall through to the +# bare /Users prefix matcher. See NOTE above. +test "scim_users" "scim_user_create" { + weight = 21 + config { + workload = "user_create" + user_count = 630 + } +} + +# user_adopt: weight=19 → ~285 requests. Pool of 570 pre-seeded unmanaged entities (2× headroom). +# Entity name ≠ alias name, mirroring real IdP login-created entities. +# Every request adopts a DIFFERENT entity — pool never cycles. +test "scim_users" "scim_user_adopt" { + weight = 19 + config { + workload = "user_adopt" + user_count = 570 + } +} + +# user_list: weight=10 → ~150 requests. Three filter shapes round-robined (~50 each). +# An empty filter string means "no filter" (returns all SCIM-owned users). +test "scim_users" "scim_user_list" { + weight = 10 + config { + workload = "user_list" + filters = [ + "", + "meta.lastModified gt \"2020-01-01T00:00:00Z\"", + "active eq true", + ] + } +} + +# ── Group workloads ─────────────────────────────────────────────────────────── + +# group_create (with members): weight=20 → ~300 requests. +# Pool of 600 unique displayNames (2× headroom). +# Each POST body includes 5 SCIM-owned member entity IDs. +# user_seed_count=5 seeds 5 SCIM-owned users during setup; members are +# sampled round-robin from those 5 for every group POST. +# To test the "too many memberships" case, raise members_per_group and +# user_seed_count together (user_seed_count must be >= members_per_group). +test "scim_groups" "scim_group_create" { + weight = 20 + config { + workload = "group_create" + group_count = 600 + members_per_group = 5 + user_seed_count = 5 + } +} + +# group_adopt: weight=10 → ~150 requests. Pool of 300 pre-seeded unmanaged groups (2× headroom). +# Every PUT adopts a DIFFERENT group — pool never cycles. +test "scim_groups" "scim_group_adopt" { + weight = 10 + config { + workload = "group_adopt" + group_count = 300 + } +} + +# group_list: weight=10 → ~150 requests. Two filter shapes round-robined (~75 each). +# Filters are URL-encoded automatically — spaces and quotes are safe to write here. +test "scim_groups" "scim_group_list" { + weight = 10 + config { + workload = "group_list" + filters = [ + "", + "displayName eq \"nonexistent-group\"", + ] + } +} + +# group_create_empty (no members): weight=10 → ~150 requests. +# Isolates the raw group-creation path from the member-validation path. +# Compare latency with scim_group_create above to see membership overhead. +# Uses workload="group_create_empty" — a dedicated workload constant — so the +# reporter can attribute results correctly (it uses a distinct ?_w=empty sentinel +# URL that is not confused with group_create's ?_w=create sentinel). +test "scim_groups" "scim_group_create_empty" { + weight = 10 + config { + workload = "group_create_empty" + group_count = 300 + } +} diff --git a/test-fixtures/configs/scim_membership.hcl b/test-fixtures/configs/scim_membership.hcl new file mode 100644 index 00000000..8241b685 --- /dev/null +++ b/test-fixtures/configs/scim_membership.hcl @@ -0,0 +1,101 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# SCIM performance benchmark — large membership stress config. +# +# Purpose: isolate the cost of group membership at extreme sizes. +# "What if the IdP sends groups with too many memberships?" +# +# HOW TO USE +# ---------- +# Step 1: Run OPTION B first (empty baseline) — establishes the floor. +# Step 2: Uncomment OPTION A and comment out OPTION B. +# Change members_per_group and user_seed_count together across runs: +# members_per_group = 10, user_seed_count = 10 +# members_per_group = 50, user_seed_count = 50 +# members_per_group = 100, user_seed_count = 100 +# members_per_group = 250, user_seed_count = 250 +# members_per_group = 500, user_seed_count = 500 +# members_per_group = 1000, user_seed_count = 1000 +# Step 3: Record mean and p99 for each run. The delta vs OPTION B = membership overhead. +# Step 4: Optionally run OPTION C to measure adoption in isolation. +# +# NOTE ON RPS +# ----------- +# rps=50 is intentionally conservative. From a remote client the effective +# throughput is limited by network RTT (~110ms to HVD), not Vault capacity. +# The goal here is relative comparison (empty vs N members), not absolute +# throughput — rps=50 gives enough samples for stable p99 without overwhelming +# the connection pool. +# +# WHAT TO RECORD PER RUN +# ---------------------- +# members_per_group | mean | p95 | p99 | successRatio +# ------------------|------|-----|-----|------------- +# 0 (baseline) | | | | +# 10 | | | | +# 50 | | | | +# 100 | | | | +# 250 | | | | +# 500 | | | | +# 1000 | | | | +# +# Prerequisites: +# export VAULT_ADDR="https://:8200" +# export VAULT_TOKEN= +# export VAULT_NAMESPACE=admin +# +# Run: +# vault-benchmark run -config=test-fixtures/configs/scim_membership.hcl + +duration = "120s" +rps = 50 +report_mode = "terse" +random_mounts = true +cleanup = true + +# ── OPTION B: empty baseline (run this first) ───────────────────────────────── +# +# Establishes the floor: what does group creation cost with zero members? +# Record mean and p99. Every other run is measured as delta from these numbers. +# +# pool = 50 rps × 120s × 2 = 12 000 + +# test "scim_groups" "scim_group_create_empty_baseline" { +# weight = 100 +# config { +# workload = "group_create_empty" +# group_count = 12000 +# } +# } + +# ── OPTION A: group_create with N members (comment in, comment out B) ───────── +# +# Change members_per_group and user_seed_count to the same value each run. +# user_seed_count members are created during setup and sampled round-robin — +# so user_seed_count=100 means 100 unique member entities are reused across all +# group POST bodies (you don't need group_count × members_per_group users). +# +test "scim_groups" "scim_group_create_large_membership" { + weight = 100 + config { + workload = "group_create" + group_count = 12000 + members_per_group = 100 + user_seed_count = 100 + } +} + +# ── OPTION C: group_adopt in isolation (optional) ───────────────────────────── +# +# Does adoption latency stay flat as the number of adoptable groups grows? +# Vary group_count across runs: 500, 2000, 5000, 9000. +# Compare mean and p99 — if they climb, the adoption lookup is O(n). +# +# test "scim_groups" "scim_group_adopt_isolated" { +# weight = 100 +# config { +# workload = "group_adopt" +# group_count = 12000 +# } +# } diff --git a/test-fixtures/configs/scim_scale.hcl b/test-fixtures/configs/scim_scale.hcl new file mode 100644 index 00000000..681afa40 --- /dev/null +++ b/test-fixtures/configs/scim_scale.hcl @@ -0,0 +1,137 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# SCIM performance benchmark — scale-up config. +# +# Use this config after the smoke test (scim.hcl) passes on a local cluster. +# Target: a real cluster (dev-enterprise or HVD) where you want to find the +# throughput ceiling and p99 behaviour under realistic IdP spike loads. +# +# Design intent: +# - Aligned with product expectation of ~9000 users from a customer IdP. +# - Pools are sized to 2× the expected request count so no name is reused +# within a single run (no cache hits, no 409 collisions). +# - Run multiple times at increasing rps to find the knee of the curve: +# rps=100 duration=120s → 12 000 requests (good baseline) +# rps=200 duration=120s → 24 000 requests (moderate spike) +# rps=500 duration=120s → 60 000 requests (large spike) +# - Keep duration=120s so each workload gets enough samples for stable p99. +# +# Prerequisites: +# export VAULT_ADDR="https://:8200" +# export VAULT_TOKEN= +# export VAULT_NAMESPACE=admin # if targeting the admin namespace +# +# Run: +# vault-benchmark run -config=test-fixtures/configs/scim_scale.hcl +# +# Sizing formula (adjust rps and duration together): +# pool_size = 2 × rps × duration_seconds × (weight / 100) +# +# At rps=200, duration=120s: +# weight=21 → ~5 040 requests → pool 10 080 +# weight=19 → ~4 560 requests → pool 9 120 +# weight=20 → ~4 800 requests → pool 9 600 +# weight=10 → ~2 400 requests → pool 4 800 +# +# NOTE: user_create has weight=21 (vs user_adopt=19) so the ?_w=create sentinel +# URL is always checked first by the reporter. See scim.hcl for full explanation. + +duration = "1400s" +rps = 30 +workers = 25 +report_mode = "terse" +random_mounts = true +cleanup = true + +# ── User workloads (~5,000–6,000 requests each) ────────────────────────────── + +# user_create: 30 rps × 1400 s × 0.15 = 6 300 requests. Pool = 10 000. +# Simulates an IdP pushing a spike of new user creations. +# Every POST hits a unique userName — no entity is created twice. +test "scim_users" "scim_user_create" { + weight = 15 + config { + workload = "user_create" + user_count = 10000 + } +} + +# user_adopt: 30 rps × 1400 s × 0.14 = 5 880 requests. Pre-seeded pool = 6 500. +# Simulates an IdP pushing adoption of existing login-created users. +# Each request targets a DIFFERENT pre-seeded entity (entity name ≠ alias name). +test "scim_users" "scim_user_adopt" { + weight = 14 + config { + workload = "user_adopt" + user_count = 6500 + } +} + +# user_list: 30 rps × 1400 s × 0.14 = 5 880 requests. +# Cycles through four filter shapes to exercise different list code paths: +# "" — no filter, full scan (worst case) +# userName eq "..." — point lookup by userName +# active eq true — boolean attribute scan +# meta.lastModified gt "..." — time-range scan +test "scim_users" "scim_user_list" { + weight = 14 + config { + workload = "user_list" + filters = [ + "", + "userName eq \"bench-create-user-x@example.com\"", + "active eq true", + "meta.lastModified gt \"2020-01-01T00:00:00Z\"", + ] + } +} + +# ── Group workloads (~5,000–6,000 requests each) ────────────────────────────── + +# group_create (with members): 30 rps × 1400 s × 0.15 = 6 300 requests. Pool = 10 000. +# members_per_group=50 / user_seed_count=100 exercises realistic group sizes. +test "scim_groups" "scim_group_create" { + weight = 15 + config { + workload = "group_create" + group_count = 10000 + members_per_group = 50 + user_seed_count = 100 + } +} + +# group_adopt: 30 rps × 1400 s × 0.14 = 5 880 requests. Pre-seeded pool = 6 500. +# Simulates an IdP adopting existing Vault groups. +# Every PUT targets a DIFFERENT pre-seeded group. +test "scim_groups" "scim_group_adopt" { + weight = 14 + config { + workload = "group_adopt" + group_count = 6500 + } +} + +# group_list: 30 rps × 1400 s × 0.14 = 5 880 requests. +# Exercises both full-scan and filtered list code paths. +test "scim_groups" "scim_group_list" { + weight = 14 + config { + workload = "group_list" + filters = [ + "", + "displayName eq \"nonexistent-group\"", + ] + } +} + +# group_create_empty: 30 rps × 1400 s × 0.14 = 5 880 requests. Pool = 10 000. +# Baseline: group creation with no members. Compare p50/p99 with scim_group_create +# above to isolate the per-member overhead at members_per_group=50. +test "scim_groups" "scim_group_create_empty" { + weight = 14 + config { + workload = "group_create_empty" + group_count = 10000 + } +} diff --git a/test-fixtures/configs/scim_spike_user_adopt.hcl b/test-fixtures/configs/scim_spike_user_adopt.hcl new file mode 100644 index 00000000..7fc8de10 --- /dev/null +++ b/test-fixtures/configs/scim_spike_user_adopt.hcl @@ -0,0 +1,28 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# SCIM performance benchmark — Pure 9,000 User Adoption Spike. +# +# Purpose: Simulates an IdP syncing ~9,000 users that already exist as +# unmanaged Vault entities (e.g. from prior SAML/OIDC logins) and adopting them into SCIM. +# +# Math: +# duration = 300s (5 minutes) +# rps = 30 +# Total requests = 30 rps × 300s = 9,000 adoptions (POST /v1/identity/scim/v2/Users) +# Pre-seeded adoption pool = 10,000 (pre-seeded during setup across 16 parallel workers) + +duration = "300s" +rps = 30 +workers = 25 +report_mode = "terse" +random_mounts = true +cleanup = true + +test "scim_users" "pure_user_adoption_spike" { + weight = 100 + config { + workload = "user_adopt" + user_count = 10000 + } +} diff --git a/test-fixtures/configs/scim_spike_user_create.hcl b/test-fixtures/configs/scim_spike_user_create.hcl new file mode 100644 index 00000000..6912887e --- /dev/null +++ b/test-fixtures/configs/scim_spike_user_create.hcl @@ -0,0 +1,28 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# SCIM performance benchmark — Pure 9,000 User Creation Spike. +# +# Purpose: Simulates an IdP (e.g. Entra ID / Okta) doing an initial sync +# of ~9,000 new users directly into Vault SCIM. +# +# Math: +# duration = 300s (5 minutes) +# rps = 30 +# Total requests = 30 rps × 300s = 9,000 user creations (POST /v1/identity/scim/v2/Users) +# Pool size = 12,000 (headroom to ensure every user is unique) + +duration = "300s" +rps = 30 +workers = 25 +report_mode = "terse" +random_mounts = true +cleanup = true + +test "scim_users" "pure_user_creation_spike" { + weight = 100 + config { + workload = "user_create" + user_count = 12000 + } +} From 643082f352affd0d7d4186315f1b74c002defcd0 Mon Sep 17 00:00:00 2001 From: Anjani Mallampati Date: Tue, 29 Sep 2026 13:42:44 -0700 Subject: [PATCH 2/3] SCIM performance testing - final load tests --- benchmarktests/concurrency_helper.go | 6 + benchmarktests/scim_helper.go | 162 +++++++++++++++++- benchmarktests/target_scim_groups.go | 96 +++++------ benchmarktests/target_scim_users.go | 45 ++++- run_all_scim_benchmarks.sh | 32 ++++ test-fixtures/configs/{ => scim}/scim.hcl | 6 +- .../configs/scim/scim_group_adopt_spike.hcl | 37 ++++ .../configs/scim/scim_group_create_spike.hcl | 33 ++++ .../configs/scim/scim_group_filter_perf.hcl | 45 +++++ .../scim/scim_group_membership_stress.hcl | 46 +++++ .../configs/{ => scim}/scim_scale.hcl | 2 +- .../configs/scim/scim_user_adopt_spike.hcl | 37 ++++ .../configs/scim/scim_user_create_spike.hcl | 31 ++++ .../configs/scim/scim_user_filter_perf.hcl | 49 ++++++ .../configs/scim/scim_user_scale_limit.hcl | 33 ++++ test-fixtures/configs/scim_membership.hcl | 101 ----------- .../configs/scim_spike_user_adopt.hcl | 28 --- .../configs/scim_spike_user_create.hcl | 28 --- 18 files changed, 589 insertions(+), 228 deletions(-) create mode 100755 run_all_scim_benchmarks.sh rename test-fixtures/configs/{ => scim}/scim.hcl (97%) create mode 100644 test-fixtures/configs/scim/scim_group_adopt_spike.hcl create mode 100644 test-fixtures/configs/scim/scim_group_create_spike.hcl create mode 100644 test-fixtures/configs/scim/scim_group_filter_perf.hcl create mode 100644 test-fixtures/configs/scim/scim_group_membership_stress.hcl rename test-fixtures/configs/{ => scim}/scim_scale.hcl (98%) create mode 100644 test-fixtures/configs/scim/scim_user_adopt_spike.hcl create mode 100644 test-fixtures/configs/scim/scim_user_create_spike.hcl create mode 100644 test-fixtures/configs/scim/scim_user_filter_perf.hcl create mode 100644 test-fixtures/configs/scim/scim_user_scale_limit.hcl delete mode 100644 test-fixtures/configs/scim_membership.hcl delete mode 100644 test-fixtures/configs/scim_spike_user_adopt.hcl delete mode 100644 test-fixtures/configs/scim_spike_user_create.hcl diff --git a/benchmarktests/concurrency_helper.go b/benchmarktests/concurrency_helper.go index 06de68d7..6db73831 100644 --- a/benchmarktests/concurrency_helper.go +++ b/benchmarktests/concurrency_helper.go @@ -22,6 +22,12 @@ const ( // backend), while KV setup hits the secrets engine (may benefit from n>1 on // integrated-storage clusters). Both default to 1 (serial). To experiment locally, // raise the relevant constant; do not raise on production clusters without profiling. + // + // NOTE for SCIM benchmarks: the SCIM seeding helpers (scim_helper.go, + // target_scim_users.go, target_scim_groups.go) create large numbers of identity + // entities/groups during setup. Raising identityConcurrency to ~16 cuts that + // setup time significantly. Before running the SCIM benchmark suite, temporarily + // set identityConcurrency = 16 here, then restore it to 1 before committing. identityConcurrency = 1 kvSeedConcurrency = 1 ) diff --git a/benchmarktests/scim_helper.go b/benchmarktests/scim_helper.go index 12693d22..a822e0ea 100644 --- a/benchmarktests/scim_helper.go +++ b/benchmarktests/scim_helper.go @@ -6,6 +6,7 @@ package benchmarktests import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -31,6 +32,14 @@ const ( // scimUserSchemaCore is the SCIM 2.0 User schema ID. scimUserSchemaCore = "urn:ietf:params:scim:schemas:core:2.0:User" scimGroupSchemaCore = "urn:ietf:params:scim:schemas:core:2.0:Group" + + // scimFilterSeedUserToken and scimFilterSeedGroupToken may appear inside a + // user_list/group_list filter string. Setup() substitutes them with the + // actual name of a pre-seeded resource (seed_user_count/seed_group_count) + // so a "point lookup" filter is guaranteed to match real, runID-scoped data + // instead of a hardcoded literal that can never match. + scimFilterSeedUserToken = "$SEED_USERNAME" + scimFilterSeedGroupToken = "$SEED_GROUPNAME" ) // scimClientPolicy is the minimum ACL policy a SCIM client token needs. @@ -78,11 +87,18 @@ path "identity/scim/client/*" { // scimSharedClient holds the state shared across all test blocks in one run. type scimSharedClient struct { - runID string - scimToken string - aliasMountAccessor string - namespace string - refCount int // number of test blocks currently holding this instance + runID string + scimToken string + aliasMountAccessor string + namespace string + refCount int // number of test blocks currently holding this instance + + // adoptUserNames / adoptGroupIDs accumulate every name seeded by user_adopt / + // group_adopt blocks sharing this client. Only the subset actually adopted + // during the attack gets removed by delete-linked-resources; the last + // Cleanup() sweeps whatever remains in these lists (see scimReleaseSharedClient). + adoptUserNames []string + adoptGroupIDs []string } var ( @@ -154,22 +170,60 @@ func scimReleaseSharedClient(client *api.Client) error { key := scimSharedKey(client) scimSharedMu.Lock() - defer scimSharedMu.Unlock() - sc, ok := scimSharedClients[key] if !ok { // Already cleaned up or never created — nothing to do. + scimSharedMu.Unlock() return nil } sc.refCount-- if sc.refCount > 0 { + scimSharedMu.Unlock() return nil } - // Last holder — tear down. + // Last holder — tear down. Copy out what the sweep needs before unlocking; + // nothing else can reach this *scimSharedClient after the map delete. delete(scimSharedClients, key) - return SCIMCleanup(client, sc.runID) + runID := sc.runID + adoptUserNames := sc.adoptUserNames + adoptGroupIDs := sc.adoptGroupIDs + scimSharedMu.Unlock() + + var errs []error + if err := SCIMCleanup(client, runID); err != nil { + errs = append(errs, err) + } + + // SCIMCleanup's delete-linked-resources=true already removed every entity/group + // that WAS adopted during the run. Anything from the seed pools still present + // at this point was never adopted, so it's safe to delete directly. + if err := sweepUnadoptedEntities(client, adoptUserNames); err != nil { + errs = append(errs, err) + } + if err := sweepUnadoptedGroups(client, adoptGroupIDs); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} + +// sweepUnadoptedEntities deletes any entity in names that Vault still has. +// Deleting an already-gone entity by name is a documented no-op (no error), so +// this is safe to call unconditionally against the full user_adopt seed pool. +func sweepUnadoptedEntities(client *api.Client, names []string) error { + return deletePhase(targetLogger, "sweep unadopted seed entities", client, "identity/entity/name/", identityConcurrency, len(names), func(idx int) string { + return names[idx] + }) +} + +// sweepUnadoptedGroups deletes any group in ids that Vault still has. +// Deleting an already-gone group by ID is a documented no-op (no error), so +// this is safe to call unconditionally against the full group_adopt seed pool. +func sweepUnadoptedGroups(client *api.Client, ids []string) error { + return deletePhase(targetLogger, "sweep unadopted seed groups", client, "identity/group/id/", identityConcurrency, len(ids), func(idx int) string { + return ids[idx] + }) } // ── Name helpers ────────────────────────────────────────────────────────────── @@ -501,3 +555,93 @@ func scimGroupAdoptBody(displayName string) []byte { return b } +// seedSCIMUsers creates SCIM-owned users via the SCIM API and returns their +// Vault entity IDs. scope keeps name pools distinct across callers that share +// a runID — e.g. "gm" for group members seeded by group_create, "lf" for +// list-filter seed data seeded by user_list. +func seedSCIMUsers(logger hclog.Logger, adminClient *api.Client, scimToken, runID, scope, namespace string, count int) ([]string, error) { + entityIDs := make([]string, count) + header := scimHeader(scimToken, namespace) + + err := runPhase(logger, "seed scim users", identityConcurrency, count, func(i int) error { + userName := scimCreateUserName(runID, scope, i) + externalID := "seed-ext-" + userName + body := scimUserCreateBody(userName, externalID) + + req, err := http.NewRequest(http.MethodPost, + adminClient.Address()+"/v1/identity/scim/v2/Users", + bytes.NewReader(body), + ) + if err != nil { + return fmt.Errorf("building seed user request %d: %w", i, err) + } + for k, vals := range header { + for _, v := range vals { + req.Header.Set(k, v) + } + } + + resp, err := adminClient.CloneConfig().HttpClient.Do(req) + if err != nil { + return fmt.Errorf("posting seed user %d: %w", i, err) + } + statusCode, respBody := debugLogSCIMResponse(resp) + logger.Debug("seed scim user response", + "index", i, "user_name", userName, "status", statusCode, "body", string(respBody), + ) + if statusCode != http.StatusCreated { + return fmt.Errorf("seed user %d (%q): unexpected status %d, body: %s", i, userName, statusCode, respBody) + } + + var parsed map[string]any + if err := json.Unmarshal(respBody, &parsed); err != nil { + return fmt.Errorf("parsing seed user response %d: %w", i, err) + } + id, ok := parsed["id"].(string) + if !ok || id == "" { + return fmt.Errorf("seed user response %d missing id field", i) + } + entityIDs[i] = id + return nil + }, "total", count) + + return entityIDs, err +} + +// seedSCIMGroups creates SCIM-owned groups (no members) via the SCIM API. +// Used to give group_list filter tests real, findable data to query against. +// scope keeps this pool distinct from other group name pools sharing the runID. +func seedSCIMGroups(logger hclog.Logger, adminClient *api.Client, scimToken, runID, scope, namespace string, count int) error { + header := scimHeader(scimToken, namespace) + + return runPhase(logger, "seed scim groups", identityConcurrency, count, func(i int) error { + displayName := scimCreateGroupDisplayName(runID, scope, i) + body := scimGroupCreateBody(displayName, nil) + + req, err := http.NewRequest(http.MethodPost, + adminClient.Address()+"/v1/identity/scim/v2/Groups", + bytes.NewReader(body), + ) + if err != nil { + return fmt.Errorf("building seed group request %d: %w", i, err) + } + for k, vals := range header { + for _, v := range vals { + req.Header.Set(k, v) + } + } + + resp, err := adminClient.CloneConfig().HttpClient.Do(req) + if err != nil { + return fmt.Errorf("posting seed group %d: %w", i, err) + } + statusCode, respBody := debugLogSCIMResponse(resp) + logger.Debug("seed scim group response", + "index", i, "display_name", displayName, "status", statusCode, "body", string(respBody), + ) + if statusCode != http.StatusCreated { + return fmt.Errorf("seed group %d (%q): unexpected status %d, body: %s", i, displayName, statusCode, respBody) + } + return nil + }, "total", count) +} diff --git a/benchmarktests/target_scim_groups.go b/benchmarktests/target_scim_groups.go index 1050f57e..d77502c8 100644 --- a/benchmarktests/target_scim_groups.go +++ b/benchmarktests/target_scim_groups.go @@ -24,12 +24,11 @@ package benchmarktests // one Entra/Okta tenant has one credential, not one per operation type. import ( - "bytes" - "encoding/json" "flag" "fmt" "net/http" "net/url" + "strings" "sync/atomic" "github.com/hashicorp/go-hclog" @@ -102,7 +101,14 @@ type SCIMGroupsConfig struct { // Filters is the list of SCIM filter query-strings cycled for group_list. // An empty string means "no filter". Defaults to [""] when omitted. + // A filter may contain the token $SEED_GROUPNAME, which is substituted with + // the displayName of a pre-seeded group (see SeedGroupCount) so a point-lookup + // filter is guaranteed to match real data. Filters []string `hcl:"filters,optional"` + + // SeedGroupCount pre-seeds this many SCIM-owned groups during setup, purely as + // queryable data for group_list filter tests. Ignored for other workloads. + SeedGroupCount int `hcl:"seed_group_count,optional"` } func (s *SCIMGroups) ParseConfig(body hcl.Body) error { @@ -145,6 +151,9 @@ func (s *SCIMGroups) ParseConfig(body hcl.Body) error { if c.UserSeedCount == 0 && c.MembersPerGroup > 0 { c.UserSeedCount = c.MembersPerGroup } + if c.SeedGroupCount < 0 { + return fmt.Errorf("scim_groups: seed_group_count must be >= 0") + } return nil } @@ -191,7 +200,7 @@ func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig seedCount = s.config.MembersPerGroup } s.logger.Info("scim_groups setup: seeding SCIM-owned users for group membership", "count", seedCount) - entityIDs, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, sc.namespace, seedCount) + entityIDs, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, "gm", sc.namespace, seedCount) if err != nil { return nil, fmt.Errorf("scim_groups setup: seeding member users: %w", err) } @@ -232,14 +241,25 @@ func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig s.logger.Info("scim_groups setup: adoption groups seeded", "count", s.config.GroupCount) case scimGroupWorkloadList: - // Nothing to seed. + // Seed real, findable groups so filter tests exercise an actual lookup + // instead of running against an empty namespace. + if s.config.SeedGroupCount > 0 { + s.logger.Info("scim_groups setup: seeding groups for filter tests", "count", s.config.SeedGroupCount) + if err := seedSCIMGroups(s.logger, client, sc.scimToken, sc.runID, "lg", sc.namespace, s.config.SeedGroupCount); err != nil { + return nil, fmt.Errorf("scim_groups setup: seeding filter-test groups: %w", err) + } + } } result.header = scimHeader(sc.scimToken, sc.namespace) if s.config.Workload == scimGroupWorkloadList { + // Deterministic — matches the name seedSCIMGroups assigned to index 0, + // with no need to inspect any HTTP response. + seedGroupName := scimCreateGroupDisplayName(sc.runID, "lg", 0) result.listURLs = make([]string, len(s.config.Filters)) for i, f := range s.config.Filters { + f = strings.ReplaceAll(f, scimFilterSeedGroupToken, seedGroupName) u := "/v1/identity/scim/v2/Groups" if f != "" { u += "?filter=" + url.QueryEscape(f) @@ -334,6 +354,21 @@ func (s *SCIMGroups) Target(client *api.Client) vegeta.Target { } func (s *SCIMGroups) Cleanup(client *api.Client) error { + // group_adopt: only the pool indices past the final atomicIdx were never + // sent as an adopt request at all, so they're guaranteed to still be plain + // unmanaged groups. Anything at or before that index may have been adopted + // (SCIM-owned) and must be left to delete-linked-resources' async cleanup + // worker instead — deleting it here directly would race that worker and + // return "SCIM-managed resources must be modified through SCIM". + if s.config.Workload == scimGroupWorkloadAdopt { + touched := int(atomic.LoadInt64(&s.atomicIdx)) + if touched < len(s.adoptGroupIDs) { + scimSharedMu.Lock() + s.sc.adoptGroupIDs = append(s.sc.adoptGroupIDs, s.adoptGroupIDs[touched:]...) + scimSharedMu.Unlock() + } + } + s.logger.Info("scim_groups cleanup: releasing shared SCIM client reference") // All SCIM-owned resources (created users, created groups, adopted users, // adopted groups) belong to the single shared client. The last block to @@ -357,56 +392,3 @@ func (s *SCIMGroups) GetTargetInfo() TargetInfo { } func (s *SCIMGroups) Flags(fs *flag.FlagSet) {} - -// seedSCIMUsers creates SCIM-owned users via the SCIM API and returns their -// Vault entity IDs. Used as group members in group_create. -func seedSCIMUsers(logger hclog.Logger, adminClient *api.Client, scimToken, runID, namespace string, count int) ([]string, error) { - entityIDs := make([]string, count) - header := scimHeader(scimToken, namespace) - - err := runPhase(logger, "seed scim users for members", identityConcurrency, count, func(i int) error { - // Scope "gm" (group members) avoids collision with user_create's "uc" pool — - // both share the same runID within a benchmark run. - userName := scimCreateUserName(runID, "gm", i) - externalID := "member-ext-" + userName - body := scimUserCreateBody(userName, externalID) - - req, err := http.NewRequest(http.MethodPost, - adminClient.Address()+"/v1/identity/scim/v2/Users", - bytes.NewReader(body), - ) - if err != nil { - return fmt.Errorf("building seed user request %d: %w", i, err) - } - for k, vals := range header { - for _, v := range vals { - req.Header.Set(k, v) - } - } - - resp, err := adminClient.CloneConfig().HttpClient.Do(req) - if err != nil { - return fmt.Errorf("posting seed user %d: %w", i, err) - } - statusCode, respBody := debugLogSCIMResponse(resp) - logger.Debug("seed scim user response", - "index", i, "user_name", userName, "status", statusCode, "body", string(respBody), - ) - if statusCode != http.StatusCreated { - return fmt.Errorf("seed user %d (%q): unexpected status %d, body: %s", i, userName, statusCode, respBody) - } - - var parsed map[string]any - if err := json.Unmarshal(respBody, &parsed); err != nil { - return fmt.Errorf("parsing seed user response %d: %w", i, err) - } - id, ok := parsed["id"].(string) - if !ok || id == "" { - return fmt.Errorf("seed user response %d missing id field", i) - } - entityIDs[i] = id - return nil - }, "total", count) - - return entityIDs, err -} diff --git a/benchmarktests/target_scim_users.go b/benchmarktests/target_scim_users.go index 86d73fe7..f688a66a 100644 --- a/benchmarktests/target_scim_users.go +++ b/benchmarktests/target_scim_users.go @@ -26,6 +26,7 @@ import ( "fmt" "net/http" "net/url" + "strings" "sync/atomic" "github.com/hashicorp/go-hclog" @@ -86,8 +87,15 @@ type SCIMUsersConfig struct { // Filters is the list of SCIM filter query-strings cycled for user_list. // An empty string means "no filter" (returns all users). // Example: ["", "userName eq \"alice@example.com\"", "meta.lastModified gt \"2020-01-01T00:00:00Z\""] - // Defaults to [""] (bare list, no filter) when omitted. + // Defaults to [""] (bare list, no filter) when omitted. A filter may contain + // the token $SEED_USERNAME, which is substituted with the userName of a + // pre-seeded user (see SeedUserCount) so a point-lookup filter is guaranteed + // to match real data. Filters []string `hcl:"filters,optional"` + + // SeedUserCount pre-seeds this many SCIM-owned users during setup, purely as + // queryable data for user_list filter tests. Ignored for other workloads. + SeedUserCount int `hcl:"seed_user_count,optional"` } func (s *SCIMUsers) ParseConfig(body hcl.Body) error { @@ -118,6 +126,9 @@ func (s *SCIMUsers) ParseConfig(body hcl.Body) error { if len(c.Filters) == 0 { c.Filters = []string{""} } + if c.SeedUserCount < 0 { + return fmt.Errorf("scim_users: seed_user_count must be >= 0") + } default: return fmt.Errorf("scim_users: invalid workload %q; must be one of %q, %q, %q", c.Workload, scimUserWorkloadCreate, scimUserWorkloadAdopt, scimUserWorkloadList) @@ -206,14 +217,25 @@ func (s *SCIMUsers) Setup(client *api.Client, mountName string, topLevelConfig * s.logger.Info("scim_users setup: adoption targets seeded", "count", s.config.UserCount) case scimUserWorkloadList: - // Nothing to seed; list URLs built below. + // Seed real, findable users so filter tests exercise an actual lookup + // instead of running against an empty namespace. + if s.config.SeedUserCount > 0 { + s.logger.Info("scim_users setup: seeding users for filter tests", "count", s.config.SeedUserCount) + if _, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, "lf", sc.namespace, s.config.SeedUserCount); err != nil { + return nil, fmt.Errorf("scim_users setup: seeding filter-test users: %w", err) + } + } } result.header = scimHeader(sc.scimToken, sc.namespace) if s.config.Workload == scimUserWorkloadList { + // Deterministic — matches the name seedSCIMUsers assigned to index 0, + // with no need to inspect any HTTP response. + seedUserName := scimCreateUserName(sc.runID, "lf", 0) result.listURLs = make([]string, len(s.config.Filters)) for i, f := range s.config.Filters { + f = strings.ReplaceAll(f, scimFilterSeedUserToken, seedUserName) u := "/v1/identity/scim/v2/Users" if f != "" { u += "?filter=" + url.QueryEscape(f) @@ -281,6 +303,25 @@ func (s *SCIMUsers) Target(client *api.Client) vegeta.Target { } func (s *SCIMUsers) Cleanup(client *api.Client) error { + // user_adopt: only the pool indices past the final atomicIdx were never sent + // as an adopt request at all, so they're guaranteed to still be plain + // unmanaged entities. Anything at or before that index may have been adopted + // (SCIM-owned) and must be left to delete-linked-resources' async cleanup + // worker instead — deleting it here directly would race that worker and + // return "SCIM-managed resources must be modified through SCIM". + if s.config.Workload == scimUserWorkloadAdopt { + touched := int(atomic.LoadInt64(&s.atomicIdx)) + if touched < len(s.adoptUserNames) { + untouched := make([]string, 0, len(s.adoptUserNames)-touched) + for i := touched; i < len(s.adoptUserNames); i++ { + untouched = append(untouched, scimAdoptEntityName(s.sc.runID, i)) + } + scimSharedMu.Lock() + s.sc.adoptUserNames = append(s.sc.adoptUserNames, untouched...) + scimSharedMu.Unlock() + } + } + s.logger.Info("scim_users cleanup: releasing shared SCIM client reference") // scimReleaseSharedClient decrements the shared refcount. The last block // to release triggers delete-linked-resources=true which removes all diff --git a/run_all_scim_benchmarks.sh b/run_all_scim_benchmarks.sh new file mode 100755 index 00000000..fc01e4b7 --- /dev/null +++ b/run_all_scim_benchmarks.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# NOTE: Before running this script, temporarily raise identityConcurrency in +# benchmarktests/concurrency_helper.go from 1 to ~16. The SCIM benchmarks seed +# large numbers of identity entities and groups, and the higher concurrency cuts +# setup time significantly. Restore the value to 1 before committing. +set -e + +echo "=== [1/8] Running User Creation Spike ===" +go run main.go run -config=test-fixtures/configs/scim/scim_user_create_spike.hcl > results_user_create_spike.log 2>&1 + +echo "=== [2/8] Running User Adoption Spike ===" +go run main.go run -config=test-fixtures/configs/scim/scim_user_adopt_spike.hcl > results_user_adopt_spike.log 2>&1 + +echo "=== [3/8] Running User Scale Limit (12k users) ===" +go run main.go run -config=test-fixtures/configs/scim/scim_user_scale_limit.hcl > results_user_scale_limit.log 2>&1 + +echo "=== [4/8] Running User Filter Performance ===" +go run main.go run -config=test-fixtures/configs/scim/scim_user_filter_perf.hcl > results_user_filter_perf.log 2>&1 + +echo "=== [5/8] Running Group Creation Spike ===" +go run main.go run -config=test-fixtures/configs/scim/scim_group_create_spike.hcl > results_group_create_spike.log 2>&1 + +echo "=== [6/8] Running Group Adoption Spike ===" +go run main.go run -config=test-fixtures/configs/scim/scim_group_adopt_spike.hcl > results_group_adopt_spike.log 2>&1 + +echo "=== [7/8] Running Group Membership Stress ===" +go run main.go run -config=test-fixtures/configs/scim/scim_group_membership_stress.hcl > results_group_membership_stress.log 2>&1 + +echo "=== [8/8] Running Group Filter Performance ===" +go run main.go run -config=test-fixtures/configs/scim/scim_group_filter_perf.hcl > results_group_filter_perf.log 2>&1 + +echo "=== All 8 SCIM Benchmark Tests Complete! ===" diff --git a/test-fixtures/configs/scim.hcl b/test-fixtures/configs/scim/scim.hcl similarity index 97% rename from test-fixtures/configs/scim.hcl rename to test-fixtures/configs/scim/scim.hcl index 00ab9346..7866934a 100644 --- a/test-fixtures/configs/scim.hcl +++ b/test-fixtures/configs/scim/scim.hcl @@ -12,7 +12,7 @@ # # Optionally: export VAULT_NAMESPACE=admin # # Run: -# vault-benchmark run -config=test-fixtures/configs/scim.hcl +# vault-benchmark run -config=test-fixtures/configs/scim/scim.hcl # # What each test block does: # @@ -56,7 +56,9 @@ # effect on load distribution at these scales. duration = "30s" rps = 50 -report_mode = "terse" +log_level = "DEBUG" +report_mode = "verbose" +# report_mode = "terse" # vault-benchmark requires random_mounts = true when cleanup = true. # SCIM targets ignore random_mounts (they scope everything by runID internally), # so this flag has no effect on SCIM setup — it just satisfies the framework check. diff --git a/test-fixtures/configs/scim/scim_group_adopt_spike.hcl b/test-fixtures/configs/scim/scim_group_adopt_spike.hcl new file mode 100644 index 00000000..834c0931 --- /dev/null +++ b/test-fixtures/configs/scim/scim_group_adopt_spike.hcl @@ -0,0 +1,37 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# Group Adoption Spike Test +# Task Question: "What if those group creations are adoptions?" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Simulates an IdP pushing a high-rate spike of group adoptions. +# Setup seeds unmanaged internal Vault groups. During the attack, each PUT +# adopts a pre-existing group into SCIM. +# +# Run at step-up RPS to compare directly against creation spike: +# Run 1: rps = 100, workers = 100 (Moderate spike) +# Run 2: rps = 250, workers = 250 (Heavy spike) +# Run 3: rps = 500, workers = 500 (Extreme spike / breaking limit) +# +# group_count carries 25% headroom over the nominal rps x duration request count +# (100 x 60s = 6,000) so a timing overshoot can't wrap the pool index back onto +# an already-adopted group, which would surface as a false 403/uniqueness error +# instead of a fresh adoption. + +duration = "60s" +rps = 100 +workers = 100 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_groups" "spike_group_adopt" { + weight = 100 + config { + workload = "group_adopt" + group_count = 7500 + } +} diff --git a/test-fixtures/configs/scim/scim_group_create_spike.hcl b/test-fixtures/configs/scim/scim_group_create_spike.hcl new file mode 100644 index 00000000..4fc1ddc3 --- /dev/null +++ b/test-fixtures/configs/scim/scim_group_create_spike.hcl @@ -0,0 +1,33 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# Group Creation Spike Test +# Task Question: "What if the IdP sends a spike of group creations?" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Simulates an IdP pushing a high-rate spike of new group creations. +# Each group is created with 50 SCIM members to reflect realistic enterprise groups. +# +# Run at step-up RPS to observe queueing and find the saturation point: +# Run 1: rps = 100, workers = 100 (Moderate spike) +# Run 2: rps = 250, workers = 250 (Heavy spike) +# Run 3: rps = 500, workers = 500 (Extreme spike / breaking limit) + +duration = "60s" +rps = 100 +workers = 100 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_groups" "spike_group_create" { + weight = 100 + config { + workload = "group_create" + group_count = 35000 + members_per_group = 50 + user_seed_count = 100 + } +} diff --git a/test-fixtures/configs/scim/scim_group_filter_perf.hcl b/test-fixtures/configs/scim/scim_group_filter_perf.hcl new file mode 100644 index 00000000..482912a2 --- /dev/null +++ b/test-fixtures/configs/scim/scim_group_filter_perf.hcl @@ -0,0 +1,45 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# GET /Groups Filter Performance Under Load +# Task Question: "Validate GET Groups performance, for different types of filters" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Exercises GET /v1/identity/scim/v2/Groups across filter shapes: +# 1. "" — Full scan (all groups) +# 2. displayName eq "nonexistent-group" — Point lookup, guaranteed miss +# 3. displayName eq "..." — Point lookup, guaranteed hit +# +# seed_group_count pre-creates 5,000 SCIM-owned groups during setup (before the +# timed run starts) so the full scan and the "hit" filter have real data to +# query instead of running against an empty namespace. The hit filter uses the +# $SEED_GROUPNAME token, which Setup() substitutes with the actual displayName +# of pre-seeded group #0. +# +# Math: +# duration = 60s +# rps = 200 +# workers = 50 +# Total queries = 12,000 GET requests (~4,000 per filter shape) + +duration = "60s" +rps = 200 +workers = 50 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_groups" "filter_perf_groups" { + weight = 100 + config { + workload = "group_list" + seed_group_count = 5000 + filters = [ + "", + "displayName eq \"nonexistent-group\"", + "displayName eq \"$SEED_GROUPNAME\"", + ] + } +} diff --git a/test-fixtures/configs/scim/scim_group_membership_stress.hcl b/test-fixtures/configs/scim/scim_group_membership_stress.hcl new file mode 100644 index 00000000..20f407be --- /dev/null +++ b/test-fixtures/configs/scim/scim_group_membership_stress.hcl @@ -0,0 +1,46 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# Extreme Group Membership Payload Stress Test +# Task Question: "What if the IdP sends groups with too many memberships?" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Isolates the exact latency and write cost of group membership payloads. +# +# user_seed_count is set to 3x members_per_group (not 1x): the member-selection +# window rotates through the seed pool per group +# (offset = poolIdx * members_per_group mod user_seed_count), so with equal +# counts every group would get the exact same members_per_group members. 3x +# headroom gives 3 distinct rotating membership sets instead of one, without +# materially changing the per-request payload size being measured. +# +# HOW TO RUN THE STEP-UP TEST: +# 1. Run Baseline (0 members) — establishes the floor latency: +# Change members_per_group = 0, user_seed_count = 1 +# 2. Increment membership sizes across subsequent runs (user_seed_count = 3x): +# members_per_group = 50, user_seed_count = 150 +# members_per_group = 100, user_seed_count = 300 +# members_per_group = 250, user_seed_count = 750 +# members_per_group = 500, user_seed_count = 1500 +# members_per_group = 1000, user_seed_count = 3000 +# members_per_group = 2500, user_seed_count = 7500 (Extreme stress) +# 3. Compare mean and p99 vs the 0-member baseline to isolate membership cost. + +duration = "60s" +rps = 50 +workers = 25 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_groups" "stress_large_membership" { + weight = 100 + config { + workload = "group_create" + group_count = 5000 + members_per_group = 100 + user_seed_count = 300 + } +} diff --git a/test-fixtures/configs/scim_scale.hcl b/test-fixtures/configs/scim/scim_scale.hcl similarity index 98% rename from test-fixtures/configs/scim_scale.hcl rename to test-fixtures/configs/scim/scim_scale.hcl index 681afa40..9ecbea89 100644 --- a/test-fixtures/configs/scim_scale.hcl +++ b/test-fixtures/configs/scim/scim_scale.hcl @@ -23,7 +23,7 @@ # export VAULT_NAMESPACE=admin # if targeting the admin namespace # # Run: -# vault-benchmark run -config=test-fixtures/configs/scim_scale.hcl +# vault-benchmark run -config=test-fixtures/configs/scim/scim_scale.hcl # # Sizing formula (adjust rps and duration together): # pool_size = 2 × rps × duration_seconds × (weight / 100) diff --git a/test-fixtures/configs/scim/scim_user_adopt_spike.hcl b/test-fixtures/configs/scim/scim_user_adopt_spike.hcl new file mode 100644 index 00000000..0a6d89ab --- /dev/null +++ b/test-fixtures/configs/scim/scim_user_adopt_spike.hcl @@ -0,0 +1,37 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# User Adoption Spike Test +# Task Question: "What if those user creations are adoptions?" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Simulates an IdP pushing a high-rate spike of user adoptions. +# Setup seeds unmanaged entities with aliases. During the attack, each POST +# targets an unmanaged entity alias, forcing Vault to execute the adoption path. +# +# Run at step-up RPS to compare directly against creation spike: +# Run 1: rps = 100, workers = 100 (Moderate spike) +# Run 2: rps = 250, workers = 250 (Heavy spike) +# Run 3: rps = 500, workers = 500 (Extreme spike / breaking limit) +# +# user_count carries 25% headroom over the nominal rps x duration request count +# (100 x 60s = 6,000) so a timing overshoot can't wrap the pool index back onto +# an already-adopted userName, which would surface as a false 409 conflict +# instead of a fresh adoption. + +duration = "60s" +rps = 100 +workers = 100 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_users" "spike_user_adopt" { + weight = 100 + config { + workload = "user_adopt" + user_count = 7500 + } +} diff --git a/test-fixtures/configs/scim/scim_user_create_spike.hcl b/test-fixtures/configs/scim/scim_user_create_spike.hcl new file mode 100644 index 00000000..725dda6a --- /dev/null +++ b/test-fixtures/configs/scim/scim_user_create_spike.hcl @@ -0,0 +1,31 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# User Creation Spike Test +# Task Question: "What happens if an IdP sends a spike of user creations?" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Simulates an IdP pushing a high-rate spike of brand new user creations. +# Every POST hits POST /v1/identity/scim/v2/Users with a unique userName. +# +# Run at step-up RPS to observe queueing and find the saturation point: +# Run 1: rps = 100, workers = 100 (Moderate spike) +# Run 2: rps = 250, workers = 250 (Heavy spike) +# Run 3: rps = 500, workers = 500 (Extreme spike / breaking limit) + +duration = "60s" +rps = 100 +workers = 100 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_users" "spike_user_create" { + weight = 100 + config { + workload = "user_create" + user_count = 35000 + } +} diff --git a/test-fixtures/configs/scim/scim_user_filter_perf.hcl b/test-fixtures/configs/scim/scim_user_filter_perf.hcl new file mode 100644 index 00000000..6fb361d8 --- /dev/null +++ b/test-fixtures/configs/scim/scim_user_filter_perf.hcl @@ -0,0 +1,49 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# GET /Users Filter Performance Under Load +# Task Question: "List GET Users performs well, with different filters" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Exercises GET /v1/identity/scim/v2/Users across four distinct filter shapes +# at high query throughput to identify filter scan vs index lookup bottlenecks: +# 1. "" — Full scan (no filter, worst-case latency) +# 2. userName eq "..." — Point lookup by exact userName +# 3. active eq true — Boolean attribute scan +# 4. meta.lastModified gt "..." — Time-range scan +# +# seed_user_count pre-creates 5,000 SCIM-owned users during setup (before the +# timed run starts) so every filter shape above has real, findable data to +# query instead of running against an empty namespace. The userName filter +# uses the $SEED_USERNAME token, which Setup() substitutes with the actual +# name of pre-seeded user #0 — a hardcoded literal can never match a +# runID-scoped name and would always return zero results. +# +# Math: +# duration = 60s +# rps = 200 +# workers = 50 +# Total queries = 12,000 GET requests (~3,000 per filter shape) + +duration = "60s" +rps = 200 +workers = 50 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_users" "filter_perf_users" { + weight = 100 + config { + workload = "user_list" + seed_user_count = 5000 + filters = [ + "", + "userName eq \"$SEED_USERNAME\"", + "active eq true", + "meta.lastModified gt \"2020-01-01T00:00:00Z\"", + ] + } +} diff --git a/test-fixtures/configs/scim/scim_user_scale_limit.hcl b/test-fixtures/configs/scim/scim_user_scale_limit.hcl new file mode 100644 index 00000000..f574ab13 --- /dev/null +++ b/test-fixtures/configs/scim/scim_user_scale_limit.hcl @@ -0,0 +1,33 @@ +# Copyright IBM Corp. 2022, 2026 +# SPDX-License-Identifier: MPL-2.0 + +# ══════════════════════════════════════════════════════════════════════════════ +# Maximum User Scale Limit Test (Pushing past 9,000 to 12,000+ users) +# Task Question: "Max users expected from IdPs is ~9000, test the limit" +# ══════════════════════════════════════════════════════════════════════════════ +# +# Simulates a continuous, sustained IdP sync stream creating > 10,000 unique users. +# Evaluates whether Vault memory, Raft log, or identity store degrade as total +# user count surpasses 9,000. +# +# Math: +# duration = 400s (~6.6 minutes) +# rps = 30 +# workers = 25 +# Total unique users created = 30 rps × 400s = 12,000 users + +duration = "400s" +rps = 30 +workers = 25 +log_level = "DEBUG" +report_mode = "verbose" +random_mounts = true +cleanup = true + +test "scim_users" "scale_user_limit_past_9k" { + weight = 100 + config { + workload = "user_create" + user_count = 15000 + } +} diff --git a/test-fixtures/configs/scim_membership.hcl b/test-fixtures/configs/scim_membership.hcl deleted file mode 100644 index 8241b685..00000000 --- a/test-fixtures/configs/scim_membership.hcl +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright IBM Corp. 2022, 2026 -# SPDX-License-Identifier: MPL-2.0 - -# SCIM performance benchmark — large membership stress config. -# -# Purpose: isolate the cost of group membership at extreme sizes. -# "What if the IdP sends groups with too many memberships?" -# -# HOW TO USE -# ---------- -# Step 1: Run OPTION B first (empty baseline) — establishes the floor. -# Step 2: Uncomment OPTION A and comment out OPTION B. -# Change members_per_group and user_seed_count together across runs: -# members_per_group = 10, user_seed_count = 10 -# members_per_group = 50, user_seed_count = 50 -# members_per_group = 100, user_seed_count = 100 -# members_per_group = 250, user_seed_count = 250 -# members_per_group = 500, user_seed_count = 500 -# members_per_group = 1000, user_seed_count = 1000 -# Step 3: Record mean and p99 for each run. The delta vs OPTION B = membership overhead. -# Step 4: Optionally run OPTION C to measure adoption in isolation. -# -# NOTE ON RPS -# ----------- -# rps=50 is intentionally conservative. From a remote client the effective -# throughput is limited by network RTT (~110ms to HVD), not Vault capacity. -# The goal here is relative comparison (empty vs N members), not absolute -# throughput — rps=50 gives enough samples for stable p99 without overwhelming -# the connection pool. -# -# WHAT TO RECORD PER RUN -# ---------------------- -# members_per_group | mean | p95 | p99 | successRatio -# ------------------|------|-----|-----|------------- -# 0 (baseline) | | | | -# 10 | | | | -# 50 | | | | -# 100 | | | | -# 250 | | | | -# 500 | | | | -# 1000 | | | | -# -# Prerequisites: -# export VAULT_ADDR="https://:8200" -# export VAULT_TOKEN= -# export VAULT_NAMESPACE=admin -# -# Run: -# vault-benchmark run -config=test-fixtures/configs/scim_membership.hcl - -duration = "120s" -rps = 50 -report_mode = "terse" -random_mounts = true -cleanup = true - -# ── OPTION B: empty baseline (run this first) ───────────────────────────────── -# -# Establishes the floor: what does group creation cost with zero members? -# Record mean and p99. Every other run is measured as delta from these numbers. -# -# pool = 50 rps × 120s × 2 = 12 000 - -# test "scim_groups" "scim_group_create_empty_baseline" { -# weight = 100 -# config { -# workload = "group_create_empty" -# group_count = 12000 -# } -# } - -# ── OPTION A: group_create with N members (comment in, comment out B) ───────── -# -# Change members_per_group and user_seed_count to the same value each run. -# user_seed_count members are created during setup and sampled round-robin — -# so user_seed_count=100 means 100 unique member entities are reused across all -# group POST bodies (you don't need group_count × members_per_group users). -# -test "scim_groups" "scim_group_create_large_membership" { - weight = 100 - config { - workload = "group_create" - group_count = 12000 - members_per_group = 100 - user_seed_count = 100 - } -} - -# ── OPTION C: group_adopt in isolation (optional) ───────────────────────────── -# -# Does adoption latency stay flat as the number of adoptable groups grows? -# Vary group_count across runs: 500, 2000, 5000, 9000. -# Compare mean and p99 — if they climb, the adoption lookup is O(n). -# -# test "scim_groups" "scim_group_adopt_isolated" { -# weight = 100 -# config { -# workload = "group_adopt" -# group_count = 12000 -# } -# } diff --git a/test-fixtures/configs/scim_spike_user_adopt.hcl b/test-fixtures/configs/scim_spike_user_adopt.hcl deleted file mode 100644 index 7fc8de10..00000000 --- a/test-fixtures/configs/scim_spike_user_adopt.hcl +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright IBM Corp. 2022, 2026 -# SPDX-License-Identifier: MPL-2.0 - -# SCIM performance benchmark — Pure 9,000 User Adoption Spike. -# -# Purpose: Simulates an IdP syncing ~9,000 users that already exist as -# unmanaged Vault entities (e.g. from prior SAML/OIDC logins) and adopting them into SCIM. -# -# Math: -# duration = 300s (5 minutes) -# rps = 30 -# Total requests = 30 rps × 300s = 9,000 adoptions (POST /v1/identity/scim/v2/Users) -# Pre-seeded adoption pool = 10,000 (pre-seeded during setup across 16 parallel workers) - -duration = "300s" -rps = 30 -workers = 25 -report_mode = "terse" -random_mounts = true -cleanup = true - -test "scim_users" "pure_user_adoption_spike" { - weight = 100 - config { - workload = "user_adopt" - user_count = 10000 - } -} diff --git a/test-fixtures/configs/scim_spike_user_create.hcl b/test-fixtures/configs/scim_spike_user_create.hcl deleted file mode 100644 index 6912887e..00000000 --- a/test-fixtures/configs/scim_spike_user_create.hcl +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright IBM Corp. 2022, 2026 -# SPDX-License-Identifier: MPL-2.0 - -# SCIM performance benchmark — Pure 9,000 User Creation Spike. -# -# Purpose: Simulates an IdP (e.g. Entra ID / Okta) doing an initial sync -# of ~9,000 new users directly into Vault SCIM. -# -# Math: -# duration = 300s (5 minutes) -# rps = 30 -# Total requests = 30 rps × 300s = 9,000 user creations (POST /v1/identity/scim/v2/Users) -# Pool size = 12,000 (headroom to ensure every user is unique) - -duration = "300s" -rps = 30 -workers = 25 -report_mode = "terse" -random_mounts = true -cleanup = true - -test "scim_users" "pure_user_creation_spike" { - weight = 100 - config { - workload = "user_create" - user_count = 12000 - } -} From e94f3235f968bb73611f42602289362c1c45c392 Mon Sep 17 00:00:00 2001 From: Anjani Mallampati Date: Thu, 8 Oct 2026 18:06:39 -0700 Subject: [PATCH 3/3] address PR comments --- benchmarktests/concurrency_helper.go | 6 ----- benchmarktests/scim_helper.go | 40 +++++++++++++++++++--------- benchmarktests/target_scim_groups.go | 18 ++++++++++--- benchmarktests/target_scim_users.go | 22 ++++++++++----- run_all_scim_benchmarks.sh | 32 ---------------------- 5 files changed, 57 insertions(+), 61 deletions(-) delete mode 100755 run_all_scim_benchmarks.sh diff --git a/benchmarktests/concurrency_helper.go b/benchmarktests/concurrency_helper.go index 6db73831..06de68d7 100644 --- a/benchmarktests/concurrency_helper.go +++ b/benchmarktests/concurrency_helper.go @@ -22,12 +22,6 @@ const ( // backend), while KV setup hits the secrets engine (may benefit from n>1 on // integrated-storage clusters). Both default to 1 (serial). To experiment locally, // raise the relevant constant; do not raise on production clusters without profiling. - // - // NOTE for SCIM benchmarks: the SCIM seeding helpers (scim_helper.go, - // target_scim_users.go, target_scim_groups.go) create large numbers of identity - // entities/groups during setup. Raising identityConcurrency to ~16 cuts that - // setup time significantly. Before running the SCIM benchmark suite, temporarily - // set identityConcurrency = 16 here, then restore it to 1 before committing. identityConcurrency = 1 kvSeedConcurrency = 1 ) diff --git a/benchmarktests/scim_helper.go b/benchmarktests/scim_helper.go index a822e0ea..9134ddd7 100644 --- a/benchmarktests/scim_helper.go +++ b/benchmarktests/scim_helper.go @@ -25,6 +25,11 @@ func generateUUID() (string, error) { } const ( + // scimDefaultSeedConcurrency is the default number of parallel workers used + // when seeding SCIM test data during setup/teardown. Override per test block + // with the seed_concurrency config option. + scimDefaultSeedConcurrency = 16 + scimPolicyName = "scim-bench-policy" scimMountBase = "userpass" scimUserPassword = "scim-bench-pw" @@ -93,6 +98,10 @@ type scimSharedClient struct { namespace string refCount int // number of test blocks currently holding this instance + // concurrency is the highest seed_concurrency requested by any block sharing + // this client. It is used for the teardown sweep, which runs once for all blocks. + concurrency int + // adoptUserNames / adoptGroupIDs accumulate every name seeded by user_adopt / // group_adopt blocks sharing this client. Only the subset actually adopted // during the attack gets removed by delete-linked-resources; the last @@ -128,7 +137,7 @@ func scimSharedKey(client *api.Client) string { // vs scimCreateUserName, etc.). // - A new run always gets a new runID, so there are no name collisions with // any state a previous run may have left behind on a persistent cluster. -func scimAcquireSharedClient(client *api.Client) (*scimSharedClient, error) { +func scimAcquireSharedClient(client *api.Client, concurrency int) (*scimSharedClient, error) { key := scimSharedKey(client) scimSharedMu.Lock() @@ -136,6 +145,9 @@ func scimAcquireSharedClient(client *api.Client) (*scimSharedClient, error) { if sc, ok := scimSharedClients[key]; ok { sc.refCount++ + if concurrency > sc.concurrency { + sc.concurrency = concurrency + } return sc, nil } @@ -158,6 +170,7 @@ func scimAcquireSharedClient(client *api.Client) (*scimSharedClient, error) { aliasMountAccessor: accessor, namespace: ns, refCount: 1, + concurrency: concurrency, } scimSharedClients[key] = sc return sc, nil @@ -189,6 +202,7 @@ func scimReleaseSharedClient(client *api.Client) error { runID := sc.runID adoptUserNames := sc.adoptUserNames adoptGroupIDs := sc.adoptGroupIDs + concurrency := sc.concurrency scimSharedMu.Unlock() var errs []error @@ -199,10 +213,10 @@ func scimReleaseSharedClient(client *api.Client) error { // SCIMCleanup's delete-linked-resources=true already removed every entity/group // that WAS adopted during the run. Anything from the seed pools still present // at this point was never adopted, so it's safe to delete directly. - if err := sweepUnadoptedEntities(client, adoptUserNames); err != nil { + if err := sweepUnadoptedEntities(client, concurrency, adoptUserNames); err != nil { errs = append(errs, err) } - if err := sweepUnadoptedGroups(client, adoptGroupIDs); err != nil { + if err := sweepUnadoptedGroups(client, concurrency, adoptGroupIDs); err != nil { errs = append(errs, err) } return errors.Join(errs...) @@ -211,8 +225,8 @@ func scimReleaseSharedClient(client *api.Client) error { // sweepUnadoptedEntities deletes any entity in names that Vault still has. // Deleting an already-gone entity by name is a documented no-op (no error), so // this is safe to call unconditionally against the full user_adopt seed pool. -func sweepUnadoptedEntities(client *api.Client, names []string) error { - return deletePhase(targetLogger, "sweep unadopted seed entities", client, "identity/entity/name/", identityConcurrency, len(names), func(idx int) string { +func sweepUnadoptedEntities(client *api.Client, concurrency int, names []string) error { + return deletePhase(targetLogger, "sweep unadopted seed entities", client, "identity/entity/name/", concurrency, len(names), func(idx int) string { return names[idx] }) } @@ -220,8 +234,8 @@ func sweepUnadoptedEntities(client *api.Client, names []string) error { // sweepUnadoptedGroups deletes any group in ids that Vault still has. // Deleting an already-gone group by ID is a documented no-op (no error), so // this is safe to call unconditionally against the full group_adopt seed pool. -func sweepUnadoptedGroups(client *api.Client, ids []string) error { - return deletePhase(targetLogger, "sweep unadopted seed groups", client, "identity/group/id/", identityConcurrency, len(ids), func(idx int) string { +func sweepUnadoptedGroups(client *api.Client, concurrency int, ids []string) error { + return deletePhase(targetLogger, "sweep unadopted seed groups", client, "identity/group/id/", concurrency, len(ids), func(idx int) string { return ids[idx] }) } @@ -474,8 +488,8 @@ func scimDeleteViaSCIMAPI(client *api.Client, scimToken, namespace, resourceType // scimDeletePhase bulk-deletes SCIM resources using the SCIM token (required // for SCIM-owned entities/groups that the admin identity API refuses to delete). -func scimDeletePhase(logger hclog.Logger, phase, resourceType string, client *api.Client, scimToken, namespace string, ids []string) error { - return runPhase(logger, phase, identityConcurrency, len(ids), func(i int) error { +func scimDeletePhase(logger hclog.Logger, phase, resourceType string, client *api.Client, scimToken, namespace string, concurrency int, ids []string) error { + return runPhase(logger, phase, concurrency, len(ids), func(i int) error { return scimDeleteViaSCIMAPI(client, scimToken, namespace, resourceType, ids[i]) }) } @@ -559,11 +573,11 @@ func scimGroupAdoptBody(displayName string) []byte { // Vault entity IDs. scope keeps name pools distinct across callers that share // a runID — e.g. "gm" for group members seeded by group_create, "lf" for // list-filter seed data seeded by user_list. -func seedSCIMUsers(logger hclog.Logger, adminClient *api.Client, scimToken, runID, scope, namespace string, count int) ([]string, error) { +func seedSCIMUsers(logger hclog.Logger, adminClient *api.Client, scimToken, runID, scope, namespace string, concurrency, count int) ([]string, error) { entityIDs := make([]string, count) header := scimHeader(scimToken, namespace) - err := runPhase(logger, "seed scim users", identityConcurrency, count, func(i int) error { + err := runPhase(logger, "seed scim users", concurrency, count, func(i int) error { userName := scimCreateUserName(runID, scope, i) externalID := "seed-ext-" + userName body := scimUserCreateBody(userName, externalID) @@ -611,10 +625,10 @@ func seedSCIMUsers(logger hclog.Logger, adminClient *api.Client, scimToken, runI // seedSCIMGroups creates SCIM-owned groups (no members) via the SCIM API. // Used to give group_list filter tests real, findable data to query against. // scope keeps this pool distinct from other group name pools sharing the runID. -func seedSCIMGroups(logger hclog.Logger, adminClient *api.Client, scimToken, runID, scope, namespace string, count int) error { +func seedSCIMGroups(logger hclog.Logger, adminClient *api.Client, scimToken, runID, scope, namespace string, concurrency, count int) error { header := scimHeader(scimToken, namespace) - return runPhase(logger, "seed scim groups", identityConcurrency, count, func(i int) error { + return runPhase(logger, "seed scim groups", concurrency, count, func(i int) error { displayName := scimCreateGroupDisplayName(runID, scope, i) body := scimGroupCreateBody(displayName, nil) diff --git a/benchmarktests/target_scim_groups.go b/benchmarktests/target_scim_groups.go index d77502c8..27955775 100644 --- a/benchmarktests/target_scim_groups.go +++ b/benchmarktests/target_scim_groups.go @@ -109,6 +109,11 @@ type SCIMGroupsConfig struct { // SeedGroupCount pre-seeds this many SCIM-owned groups during setup, purely as // queryable data for group_list filter tests. Ignored for other workloads. SeedGroupCount int `hcl:"seed_group_count,optional"` + + // SeedConcurrency is the number of parallel workers used for setup seeding + // (group_adopt groups, group member users, group_list seed groups) and for + // the teardown sweep. Defaults to 16. It does not affect the measured attack phase. + SeedConcurrency int `hcl:"seed_concurrency,optional"` } func (s *SCIMGroups) ParseConfig(body hcl.Body) error { @@ -120,6 +125,7 @@ func (s *SCIMGroups) ParseConfig(body hcl.Body) error { GroupCount: 100, MembersPerGroup: 0, Filters: []string{""}, + SeedConcurrency: scimDefaultSeedConcurrency, }, } @@ -131,6 +137,10 @@ func (s *SCIMGroups) ParseConfig(body hcl.Body) error { c := testConfig.Config s.config = c + if c.SeedConcurrency < 1 { + return fmt.Errorf("scim_groups: seed_concurrency must be >= 1") + } + switch c.Workload { case scimGroupWorkloadCreate, scimGroupWorkloadCreateEmpty, scimGroupWorkloadAdopt: if c.GroupCount <= 0 { @@ -163,7 +173,7 @@ func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig // Acquire the shared SCIM client. s.logger.Info("scim_groups setup: acquiring shared SCIM client", "workload", s.config.Workload) - sc, err := scimAcquireSharedClient(client) + sc, err := scimAcquireSharedClient(client, s.config.SeedConcurrency) if err != nil { return nil, fmt.Errorf("scim_groups setup: %w", err) } @@ -200,7 +210,7 @@ func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig seedCount = s.config.MembersPerGroup } s.logger.Info("scim_groups setup: seeding SCIM-owned users for group membership", "count", seedCount) - entityIDs, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, "gm", sc.namespace, seedCount) + entityIDs, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, "gm", sc.namespace, s.config.SeedConcurrency, seedCount) if err != nil { return nil, fmt.Errorf("scim_groups setup: seeding member users: %w", err) } @@ -213,7 +223,7 @@ func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig groupIDs := make([]string, s.config.GroupCount) groupNames := make([]string, s.config.GroupCount) - err = runPhase(s.logger, "seed adoption groups", identityConcurrency, s.config.GroupCount, func(i int) error { + err = runPhase(s.logger, "seed adoption groups", s.config.SeedConcurrency, s.config.GroupCount, func(i int) error { name := scimAdoptGroupName(sc.runID, i) resp, err := client.Logical().Write("identity/group", map[string]any{ "name": name, @@ -245,7 +255,7 @@ func (s *SCIMGroups) Setup(client *api.Client, mountName string, topLevelConfig // instead of running against an empty namespace. if s.config.SeedGroupCount > 0 { s.logger.Info("scim_groups setup: seeding groups for filter tests", "count", s.config.SeedGroupCount) - if err := seedSCIMGroups(s.logger, client, sc.scimToken, sc.runID, "lg", sc.namespace, s.config.SeedGroupCount); err != nil { + if err := seedSCIMGroups(s.logger, client, sc.scimToken, sc.runID, "lg", sc.namespace, s.config.SeedConcurrency, s.config.SeedGroupCount); err != nil { return nil, fmt.Errorf("scim_groups setup: seeding filter-test groups: %w", err) } } diff --git a/benchmarktests/target_scim_users.go b/benchmarktests/target_scim_users.go index f688a66a..07d28991 100644 --- a/benchmarktests/target_scim_users.go +++ b/benchmarktests/target_scim_users.go @@ -96,6 +96,11 @@ type SCIMUsersConfig struct { // SeedUserCount pre-seeds this many SCIM-owned users during setup, purely as // queryable data for user_list filter tests. Ignored for other workloads. SeedUserCount int `hcl:"seed_user_count,optional"` + + // SeedConcurrency is the number of parallel workers used for setup seeding + // (user_adopt entities, user_list seed users) and for the teardown sweep. + // Defaults to 16. It does not affect the measured attack phase. + SeedConcurrency int `hcl:"seed_concurrency,optional"` } func (s *SCIMUsers) ParseConfig(body hcl.Body) error { @@ -103,9 +108,10 @@ func (s *SCIMUsers) ParseConfig(body hcl.Body) error { Config *SCIMUsersConfig `hcl:"config,block"` }{ Config: &SCIMUsersConfig{ - Workload: scimUserWorkloadCreate, - UserCount: 200, - Filters: []string{""}, + Workload: scimUserWorkloadCreate, + UserCount: 200, + Filters: []string{""}, + SeedConcurrency: scimDefaultSeedConcurrency, }, } @@ -117,6 +123,10 @@ func (s *SCIMUsers) ParseConfig(body hcl.Body) error { c := testConfig.Config s.config = c + if c.SeedConcurrency < 1 { + return fmt.Errorf("scim_users: seed_concurrency must be >= 1") + } + switch c.Workload { case scimUserWorkloadCreate, scimUserWorkloadAdopt: if c.UserCount <= 0 { @@ -143,7 +153,7 @@ func (s *SCIMUsers) Setup(client *api.Client, mountName string, topLevelConfig * // Acquire the shared SCIM client — created once for the whole run, // reused by every scim_users and scim_groups test block. s.logger.Info("scim_users setup: acquiring shared SCIM client", "workload", s.config.Workload) - sc, err := scimAcquireSharedClient(client) + sc, err := scimAcquireSharedClient(client, s.config.SeedConcurrency) if err != nil { return nil, fmt.Errorf("scim_users setup: %w", err) } @@ -181,7 +191,7 @@ func (s *SCIMUsers) Setup(client *api.Client, mountName string, topLevelConfig * s.logger.Info("scim_users setup: seeding unmanaged adoption targets", "count", s.config.UserCount) aliasNames := make([]string, s.config.UserCount) - err = runPhase(s.logger, "seed adoption entities", identityConcurrency, s.config.UserCount, func(i int) error { + err = runPhase(s.logger, "seed adoption entities", s.config.SeedConcurrency, s.config.UserCount, func(i int) error { entityName := scimAdoptEntityName(sc.runID, i) aliasName := scimAdoptUserAliasName(sc.runID, i) @@ -221,7 +231,7 @@ func (s *SCIMUsers) Setup(client *api.Client, mountName string, topLevelConfig * // instead of running against an empty namespace. if s.config.SeedUserCount > 0 { s.logger.Info("scim_users setup: seeding users for filter tests", "count", s.config.SeedUserCount) - if _, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, "lf", sc.namespace, s.config.SeedUserCount); err != nil { + if _, err := seedSCIMUsers(s.logger, client, sc.scimToken, sc.runID, "lf", sc.namespace, s.config.SeedConcurrency, s.config.SeedUserCount); err != nil { return nil, fmt.Errorf("scim_users setup: seeding filter-test users: %w", err) } } diff --git a/run_all_scim_benchmarks.sh b/run_all_scim_benchmarks.sh deleted file mode 100755 index fc01e4b7..00000000 --- a/run_all_scim_benchmarks.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# NOTE: Before running this script, temporarily raise identityConcurrency in -# benchmarktests/concurrency_helper.go from 1 to ~16. The SCIM benchmarks seed -# large numbers of identity entities and groups, and the higher concurrency cuts -# setup time significantly. Restore the value to 1 before committing. -set -e - -echo "=== [1/8] Running User Creation Spike ===" -go run main.go run -config=test-fixtures/configs/scim/scim_user_create_spike.hcl > results_user_create_spike.log 2>&1 - -echo "=== [2/8] Running User Adoption Spike ===" -go run main.go run -config=test-fixtures/configs/scim/scim_user_adopt_spike.hcl > results_user_adopt_spike.log 2>&1 - -echo "=== [3/8] Running User Scale Limit (12k users) ===" -go run main.go run -config=test-fixtures/configs/scim/scim_user_scale_limit.hcl > results_user_scale_limit.log 2>&1 - -echo "=== [4/8] Running User Filter Performance ===" -go run main.go run -config=test-fixtures/configs/scim/scim_user_filter_perf.hcl > results_user_filter_perf.log 2>&1 - -echo "=== [5/8] Running Group Creation Spike ===" -go run main.go run -config=test-fixtures/configs/scim/scim_group_create_spike.hcl > results_group_create_spike.log 2>&1 - -echo "=== [6/8] Running Group Adoption Spike ===" -go run main.go run -config=test-fixtures/configs/scim/scim_group_adopt_spike.hcl > results_group_adopt_spike.log 2>&1 - -echo "=== [7/8] Running Group Membership Stress ===" -go run main.go run -config=test-fixtures/configs/scim/scim_group_membership_stress.hcl > results_group_membership_stress.log 2>&1 - -echo "=== [8/8] Running Group Filter Performance ===" -go run main.go run -config=test-fixtures/configs/scim/scim_group_filter_perf.hcl > results_group_filter_perf.log 2>&1 - -echo "=== All 8 SCIM Benchmark Tests Complete! ==="