Skip to content

ROX-35013: Replace full-table Walks with targeted WalkByQuery in compliance operator manager #21059

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dashrews78 wants to merge 1 commit into
base: dashrews/compliance-v1-startup
Choose a base branch
from
Draft

ROX-35013: Replace full-table Walks with targeted WalkByQuery in compliance operator manager #21059

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions central/complianceoperator/manager/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/stackrox/rox/pkg/logging"
"github.com/stackrox/rox/pkg/postgres/pgutils"
"github.com/stackrox/rox/pkg/sac"
"github.com/stackrox/rox/pkg/search"
"github.com/stackrox/rox/pkg/set"
"github.com/stackrox/rox/pkg/sync"
)
Expand Down Expand Up @@ -167,13 +168,14 @@ func (m *managerImpl) AddProfile(profile *storage.ComplianceOperatorProfile) err
}

func (m *managerImpl) addProfileNoLock(profile *storage.ComplianceOperatorProfile) error {
var existingProfiles []*storage.ComplianceOperatorProfile
existingProfiles := []*storage.ComplianceOperatorProfile{profile}
q := search.NewQueryBuilder().
AddExactMatches(search.ComplianceOperatorV1ProfileName, profile.GetName()).
ProtoQuery()
walkFn := func() error {
existingProfiles = []*storage.ComplianceOperatorProfile{
profile,
}
return m.profiles.Walk(allAccessCtx, func(existingProfile *storage.ComplianceOperatorProfile) error {
if existingProfile.GetClusterId() != profile.GetClusterId() && existingProfile.GetName() == profile.GetName() {
existingProfiles = []*storage.ComplianceOperatorProfile{profile}
return m.profiles.WalkByQuery(allAccessCtx, q, func(existingProfile *storage.ComplianceOperatorProfile) error {
if existingProfile.GetClusterId() != profile.GetClusterId() {
existingProfiles = append(existingProfiles, existingProfile)
}
return nil
Expand Down Expand Up @@ -274,8 +276,11 @@ func (m *managerImpl) DeleteProfile(deletedProfile *storage.ComplianceOperatorPr

var found bool
rulesFound := set.NewStringSet()
err := m.profiles.Walk(allAccessCtx, func(profile *storage.ComplianceOperatorProfile) error {
if deletedProfile.GetId() != profile.GetId() && deletedProfile.GetName() == profile.GetName() {
q := search.NewQueryBuilder().
AddExactMatches(search.ComplianceOperatorV1ProfileName, deletedProfile.GetName()).
ProtoQuery()
err := m.profiles.WalkByQuery(allAccessCtx, q, func(profile *storage.ComplianceOperatorProfile) error {
if deletedProfile.GetId() != profile.GetId() {
found = true
for _, rule := range profile.GetRules() {
rulesFound.Add(rule.GetName())
Expand Down Expand Up @@ -405,10 +410,13 @@ func (m *managerImpl) getRule(name string) (*storage.ComplianceOperatorRule, err

func (m *managerImpl) GetMachineConfigs(clusterID string) (map[string][]string, error) {
profileIDsToNames := make(map[string]string)
q := search.NewQueryBuilder().
AddExactMatches(search.ComplianceOperatorV1ProfileClusterID, clusterID).
ProtoQuery()
walkFn := func() error {
profileIDsToNames = make(map[string]string)
return m.profiles.Walk(allAccessCtx, func(profile *storage.ComplianceOperatorProfile) error {
if profile.GetClusterId() == clusterID && profile.GetAnnotations()[v1alpha1.ProductTypeAnnotation] == string(v1alpha1.ScanTypeNode) {
return m.profiles.WalkByQuery(allAccessCtx, q, func(profile *storage.ComplianceOperatorProfile) error {
if profile.GetAnnotations()[v1alpha1.ProductTypeAnnotation] == string(v1alpha1.ScanTypeNode) {
profileIDsToNames[profile.GetProfileId()] = profile.GetName()
}
return nil
Expand Down
11 changes: 11 additions & 0 deletions central/complianceoperator/profiles/datastore/datastore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/pkg/errors"
store "github.com/stackrox/rox/central/complianceoperator/profiles/store"
v1 "github.com/stackrox/rox/generated/api/v1"
"github.com/stackrox/rox/generated/storage"
"github.com/stackrox/rox/pkg/sac"
"github.com/stackrox/rox/pkg/sac/resources"
Expand All @@ -19,6 +20,7 @@ var (
//go:generate mockgen-wrapper
type DataStore interface {
Walk(ctx context.Context, fn func(result *storage.ComplianceOperatorProfile) error) error
WalkByQuery(ctx context.Context, query *v1.Query, fn func(result *storage.ComplianceOperatorProfile) error) error
Upsert(ctx context.Context, result *storage.ComplianceOperatorProfile) error
Delete(ctx context.Context, id string) error
}
Expand All @@ -42,6 +44,15 @@ func (d *datastoreImpl) Walk(ctx context.Context, fn func(result *storage.Compli
return d.store.Walk(ctx, fn)
}

func (d *datastoreImpl) WalkByQuery(ctx context.Context, query *v1.Query, fn func(result *storage.ComplianceOperatorProfile) error) error {
if ok, err := complianceOperatorSAC.ReadAllowed(ctx); err != nil {
return err
} else if !ok {
return errors.Wrap(sac.ErrResourceAccessDenied, "compliance operator profiles read")
}
return d.store.WalkByQuery(ctx, query, fn)
}

func (d *datastoreImpl) Upsert(ctx context.Context, result *storage.ComplianceOperatorProfile) error {
if ok, err := complianceOperatorSAC.WriteAllowed(ctx); err != nil {
return err
Expand Down
15 changes: 15 additions & 0 deletions central/complianceoperator/profiles/datastore/mocks/datastore.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion central/complianceoperator/profiles/store/postgres/gen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package postgres

//go:generate pg-table-bindings-wrapper --type=storage.ComplianceOperatorProfile
//go:generate pg-table-bindings-wrapper --type=storage.ComplianceOperatorProfile --search-category 201
11 changes: 10 additions & 1 deletion central/complianceoperator/profiles/store/postgres/store.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions central/complianceoperator/profiles/store/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package store
import (
"context"

v1 "github.com/stackrox/rox/generated/api/v1"
"github.com/stackrox/rox/generated/storage"
)

Expand All @@ -11,4 +12,5 @@ type Store interface {
Upsert(ctx context.Context, obj *storage.ComplianceOperatorProfile) error
Delete(ctx context.Context, id string) error
Walk(ctx context.Context, fn func(obj *storage.ComplianceOperatorProfile) error) error
WalkByQuery(ctx context.Context, query *v1.Query, fn func(obj *storage.ComplianceOperatorProfile) error) error
}
4 changes: 2 additions & 2 deletions generated/storage/compliance_operator.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions pkg/postgres/schema/compliance_operator_profiles.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pkg/search/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,10 @@ var (

TestShortCircuitID = newFieldLabel("Test ShortCircuit ID")

// Compliance Operator V1
ComplianceOperatorV1ProfileName = newFieldLabel("Compliance Operator V1 Profile Name")
ComplianceOperatorV1ProfileClusterID = newFieldLabel("Compliance Operator V1 Profile Cluster ID")

// Derived test fields
// The derived fields depending of fields with map and scalar data type array data structures are unsupported.
TestGrandparentCount = newDerivedFieldLabel("Test Grandparent Count", TestGrandparentID, CountDerivationType)
Expand Down
4 changes: 2 additions & 2 deletions proto/storage/compliance_operator.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ message ComplianceOperatorCheckResult {
message ComplianceOperatorProfile {
string id = 1; // @gotags: sql:"pk"
string profile_id = 2;
string name = 3;
string cluster_id = 4;
string name = 3; // @gotags: search:"Compliance Operator V1 Profile Name,hidden"
string cluster_id = 4; // @gotags: search:"Compliance Operator V1 Profile Cluster ID,hidden"
map<string, string> labels = 5;
map<string, string> annotations = 6;
string description = 7;
Expand Down
Loading