From 50f776231d3d0da8cc1d5d57b84e147af2bc65b0 Mon Sep 17 00:00:00 2001 From: Kevin Joiner <10265309+KevinJoiner@users.noreply.github.com> Date: Sun, 14 Sep 2025 16:10:22 -0400 Subject: [PATCH 1/3] refactor: use not nesting instead of negate --- pkg/eventrepo/event_repo_test.go | 20 ++++--- pkg/eventrepo/eventrepo.go | 68 +++++++++++---------- pkg/grpc/fetch-api.pb.go | 100 ++++++++++++++++--------------- pkg/grpc/fetch-api.proto | 17 +++--- 4 files changed, 111 insertions(+), 94 deletions(-) diff --git a/pkg/eventrepo/event_repo_test.go b/pkg/eventrepo/event_repo_test.go index 003c7a0..9523970 100644 --- a/pkg/eventrepo/event_repo_test.go +++ b/pkg/eventrepo/event_repo_test.go @@ -604,8 +604,9 @@ func TestListIndexesAdvanced(t *testing.T) { HasAny: []string{subject1}, }, Type: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeFingerprint}, - Negate: true, + Not: &grpc.StringFilterOption{ + HasAny: []string{cloudevent.TypeFingerprint}, + }, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, @@ -646,8 +647,9 @@ func TestListIndexesAdvanced(t *testing.T) { Type: &grpc.StringFilterOption{ HasAny: []string{cloudevent.TypeAttestation}, Or: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeStatus}, - Negate: true, + Not: &grpc.StringFilterOption{ + HasAny: []string{cloudevent.TypeStatus}, + }, }, }, }, @@ -709,8 +711,9 @@ func TestListIndexesAdvanced(t *testing.T) { HasAny: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAny: []string{"vehicle"}, - Negate: true, + Not: &grpc.ArrayFilterOption{ + HasAny: []string{"vehicle"}, + }, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeFingerprintSource2Producer2}, @@ -752,8 +755,9 @@ func TestListIndexesAdvanced(t *testing.T) { HasAny: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAny: []string{"fingerprint", "telemetry"}, - Negate: true, // Does NOT have fingerprint or telemetry + Not: &grpc.ArrayFilterOption{ + HasAny: []string{"fingerprint", "telemetry"}, + }, Or: &grpc.ArrayFilterOption{ HasAll: []string{"telemetry", "status"}, }, diff --git a/pkg/eventrepo/eventrepo.go b/pkg/eventrepo/eventrepo.go index 879d10a..2668059 100644 --- a/pkg/eventrepo/eventrepo.go +++ b/pkg/eventrepo/eventrepo.go @@ -375,85 +375,93 @@ func AdvancedSearchOptionsToQueryMod(opts *grpc.AdvancedSearchOptions) []qm.Quer // Handle advanced filtering for each field if opts.GetType() != nil { - mods = appendStringFilterMods(mods, opts.GetType(), chindexer.TypeColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetType(), chindexer.TypeColumn, false)...)) } if opts.GetDataVersion() != nil { - mods = appendStringFilterMods(mods, opts.GetDataVersion(), chindexer.DataVersionColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetDataVersion(), chindexer.DataVersionColumn, false)...)) } if opts.GetSubject() != nil { - mods = appendStringFilterMods(mods, opts.GetSubject(), chindexer.SubjectColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetSubject(), chindexer.SubjectColumn, false)...)) } if opts.GetSource() != nil { - mods = appendStringFilterMods(mods, opts.GetSource(), chindexer.SourceColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetSource(), chindexer.SourceColumn, false)...)) } if opts.GetProducer() != nil { - mods = appendStringFilterMods(mods, opts.GetProducer(), chindexer.ProducerColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetProducer(), chindexer.ProducerColumn, false)...)) } if opts.GetExtras() != nil { - mods = appendStringFilterMods(mods, opts.GetExtras(), chindexer.ExtrasColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetExtras(), chindexer.ExtrasColumn, false)...)) } if opts.GetId() != nil { - mods = appendStringFilterMods(mods, opts.GetId(), chindexer.IDColumn) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetId(), chindexer.IDColumn, false)...)) } if opts.GetTags() != nil { - mods = appendArrayFilterMods(mods, opts.GetTags(), tagsColumn) + mods = append(mods, qm.Expr(arrayFilterMods(opts.GetTags(), tagsColumn, false)...)) } return mods } -// appendStringFilterMods converts a StringFilterOption to query modifications. -func appendStringFilterMods(mods []qm.QueryMod, filter *grpc.StringFilterOption, columnName string) []qm.QueryMod { +// stringFilterMods converts a StringFilterOption to query modifications. +func stringFilterMods(filter *grpc.StringFilterOption, columnName string, negate bool) []qm.QueryMod { + var mods []qm.QueryMod if filter == nil { return nil } + var negateClause string + if negate { + negateClause = " NOT" + } // Process has_any (OR logic) if len(filter.GetHasAny()) > 0 { - if filter.GetNegate() { - mods = append(mods, qm.Where(columnName+" NOT IN ?", filter.GetHasAny())) - } else { - mods = append(mods, qm.WhereIn(columnName+" IN ?", filter.GetHasAny())) - } + mods = append(mods, qm.Where(columnName+negateClause+" IN (?)", filter.GetHasAny())) + } + if filter.GetNot() != nil { + orFilter := &grpc.StringFilterOption{Or: filter.GetNot()} + notMods := stringFilterMods(orFilter, columnName, !negate) + mods = append(mods, notMods...) } if filter.GetOr() != nil { - orMods := appendStringFilterMods(nil, filter.GetOr(), columnName) - if len(orMods) > 0 { - mods = append(mods, qm.Or2(qm.Expr(orMods...))) - } + orMods := stringFilterMods(filter.GetOr(), columnName, negate) + mods = append(mods, qm.Or2(qm.Expr(orMods...))) } return mods } -// appendArrayFilterMods converts an ArrayFilterOption to query modifications. -func appendArrayFilterMods(mods []qm.QueryMod, filter *grpc.ArrayFilterOption, columnName string) []qm.QueryMod { +// arrayFilterMods converts an ArrayFilterOption to query modifications. +func arrayFilterMods(filter *grpc.ArrayFilterOption, columnName string, negate bool) []qm.QueryMod { + var mods []qm.QueryMod if filter == nil { return mods } - var clause string - if filter.GetNegate() { - clause = "NOT " + var negateClause string + if negate { + negateClause = "NOT " } if len(filter.GetHasAny()) > 0 { - mods = append(mods, qm.Where(clause+"hasAny("+columnName+", ?)", filter.GetHasAny())) + mods = append(mods, qm.Where(negateClause+"hasAny("+columnName+", ?)", filter.GetHasAny())) } if len(filter.GetHasAll()) > 0 { - mods = append(mods, qm.Where(clause+"hasAll("+columnName+", ?)", filter.GetHasAll())) + mods = append(mods, qm.Where(negateClause+"hasAll("+columnName+", ?)", filter.GetHasAll())) + } + if filter.GetNot() != nil { + orFilter := &grpc.ArrayFilterOption{Or: filter.GetNot()} + newMods := arrayFilterMods(orFilter, columnName, !negate) + mods = append(mods, newMods...) } // Process OR condition recursively if filter.GetOr() != nil { - orMods := appendArrayFilterMods(nil, filter.GetOr(), columnName) - if len(orMods) > 0 { - mods = append(mods, qm.Or2(qm.Expr(orMods...))) - } + orMods := arrayFilterMods(filter.GetOr(), columnName, negate) + mods = append(mods, qm.Or2(qm.Expr(orMods...))) } return mods diff --git a/pkg/grpc/fetch-api.pb.go b/pkg/grpc/fetch-api.pb.go index 36d58aa..d7f10ad 100644 --- a/pkg/grpc/fetch-api.pb.go +++ b/pkg/grpc/fetch-api.pb.go @@ -296,8 +296,8 @@ type ArrayFilterOption struct { HasAny []string `protobuf:"bytes,1,rep,name=has_any,json=hasAny,proto3" json:"has_any,omitempty"` // Match if the field has all of these values. HasAll []string `protobuf:"bytes,2,rep,name=has_all,json=hasAll,proto3" json:"has_all,omitempty"` - // Negate all matches in this filter. - Negate bool `protobuf:"varint,3,opt,name=negate,proto3" json:"negate,omitempty"` + // Additional filter condition to combine with this one using NOT logic. + Not *ArrayFilterOption `protobuf:"bytes,3,opt,name=not,proto3" json:"not,omitempty"` // Additional filter condition to combine with this one using OR logic. Or *ArrayFilterOption `protobuf:"bytes,4,opt,name=or,proto3" json:"or,omitempty"` unknownFields protoimpl.UnknownFields @@ -348,11 +348,11 @@ func (x *ArrayFilterOption) GetHasAll() []string { return nil } -func (x *ArrayFilterOption) GetNegate() bool { +func (x *ArrayFilterOption) GetNot() *ArrayFilterOption { if x != nil { - return x.Negate + return x.Not } - return false + return nil } func (x *ArrayFilterOption) GetOr() *ArrayFilterOption { @@ -366,8 +366,8 @@ type StringFilterOption struct { state protoimpl.MessageState `protogen:"open.v1"` // Match if the field has any of these values (OR logic) HasAny []string `protobuf:"bytes,1,rep,name=has_any,json=hasAny,proto3" json:"has_any,omitempty"` - // Negate all matches in this filter. - Negate bool `protobuf:"varint,2,opt,name=negate,proto3" json:"negate,omitempty"` + // Additional filter condition to combine with this one using NOT logic. + Not *StringFilterOption `protobuf:"bytes,2,opt,name=not,proto3" json:"not,omitempty"` // Additional filter condition to combine with this one using OR logic. Or *StringFilterOption `protobuf:"bytes,3,opt,name=or,proto3" json:"or,omitempty"` unknownFields protoimpl.UnknownFields @@ -411,11 +411,11 @@ func (x *StringFilterOption) GetHasAny() []string { return nil } -func (x *StringFilterOption) GetNegate() bool { +func (x *StringFilterOption) GetNot() *StringFilterOption { if x != nil { - return x.Negate + return x.Not } - return false + return nil } func (x *StringFilterOption) GetOr() *StringFilterOption { @@ -1065,15 +1065,15 @@ const file_pkg_grpc_fetch_api_proto_rawDesc = "" + "\x06extras\x18\n" + " \x01(\v2\x18.grpc.StringFilterOptionR\x06extras\x12(\n" + "\x02id\x18\v \x01(\v2\x18.grpc.StringFilterOptionR\x02id\x12+\n" + - "\x04tags\x18\f \x01(\v2\x17.grpc.ArrayFilterOptionR\x04tags\"\x86\x01\n" + + "\x04tags\x18\f \x01(\v2\x17.grpc.ArrayFilterOptionR\x04tags\"\x99\x01\n" + "\x11ArrayFilterOption\x12\x17\n" + "\ahas_any\x18\x01 \x03(\tR\x06hasAny\x12\x17\n" + - "\ahas_all\x18\x02 \x03(\tR\x06hasAll\x12\x16\n" + - "\x06negate\x18\x03 \x01(\bR\x06negate\x12'\n" + - "\x02or\x18\x04 \x01(\v2\x17.grpc.ArrayFilterOptionR\x02or\"o\n" + + "\ahas_all\x18\x02 \x03(\tR\x06hasAll\x12)\n" + + "\x03not\x18\x03 \x01(\v2\x17.grpc.ArrayFilterOptionR\x03not\x12'\n" + + "\x02or\x18\x04 \x01(\v2\x17.grpc.ArrayFilterOptionR\x02or\"\x83\x01\n" + "\x12StringFilterOption\x12\x17\n" + - "\ahas_any\x18\x01 \x03(\tR\x06hasAny\x12\x16\n" + - "\x06negate\x18\x02 \x01(\bR\x06negate\x12(\n" + + "\ahas_any\x18\x01 \x03(\tR\x06hasAny\x12*\n" + + "\x03not\x18\x02 \x01(\v2\x18.grpc.StringFilterOptionR\x03not\x12(\n" + "\x02or\x18\x03 \x01(\v2\x18.grpc.StringFilterOptionR\x02or\"m\n" + "\x0fCloudEventIndex\x124\n" + "\x06header\x18\x01 \x01(\v2\x1c.cloudevent.CloudEventHeaderR\x06header\x12$\n" + @@ -1173,39 +1173,41 @@ var file_pkg_grpc_fetch_api_proto_depIdxs = []int32{ 3, // 18: grpc.AdvancedSearchOptions.extras:type_name -> grpc.StringFilterOption 3, // 19: grpc.AdvancedSearchOptions.id:type_name -> grpc.StringFilterOption 2, // 20: grpc.AdvancedSearchOptions.tags:type_name -> grpc.ArrayFilterOption - 2, // 21: grpc.ArrayFilterOption.or:type_name -> grpc.ArrayFilterOption - 3, // 22: grpc.StringFilterOption.or:type_name -> grpc.StringFilterOption - 19, // 23: grpc.CloudEventIndex.header:type_name -> cloudevent.CloudEventHeader - 5, // 24: grpc.CloudEventIndex.data:type_name -> grpc.ObjectInfo - 0, // 25: grpc.GetLatestIndexRequest.options:type_name -> grpc.SearchOptions - 1, // 26: grpc.GetLatestIndexRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 4, // 27: grpc.GetLatestIndexResponse.index:type_name -> grpc.CloudEventIndex - 0, // 28: grpc.ListIndexesRequest.options:type_name -> grpc.SearchOptions - 1, // 29: grpc.ListIndexesRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 4, // 30: grpc.ListIndexesResponse.indexes:type_name -> grpc.CloudEventIndex - 0, // 31: grpc.GetLatestCloudEventRequest.options:type_name -> grpc.SearchOptions - 1, // 32: grpc.GetLatestCloudEventRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 20, // 33: grpc.GetLatestCloudEventResponse.cloud_event:type_name -> cloudevent.CloudEvent - 0, // 34: grpc.ListCloudEventsRequest.options:type_name -> grpc.SearchOptions - 1, // 35: grpc.ListCloudEventsRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 20, // 36: grpc.ListCloudEventsResponse.cloud_events:type_name -> cloudevent.CloudEvent - 4, // 37: grpc.ListCloudEventsFromKeysRequest.indexes:type_name -> grpc.CloudEventIndex - 20, // 38: grpc.ListCloudEventsFromKeysResponse.cloud_events:type_name -> cloudevent.CloudEvent - 6, // 39: grpc.FetchService.GetLatestIndex:input_type -> grpc.GetLatestIndexRequest - 8, // 40: grpc.FetchService.ListIndexes:input_type -> grpc.ListIndexesRequest - 10, // 41: grpc.FetchService.GetLatestCloudEvent:input_type -> grpc.GetLatestCloudEventRequest - 12, // 42: grpc.FetchService.ListCloudEvents:input_type -> grpc.ListCloudEventsRequest - 14, // 43: grpc.FetchService.ListCloudEventsFromIndex:input_type -> grpc.ListCloudEventsFromKeysRequest - 7, // 44: grpc.FetchService.GetLatestIndex:output_type -> grpc.GetLatestIndexResponse - 9, // 45: grpc.FetchService.ListIndexes:output_type -> grpc.ListIndexesResponse - 11, // 46: grpc.FetchService.GetLatestCloudEvent:output_type -> grpc.GetLatestCloudEventResponse - 13, // 47: grpc.FetchService.ListCloudEvents:output_type -> grpc.ListCloudEventsResponse - 15, // 48: grpc.FetchService.ListCloudEventsFromIndex:output_type -> grpc.ListCloudEventsFromKeysResponse - 44, // [44:49] is the sub-list for method output_type - 39, // [39:44] is the sub-list for method input_type - 39, // [39:39] is the sub-list for extension type_name - 39, // [39:39] is the sub-list for extension extendee - 0, // [0:39] is the sub-list for field type_name + 2, // 21: grpc.ArrayFilterOption.not:type_name -> grpc.ArrayFilterOption + 2, // 22: grpc.ArrayFilterOption.or:type_name -> grpc.ArrayFilterOption + 3, // 23: grpc.StringFilterOption.not:type_name -> grpc.StringFilterOption + 3, // 24: grpc.StringFilterOption.or:type_name -> grpc.StringFilterOption + 19, // 25: grpc.CloudEventIndex.header:type_name -> cloudevent.CloudEventHeader + 5, // 26: grpc.CloudEventIndex.data:type_name -> grpc.ObjectInfo + 0, // 27: grpc.GetLatestIndexRequest.options:type_name -> grpc.SearchOptions + 1, // 28: grpc.GetLatestIndexRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 4, // 29: grpc.GetLatestIndexResponse.index:type_name -> grpc.CloudEventIndex + 0, // 30: grpc.ListIndexesRequest.options:type_name -> grpc.SearchOptions + 1, // 31: grpc.ListIndexesRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 4, // 32: grpc.ListIndexesResponse.indexes:type_name -> grpc.CloudEventIndex + 0, // 33: grpc.GetLatestCloudEventRequest.options:type_name -> grpc.SearchOptions + 1, // 34: grpc.GetLatestCloudEventRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 20, // 35: grpc.GetLatestCloudEventResponse.cloud_event:type_name -> cloudevent.CloudEvent + 0, // 36: grpc.ListCloudEventsRequest.options:type_name -> grpc.SearchOptions + 1, // 37: grpc.ListCloudEventsRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 20, // 38: grpc.ListCloudEventsResponse.cloud_events:type_name -> cloudevent.CloudEvent + 4, // 39: grpc.ListCloudEventsFromKeysRequest.indexes:type_name -> grpc.CloudEventIndex + 20, // 40: grpc.ListCloudEventsFromKeysResponse.cloud_events:type_name -> cloudevent.CloudEvent + 6, // 41: grpc.FetchService.GetLatestIndex:input_type -> grpc.GetLatestIndexRequest + 8, // 42: grpc.FetchService.ListIndexes:input_type -> grpc.ListIndexesRequest + 10, // 43: grpc.FetchService.GetLatestCloudEvent:input_type -> grpc.GetLatestCloudEventRequest + 12, // 44: grpc.FetchService.ListCloudEvents:input_type -> grpc.ListCloudEventsRequest + 14, // 45: grpc.FetchService.ListCloudEventsFromIndex:input_type -> grpc.ListCloudEventsFromKeysRequest + 7, // 46: grpc.FetchService.GetLatestIndex:output_type -> grpc.GetLatestIndexResponse + 9, // 47: grpc.FetchService.ListIndexes:output_type -> grpc.ListIndexesResponse + 11, // 48: grpc.FetchService.GetLatestCloudEvent:output_type -> grpc.GetLatestCloudEventResponse + 13, // 49: grpc.FetchService.ListCloudEvents:output_type -> grpc.ListCloudEventsResponse + 15, // 50: grpc.FetchService.ListCloudEventsFromIndex:output_type -> grpc.ListCloudEventsFromKeysResponse + 46, // [46:51] is the sub-list for method output_type + 41, // [41:46] is the sub-list for method input_type + 41, // [41:41] is the sub-list for extension type_name + 41, // [41:41] is the sub-list for extension extendee + 0, // [0:41] is the sub-list for field type_name } func init() { file_pkg_grpc_fetch_api_proto_init() } diff --git a/pkg/grpc/fetch-api.proto b/pkg/grpc/fetch-api.proto index b127bad..f9033c3 100644 --- a/pkg/grpc/fetch-api.proto +++ b/pkg/grpc/fetch-api.proto @@ -102,22 +102,25 @@ message AdvancedSearchOptions { message ArrayFilterOption { // Match if the field has any of these values. repeated string has_any = 1; - + // Match if the field has all of these values. repeated string has_all = 2; - - // Negate all matches in this filter. - bool negate = 3; - + + // Additional filter condition to combine with this one using NOT logic. + ArrayFilterOption not = 3; + // Additional filter condition to combine with this one using OR logic. ArrayFilterOption or = 4; + } message StringFilterOption { // Match if the field has any of these values (OR logic) repeated string has_any = 1; - // Negate all matches in this filter. - bool negate = 2; + + // Additional filter condition to combine with this one using NOT logic. + StringFilterOption not = 2; + // Additional filter condition to combine with this one using OR logic. StringFilterOption or = 3; } From 2cbed1fa1ace7408fa50e51049ea41902f3792d0 Mon Sep 17 00:00:00 2001 From: Kevin Joiner <10265309+KevinJoiner@users.noreply.github.com> Date: Mon, 15 Sep 2025 09:40:10 -0400 Subject: [PATCH 2/3] refactor: remove not op --- pkg/eventrepo/event_repo_test.go | 91 +++++++++++---------- pkg/eventrepo/eventrepo.go | 74 ++++++++--------- pkg/grpc/fetch-api.pb.go | 134 ++++++++++++++++--------------- pkg/grpc/fetch-api.proto | 23 +++--- 4 files changed, 166 insertions(+), 156 deletions(-) diff --git a/pkg/eventrepo/event_repo_test.go b/pkg/eventrepo/event_repo_test.go index 9523970..12e3a06 100644 --- a/pkg/eventrepo/event_repo_test.go +++ b/pkg/eventrepo/event_repo_test.go @@ -577,10 +577,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by single type status events", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Type: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeStatus}, + In: []string{cloudevent.TypeStatus}, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, @@ -589,10 +589,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by multiple types with OR logic", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Type: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeStatus, cloudevent.TypeFingerprint}, + In: []string{cloudevent.TypeStatus, cloudevent.TypeFingerprint}, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeStatusSource1Producer3, keyTypeFingerprintSource2Producer2, keyTypeStatusSource1Producer1}, @@ -601,12 +601,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by type with negation", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Type: &grpc.StringFilterOption{ - Not: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeFingerprint}, - }, + NotIn: []string{cloudevent.TypeFingerprint}, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, @@ -615,10 +613,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by source: multiple results", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Source: &grpc.StringFilterOption{ - HasAny: []string{"source-1"}, + In: []string{"source-1"}, }, }, expectedIndexKeys: []string{keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, @@ -627,13 +625,13 @@ func TestListIndexesAdvanced(t *testing.T) { name: "combine multiple filters (AND logic)", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Type: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeStatus}, + In: []string{cloudevent.TypeStatus}, }, Source: &grpc.StringFilterOption{ - HasAny: []string{"source-1"}, + In: []string{"source-1"}, }, }, expectedIndexKeys: []string{keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, @@ -642,14 +640,12 @@ func TestListIndexesAdvanced(t *testing.T) { name: "OR logic within StringFilterOption", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Type: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeAttestation}, + In: []string{cloudevent.TypeAttestation}, Or: &grpc.StringFilterOption{ - Not: &grpc.StringFilterOption{ - HasAny: []string{cloudevent.TypeStatus}, - }, + NotIn: []string{cloudevent.TypeStatus}, }, }, }, @@ -659,10 +655,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "no matching records", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Type: &grpc.StringFilterOption{ - HasAny: []string{"non-existent-type"}, + In: []string{"non-existent-type"}, }, }, expectedIndexKeys: []string{}, @@ -672,10 +668,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by tags: has any (telemetry)", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAny: []string{"telemetry"}, + ContainsAny: []string{"telemetry"}, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeStatusSource1Producer3}, @@ -684,10 +680,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by tags: has all (vehicle AND status)", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAll: []string{"vehicle", "status"}, + ContainsAll: []string{"vehicle", "status"}, }, }, expectedIndexKeys: []string{keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, @@ -696,10 +692,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by tags: has any with multiple values", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAny: []string{"security", "realtime"}, + ContainsAny: []string{"security", "realtime"}, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeFingerprintSource2Producer2}, @@ -708,12 +704,10 @@ func TestListIndexesAdvanced(t *testing.T) { name: "filter by tags: negated has_any", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - Not: &grpc.ArrayFilterOption{ - HasAny: []string{"vehicle"}, - }, + NotContainsAny: []string{"vehicle"}, }, }, expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeFingerprintSource2Producer2}, @@ -722,12 +716,12 @@ func TestListIndexesAdvanced(t *testing.T) { name: "complex tags filter: has_any OR has_all combination", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAny: []string{"fingerprint"}, + ContainsAny: []string{"fingerprint"}, Or: &grpc.ArrayFilterOption{ - HasAll: []string{"telemetry", "realtime"}, + ContainsAll: []string{"telemetry", "realtime"}, }, }, }, @@ -737,12 +731,12 @@ func TestListIndexesAdvanced(t *testing.T) { name: "complex tags filter: has_all with OR chain", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - HasAll: []string{"vehicle", "telemetry"}, + ContainsAll: []string{"vehicle", "telemetry"}, Or: &grpc.ArrayFilterOption{ - HasAny: []string{"security"}, + ContainsAny: []string{"security"}, }, }, }, @@ -752,24 +746,37 @@ func TestListIndexesAdvanced(t *testing.T) { name: "complex negated tags with multiple conditions", advancedOpts: &grpc.AdvancedSearchOptions{ Subject: &grpc.StringFilterOption{ - HasAny: []string{subject1}, + In: []string{subject1}, }, Tags: &grpc.ArrayFilterOption{ - Not: &grpc.ArrayFilterOption{ - HasAny: []string{"fingerprint", "telemetry"}, - }, + NotContainsAny: []string{"fingerprint", "telemetry"}, Or: &grpc.ArrayFilterOption{ - HasAll: []string{"telemetry", "status"}, + ContainsAll: []string{"telemetry", "status"}, }, }, }, expectedIndexKeys: []string{keyTypeStatusSource1Producer3, keyTypeStatusSource1Producer1}, }, + { + name: "complex negated tags with or override", + advancedOpts: &grpc.AdvancedSearchOptions{ + Subject: &grpc.StringFilterOption{ + In: []string{subject1}, + }, + Tags: &grpc.ArrayFilterOption{ + ContainsAll: []string{"vehicle", "telemetry"}, + NotContainsAny: []string{"fingerprint", "telemetry", "security"}, + Or: &grpc.ArrayFilterOption{ + ContainsAll: []string{"telemetry", "status"}, + }, + }, + }, + expectedIndexKeys: []string{keyTypeStatusSource1Producer3}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Parallel() results, err := indexService.ListIndexesAdvanced(t.Context(), 10, tt.advancedOpts) if tt.expectedError { require.Error(t, err, "Expected error but got none") diff --git a/pkg/eventrepo/eventrepo.go b/pkg/eventrepo/eventrepo.go index 2668059..50e0961 100644 --- a/pkg/eventrepo/eventrepo.go +++ b/pkg/eventrepo/eventrepo.go @@ -322,37 +322,37 @@ func convertSearchOptionsToAdvanced(opts *grpc.SearchOptions) *grpc.AdvancedSear // Convert each field to StringFilterOption with has_any logic if opts.GetType() != nil { advanced.Type = &grpc.StringFilterOption{ - HasAny: []string{opts.GetType().GetValue()}, + In: []string{opts.GetType().GetValue()}, } } if opts.GetDataVersion() != nil { advanced.DataVersion = &grpc.StringFilterOption{ - HasAny: []string{opts.GetDataVersion().GetValue()}, + In: []string{opts.GetDataVersion().GetValue()}, } } if opts.GetSubject() != nil { advanced.Subject = &grpc.StringFilterOption{ - HasAny: []string{opts.GetSubject().GetValue()}, + In: []string{opts.GetSubject().GetValue()}, } } if opts.GetSource() != nil { advanced.Source = &grpc.StringFilterOption{ - HasAny: []string{opts.GetSource().GetValue()}, + In: []string{opts.GetSource().GetValue()}, } } if opts.GetProducer() != nil { advanced.Producer = &grpc.StringFilterOption{ - HasAny: []string{opts.GetProducer().GetValue()}, + In: []string{opts.GetProducer().GetValue()}, } } if opts.GetExtras() != nil { advanced.Extras = &grpc.StringFilterOption{ - HasAny: []string{opts.GetExtras().GetValue()}, + In: []string{opts.GetExtras().GetValue()}, } } if opts.GetId() != nil { advanced.Id = &grpc.StringFilterOption{ - HasAny: []string{opts.GetId().GetValue()}, + In: []string{opts.GetId().GetValue()}, } } @@ -375,92 +375,84 @@ func AdvancedSearchOptionsToQueryMod(opts *grpc.AdvancedSearchOptions) []qm.Quer // Handle advanced filtering for each field if opts.GetType() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetType(), chindexer.TypeColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetType(), chindexer.TypeColumn)...)) } if opts.GetDataVersion() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetDataVersion(), chindexer.DataVersionColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetDataVersion(), chindexer.DataVersionColumn)...)) } if opts.GetSubject() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetSubject(), chindexer.SubjectColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetSubject(), chindexer.SubjectColumn)...)) } if opts.GetSource() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetSource(), chindexer.SourceColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetSource(), chindexer.SourceColumn)...)) } if opts.GetProducer() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetProducer(), chindexer.ProducerColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetProducer(), chindexer.ProducerColumn)...)) } if opts.GetExtras() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetExtras(), chindexer.ExtrasColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetExtras(), chindexer.ExtrasColumn)...)) } if opts.GetId() != nil { - mods = append(mods, qm.Expr(stringFilterMods(opts.GetId(), chindexer.IDColumn, false)...)) + mods = append(mods, qm.Expr(stringFilterMods(opts.GetId(), chindexer.IDColumn)...)) } if opts.GetTags() != nil { - mods = append(mods, qm.Expr(arrayFilterMods(opts.GetTags(), tagsColumn, false)...)) + mods = append(mods, qm.Expr(arrayFilterMods(opts.GetTags(), tagsColumn)...)) } return mods } // stringFilterMods converts a StringFilterOption to query modifications. -func stringFilterMods(filter *grpc.StringFilterOption, columnName string, negate bool) []qm.QueryMod { +func stringFilterMods(filter *grpc.StringFilterOption, columnName string) []qm.QueryMod { var mods []qm.QueryMod if filter == nil { return nil } - var negateClause string - if negate { - negateClause = " NOT" - } // Process has_any (OR logic) - if len(filter.GetHasAny()) > 0 { - mods = append(mods, qm.Where(columnName+negateClause+" IN (?)", filter.GetHasAny())) + if len(filter.GetIn()) > 0 { + mods = append(mods, qm.Where(columnName+" IN (?)", filter.GetIn())) } - if filter.GetNot() != nil { - orFilter := &grpc.StringFilterOption{Or: filter.GetNot()} - notMods := stringFilterMods(orFilter, columnName, !negate) - mods = append(mods, notMods...) + if len(filter.GetNotIn()) > 0 { + mods = append(mods, qm.Where(columnName+" NOT IN (?)", filter.GetNotIn())) } if filter.GetOr() != nil { - orMods := stringFilterMods(filter.GetOr(), columnName, negate) + orMods := stringFilterMods(filter.GetOr(), columnName) mods = append(mods, qm.Or2(qm.Expr(orMods...))) } return mods } // arrayFilterMods converts an ArrayFilterOption to query modifications. -func arrayFilterMods(filter *grpc.ArrayFilterOption, columnName string, negate bool) []qm.QueryMod { +func arrayFilterMods(filter *grpc.ArrayFilterOption, columnName string) []qm.QueryMod { var mods []qm.QueryMod if filter == nil { return mods } - var negateClause string - if negate { - negateClause = "NOT " - } - if len(filter.GetHasAny()) > 0 { - mods = append(mods, qm.Where(negateClause+"hasAny("+columnName+", ?)", filter.GetHasAny())) + if len(filter.GetContainsAny()) > 0 { + mods = append(mods, qm.Where("hasAny("+columnName+", ?)", filter.GetContainsAny())) + } + if len(filter.GetContainsAll()) > 0 { + mods = append(mods, qm.Where("hasAll("+columnName+", ?)", filter.GetContainsAll())) } - if len(filter.GetHasAll()) > 0 { - mods = append(mods, qm.Where(negateClause+"hasAll("+columnName+", ?)", filter.GetHasAll())) + if len(filter.GetNotContainsAny()) > 0 { + mods = append(mods, qm.Where("NOT hasAny("+columnName+", ?)", filter.GetNotContainsAny())) } - if filter.GetNot() != nil { - orFilter := &grpc.ArrayFilterOption{Or: filter.GetNot()} - newMods := arrayFilterMods(orFilter, columnName, !negate) - mods = append(mods, newMods...) + if len(filter.GetNotContainsAll()) > 0 { + mods = append(mods, qm.Where("NOT hasAll("+columnName+", ?)", filter.GetNotContainsAll())) } // Process OR condition recursively if filter.GetOr() != nil { - orMods := arrayFilterMods(filter.GetOr(), columnName, negate) + var orMods []qm.QueryMod + orMods = arrayFilterMods(filter.GetOr(), columnName) mods = append(mods, qm.Or2(qm.Expr(orMods...))) } diff --git a/pkg/grpc/fetch-api.pb.go b/pkg/grpc/fetch-api.pb.go index d7f10ad..d0aecf9 100644 --- a/pkg/grpc/fetch-api.pb.go +++ b/pkg/grpc/fetch-api.pb.go @@ -293,13 +293,15 @@ func (x *AdvancedSearchOptions) GetTags() *ArrayFilterOption { type ArrayFilterOption struct { state protoimpl.MessageState `protogen:"open.v1"` // Match if the field has any of these values. - HasAny []string `protobuf:"bytes,1,rep,name=has_any,json=hasAny,proto3" json:"has_any,omitempty"` + ContainsAny []string `protobuf:"bytes,1,rep,name=contains_any,json=containsAny,proto3" json:"contains_any,omitempty"` // Match if the field has all of these values. - HasAll []string `protobuf:"bytes,2,rep,name=has_all,json=hasAll,proto3" json:"has_all,omitempty"` - // Additional filter condition to combine with this one using NOT logic. - Not *ArrayFilterOption `protobuf:"bytes,3,opt,name=not,proto3" json:"not,omitempty"` + ContainsAll []string `protobuf:"bytes,2,rep,name=contains_all,json=containsAll,proto3" json:"contains_all,omitempty"` + // Match if the field does not have any of these values. + NotContainsAny []string `protobuf:"bytes,3,rep,name=not_contains_any,json=notContainsAny,proto3" json:"not_contains_any,omitempty"` + // Match if the field does not have all of these values. + NotContainsAll []string `protobuf:"bytes,4,rep,name=not_contains_all,json=notContainsAll,proto3" json:"not_contains_all,omitempty"` // Additional filter condition to combine with this one using OR logic. - Or *ArrayFilterOption `protobuf:"bytes,4,opt,name=or,proto3" json:"or,omitempty"` + Or *ArrayFilterOption `protobuf:"bytes,5,opt,name=or,proto3" json:"or,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -334,23 +336,30 @@ func (*ArrayFilterOption) Descriptor() ([]byte, []int) { return file_pkg_grpc_fetch_api_proto_rawDescGZIP(), []int{2} } -func (x *ArrayFilterOption) GetHasAny() []string { +func (x *ArrayFilterOption) GetContainsAny() []string { if x != nil { - return x.HasAny + return x.ContainsAny } return nil } -func (x *ArrayFilterOption) GetHasAll() []string { +func (x *ArrayFilterOption) GetContainsAll() []string { if x != nil { - return x.HasAll + return x.ContainsAll } return nil } -func (x *ArrayFilterOption) GetNot() *ArrayFilterOption { +func (x *ArrayFilterOption) GetNotContainsAny() []string { if x != nil { - return x.Not + return x.NotContainsAny + } + return nil +} + +func (x *ArrayFilterOption) GetNotContainsAll() []string { + if x != nil { + return x.NotContainsAll } return nil } @@ -364,10 +373,10 @@ func (x *ArrayFilterOption) GetOr() *ArrayFilterOption { type StringFilterOption struct { state protoimpl.MessageState `protogen:"open.v1"` - // Match if the field has any of these values (OR logic) - HasAny []string `protobuf:"bytes,1,rep,name=has_any,json=hasAny,proto3" json:"has_any,omitempty"` - // Additional filter condition to combine with this one using NOT logic. - Not *StringFilterOption `protobuf:"bytes,2,opt,name=not,proto3" json:"not,omitempty"` + // Match if the field is in the list of values. + In []string `protobuf:"bytes,1,rep,name=in,proto3" json:"in,omitempty"` + // Match if the field is not in the list of values. + NotIn []string `protobuf:"bytes,2,rep,name=not_in,json=notIn,proto3" json:"not_in,omitempty"` // Additional filter condition to combine with this one using OR logic. Or *StringFilterOption `protobuf:"bytes,3,opt,name=or,proto3" json:"or,omitempty"` unknownFields protoimpl.UnknownFields @@ -404,16 +413,16 @@ func (*StringFilterOption) Descriptor() ([]byte, []int) { return file_pkg_grpc_fetch_api_proto_rawDescGZIP(), []int{3} } -func (x *StringFilterOption) GetHasAny() []string { +func (x *StringFilterOption) GetIn() []string { if x != nil { - return x.HasAny + return x.In } return nil } -func (x *StringFilterOption) GetNot() *StringFilterOption { +func (x *StringFilterOption) GetNotIn() []string { if x != nil { - return x.Not + return x.NotIn } return nil } @@ -1065,15 +1074,16 @@ const file_pkg_grpc_fetch_api_proto_rawDesc = "" + "\x06extras\x18\n" + " \x01(\v2\x18.grpc.StringFilterOptionR\x06extras\x12(\n" + "\x02id\x18\v \x01(\v2\x18.grpc.StringFilterOptionR\x02id\x12+\n" + - "\x04tags\x18\f \x01(\v2\x17.grpc.ArrayFilterOptionR\x04tags\"\x99\x01\n" + - "\x11ArrayFilterOption\x12\x17\n" + - "\ahas_any\x18\x01 \x03(\tR\x06hasAny\x12\x17\n" + - "\ahas_all\x18\x02 \x03(\tR\x06hasAll\x12)\n" + - "\x03not\x18\x03 \x01(\v2\x17.grpc.ArrayFilterOptionR\x03not\x12'\n" + - "\x02or\x18\x04 \x01(\v2\x17.grpc.ArrayFilterOptionR\x02or\"\x83\x01\n" + - "\x12StringFilterOption\x12\x17\n" + - "\ahas_any\x18\x01 \x03(\tR\x06hasAny\x12*\n" + - "\x03not\x18\x02 \x01(\v2\x18.grpc.StringFilterOptionR\x03not\x12(\n" + + "\x04tags\x18\f \x01(\v2\x17.grpc.ArrayFilterOptionR\x04tags\"\xd6\x01\n" + + "\x11ArrayFilterOption\x12!\n" + + "\fcontains_any\x18\x01 \x03(\tR\vcontainsAny\x12!\n" + + "\fcontains_all\x18\x02 \x03(\tR\vcontainsAll\x12(\n" + + "\x10not_contains_any\x18\x03 \x03(\tR\x0enotContainsAny\x12(\n" + + "\x10not_contains_all\x18\x04 \x03(\tR\x0enotContainsAll\x12'\n" + + "\x02or\x18\x05 \x01(\v2\x17.grpc.ArrayFilterOptionR\x02or\"e\n" + + "\x12StringFilterOption\x12\x0e\n" + + "\x02in\x18\x01 \x03(\tR\x02in\x12\x15\n" + + "\x06not_in\x18\x02 \x03(\tR\x05notIn\x12(\n" + "\x02or\x18\x03 \x01(\v2\x18.grpc.StringFilterOptionR\x02or\"m\n" + "\x0fCloudEventIndex\x124\n" + "\x06header\x18\x01 \x01(\v2\x1c.cloudevent.CloudEventHeaderR\x06header\x12$\n" + @@ -1173,41 +1183,39 @@ var file_pkg_grpc_fetch_api_proto_depIdxs = []int32{ 3, // 18: grpc.AdvancedSearchOptions.extras:type_name -> grpc.StringFilterOption 3, // 19: grpc.AdvancedSearchOptions.id:type_name -> grpc.StringFilterOption 2, // 20: grpc.AdvancedSearchOptions.tags:type_name -> grpc.ArrayFilterOption - 2, // 21: grpc.ArrayFilterOption.not:type_name -> grpc.ArrayFilterOption - 2, // 22: grpc.ArrayFilterOption.or:type_name -> grpc.ArrayFilterOption - 3, // 23: grpc.StringFilterOption.not:type_name -> grpc.StringFilterOption - 3, // 24: grpc.StringFilterOption.or:type_name -> grpc.StringFilterOption - 19, // 25: grpc.CloudEventIndex.header:type_name -> cloudevent.CloudEventHeader - 5, // 26: grpc.CloudEventIndex.data:type_name -> grpc.ObjectInfo - 0, // 27: grpc.GetLatestIndexRequest.options:type_name -> grpc.SearchOptions - 1, // 28: grpc.GetLatestIndexRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 4, // 29: grpc.GetLatestIndexResponse.index:type_name -> grpc.CloudEventIndex - 0, // 30: grpc.ListIndexesRequest.options:type_name -> grpc.SearchOptions - 1, // 31: grpc.ListIndexesRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 4, // 32: grpc.ListIndexesResponse.indexes:type_name -> grpc.CloudEventIndex - 0, // 33: grpc.GetLatestCloudEventRequest.options:type_name -> grpc.SearchOptions - 1, // 34: grpc.GetLatestCloudEventRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 20, // 35: grpc.GetLatestCloudEventResponse.cloud_event:type_name -> cloudevent.CloudEvent - 0, // 36: grpc.ListCloudEventsRequest.options:type_name -> grpc.SearchOptions - 1, // 37: grpc.ListCloudEventsRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions - 20, // 38: grpc.ListCloudEventsResponse.cloud_events:type_name -> cloudevent.CloudEvent - 4, // 39: grpc.ListCloudEventsFromKeysRequest.indexes:type_name -> grpc.CloudEventIndex - 20, // 40: grpc.ListCloudEventsFromKeysResponse.cloud_events:type_name -> cloudevent.CloudEvent - 6, // 41: grpc.FetchService.GetLatestIndex:input_type -> grpc.GetLatestIndexRequest - 8, // 42: grpc.FetchService.ListIndexes:input_type -> grpc.ListIndexesRequest - 10, // 43: grpc.FetchService.GetLatestCloudEvent:input_type -> grpc.GetLatestCloudEventRequest - 12, // 44: grpc.FetchService.ListCloudEvents:input_type -> grpc.ListCloudEventsRequest - 14, // 45: grpc.FetchService.ListCloudEventsFromIndex:input_type -> grpc.ListCloudEventsFromKeysRequest - 7, // 46: grpc.FetchService.GetLatestIndex:output_type -> grpc.GetLatestIndexResponse - 9, // 47: grpc.FetchService.ListIndexes:output_type -> grpc.ListIndexesResponse - 11, // 48: grpc.FetchService.GetLatestCloudEvent:output_type -> grpc.GetLatestCloudEventResponse - 13, // 49: grpc.FetchService.ListCloudEvents:output_type -> grpc.ListCloudEventsResponse - 15, // 50: grpc.FetchService.ListCloudEventsFromIndex:output_type -> grpc.ListCloudEventsFromKeysResponse - 46, // [46:51] is the sub-list for method output_type - 41, // [41:46] is the sub-list for method input_type - 41, // [41:41] is the sub-list for extension type_name - 41, // [41:41] is the sub-list for extension extendee - 0, // [0:41] is the sub-list for field type_name + 2, // 21: grpc.ArrayFilterOption.or:type_name -> grpc.ArrayFilterOption + 3, // 22: grpc.StringFilterOption.or:type_name -> grpc.StringFilterOption + 19, // 23: grpc.CloudEventIndex.header:type_name -> cloudevent.CloudEventHeader + 5, // 24: grpc.CloudEventIndex.data:type_name -> grpc.ObjectInfo + 0, // 25: grpc.GetLatestIndexRequest.options:type_name -> grpc.SearchOptions + 1, // 26: grpc.GetLatestIndexRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 4, // 27: grpc.GetLatestIndexResponse.index:type_name -> grpc.CloudEventIndex + 0, // 28: grpc.ListIndexesRequest.options:type_name -> grpc.SearchOptions + 1, // 29: grpc.ListIndexesRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 4, // 30: grpc.ListIndexesResponse.indexes:type_name -> grpc.CloudEventIndex + 0, // 31: grpc.GetLatestCloudEventRequest.options:type_name -> grpc.SearchOptions + 1, // 32: grpc.GetLatestCloudEventRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 20, // 33: grpc.GetLatestCloudEventResponse.cloud_event:type_name -> cloudevent.CloudEvent + 0, // 34: grpc.ListCloudEventsRequest.options:type_name -> grpc.SearchOptions + 1, // 35: grpc.ListCloudEventsRequest.advanced_options:type_name -> grpc.AdvancedSearchOptions + 20, // 36: grpc.ListCloudEventsResponse.cloud_events:type_name -> cloudevent.CloudEvent + 4, // 37: grpc.ListCloudEventsFromKeysRequest.indexes:type_name -> grpc.CloudEventIndex + 20, // 38: grpc.ListCloudEventsFromKeysResponse.cloud_events:type_name -> cloudevent.CloudEvent + 6, // 39: grpc.FetchService.GetLatestIndex:input_type -> grpc.GetLatestIndexRequest + 8, // 40: grpc.FetchService.ListIndexes:input_type -> grpc.ListIndexesRequest + 10, // 41: grpc.FetchService.GetLatestCloudEvent:input_type -> grpc.GetLatestCloudEventRequest + 12, // 42: grpc.FetchService.ListCloudEvents:input_type -> grpc.ListCloudEventsRequest + 14, // 43: grpc.FetchService.ListCloudEventsFromIndex:input_type -> grpc.ListCloudEventsFromKeysRequest + 7, // 44: grpc.FetchService.GetLatestIndex:output_type -> grpc.GetLatestIndexResponse + 9, // 45: grpc.FetchService.ListIndexes:output_type -> grpc.ListIndexesResponse + 11, // 46: grpc.FetchService.GetLatestCloudEvent:output_type -> grpc.GetLatestCloudEventResponse + 13, // 47: grpc.FetchService.ListCloudEvents:output_type -> grpc.ListCloudEventsResponse + 15, // 48: grpc.FetchService.ListCloudEventsFromIndex:output_type -> grpc.ListCloudEventsFromKeysResponse + 44, // [44:49] is the sub-list for method output_type + 39, // [39:44] is the sub-list for method input_type + 39, // [39:39] is the sub-list for extension type_name + 39, // [39:39] is the sub-list for extension extendee + 0, // [0:39] is the sub-list for field type_name } func init() { file_pkg_grpc_fetch_api_proto_init() } diff --git a/pkg/grpc/fetch-api.proto b/pkg/grpc/fetch-api.proto index f9033c3..e659c7c 100644 --- a/pkg/grpc/fetch-api.proto +++ b/pkg/grpc/fetch-api.proto @@ -101,26 +101,29 @@ message AdvancedSearchOptions { // every thing is implicitly ANDed together. message ArrayFilterOption { // Match if the field has any of these values. - repeated string has_any = 1; + repeated string contains_any = 1; // Match if the field has all of these values. - repeated string has_all = 2; + repeated string contains_all = 2; - // Additional filter condition to combine with this one using NOT logic. - ArrayFilterOption not = 3; + // Match if the field does not have any of these values. + repeated string not_contains_any = 3; + + // Match if the field does not have all of these values. + repeated string not_contains_all = 4; // Additional filter condition to combine with this one using OR logic. - ArrayFilterOption or = 4; + ArrayFilterOption or = 5; } message StringFilterOption { - // Match if the field has any of these values (OR logic) - repeated string has_any = 1; + // Match if the field is in the list of values. + repeated string in = 1; + + // Match if the field is not in the list of values. + repeated string not_in = 2; - // Additional filter condition to combine with this one using NOT logic. - StringFilterOption not = 2; - // Additional filter condition to combine with this one using OR logic. StringFilterOption or = 3; } From 579e3697b99b4741e7d0d5e604c0a8930e837281 Mon Sep 17 00:00:00 2001 From: Kevin Joiner <10265309+KevinJoiner@users.noreply.github.com> Date: Mon, 15 Sep 2025 10:35:52 -0400 Subject: [PATCH 3/3] refactor: make or a list --- pkg/eventrepo/event_repo_test.go | 26 ++++++++++++++------------ pkg/eventrepo/eventrepo.go | 24 +++++++++++++++++------- pkg/grpc/fetch-api.pb.go | 12 ++++++------ pkg/grpc/fetch-api.proto | 4 ++-- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/pkg/eventrepo/event_repo_test.go b/pkg/eventrepo/event_repo_test.go index 12e3a06..18f16fa 100644 --- a/pkg/eventrepo/event_repo_test.go +++ b/pkg/eventrepo/event_repo_test.go @@ -547,7 +547,6 @@ func TestListIndexesAdvanced(t *testing.T) { Tags: []string{"vehicle", "telemetry", "status"}, } - // Add an event with different tags for testing ArrayFilterOption eventIdx4 := &cloudevent.CloudEventHeader{ ID: "event-source3-producer4", Subject: subject1, @@ -644,8 +643,8 @@ func TestListIndexesAdvanced(t *testing.T) { }, Type: &grpc.StringFilterOption{ In: []string{cloudevent.TypeAttestation}, - Or: &grpc.StringFilterOption{ - NotIn: []string{cloudevent.TypeStatus}, + Or: []*grpc.StringFilterOption{ + {NotIn: []string{cloudevent.TypeStatus}}, }, }, }, @@ -696,9 +695,12 @@ func TestListIndexesAdvanced(t *testing.T) { }, Tags: &grpc.ArrayFilterOption{ ContainsAny: []string{"security", "realtime"}, + Or: []*grpc.ArrayFilterOption{ + {ContainsAll: []string{"vehicle", "status"}}, + }, }, }, - expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeFingerprintSource2Producer2}, + expectedIndexKeys: []string{keyTypeStatusSource3Producer4, keyTypeStatusSource1Producer3, keyTypeFingerprintSource2Producer2, keyTypeStatusSource1Producer1}, }, { name: "filter by tags: negated has_any", @@ -720,8 +722,8 @@ func TestListIndexesAdvanced(t *testing.T) { }, Tags: &grpc.ArrayFilterOption{ ContainsAny: []string{"fingerprint"}, - Or: &grpc.ArrayFilterOption{ - ContainsAll: []string{"telemetry", "realtime"}, + Or: []*grpc.ArrayFilterOption{ + {ContainsAll: []string{"telemetry", "realtime"}}, }, }, }, @@ -735,8 +737,8 @@ func TestListIndexesAdvanced(t *testing.T) { }, Tags: &grpc.ArrayFilterOption{ ContainsAll: []string{"vehicle", "telemetry"}, - Or: &grpc.ArrayFilterOption{ - ContainsAny: []string{"security"}, + Or: []*grpc.ArrayFilterOption{ + {ContainsAny: []string{"security"}}, }, }, }, @@ -750,8 +752,8 @@ func TestListIndexesAdvanced(t *testing.T) { }, Tags: &grpc.ArrayFilterOption{ NotContainsAny: []string{"fingerprint", "telemetry"}, - Or: &grpc.ArrayFilterOption{ - ContainsAll: []string{"telemetry", "status"}, + Or: []*grpc.ArrayFilterOption{ + {ContainsAll: []string{"telemetry", "status"}}, }, }, }, @@ -766,8 +768,8 @@ func TestListIndexesAdvanced(t *testing.T) { Tags: &grpc.ArrayFilterOption{ ContainsAll: []string{"vehicle", "telemetry"}, NotContainsAny: []string{"fingerprint", "telemetry", "security"}, - Or: &grpc.ArrayFilterOption{ - ContainsAll: []string{"telemetry", "status"}, + Or: []*grpc.ArrayFilterOption{ + {ContainsAll: []string{"telemetry", "status"}}, }, }, }, diff --git a/pkg/eventrepo/eventrepo.go b/pkg/eventrepo/eventrepo.go index 50e0961..3372dca 100644 --- a/pkg/eventrepo/eventrepo.go +++ b/pkg/eventrepo/eventrepo.go @@ -423,9 +423,15 @@ func stringFilterMods(filter *grpc.StringFilterOption, columnName string) []qm.Q if len(filter.GetNotIn()) > 0 { mods = append(mods, qm.Where(columnName+" NOT IN (?)", filter.GetNotIn())) } - if filter.GetOr() != nil { - orMods := stringFilterMods(filter.GetOr(), columnName) - mods = append(mods, qm.Or2(qm.Expr(orMods...))) + for _, cond := range filter.GetOr() { + clauseMods := stringFilterMods(cond, columnName) + if len(clauseMods) != 0 { + mods = append(mods, qm.Or2(qm.Expr(clauseMods...))) + } + } + + if len(filter.GetOr()) != 0 { + mods = []qm.QueryMod{qm.Expr(mods...)} } return mods } @@ -450,10 +456,14 @@ func arrayFilterMods(filter *grpc.ArrayFilterOption, columnName string) []qm.Que mods = append(mods, qm.Where("NOT hasAll("+columnName+", ?)", filter.GetNotContainsAll())) } // Process OR condition recursively - if filter.GetOr() != nil { - var orMods []qm.QueryMod - orMods = arrayFilterMods(filter.GetOr(), columnName) - mods = append(mods, qm.Or2(qm.Expr(orMods...))) + for _, cond := range filter.GetOr() { + clauseMods := arrayFilterMods(cond, columnName) + if len(clauseMods) != 0 { + mods = append(mods, qm.Or2(qm.Expr(clauseMods...))) + } + } + if len(filter.GetOr()) != 0 { + mods = []qm.QueryMod{qm.Expr(mods...)} } return mods diff --git a/pkg/grpc/fetch-api.pb.go b/pkg/grpc/fetch-api.pb.go index d0aecf9..e4541ce 100644 --- a/pkg/grpc/fetch-api.pb.go +++ b/pkg/grpc/fetch-api.pb.go @@ -301,7 +301,7 @@ type ArrayFilterOption struct { // Match if the field does not have all of these values. NotContainsAll []string `protobuf:"bytes,4,rep,name=not_contains_all,json=notContainsAll,proto3" json:"not_contains_all,omitempty"` // Additional filter condition to combine with this one using OR logic. - Or *ArrayFilterOption `protobuf:"bytes,5,opt,name=or,proto3" json:"or,omitempty"` + Or []*ArrayFilterOption `protobuf:"bytes,5,rep,name=or,proto3" json:"or,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -364,7 +364,7 @@ func (x *ArrayFilterOption) GetNotContainsAll() []string { return nil } -func (x *ArrayFilterOption) GetOr() *ArrayFilterOption { +func (x *ArrayFilterOption) GetOr() []*ArrayFilterOption { if x != nil { return x.Or } @@ -378,7 +378,7 @@ type StringFilterOption struct { // Match if the field is not in the list of values. NotIn []string `protobuf:"bytes,2,rep,name=not_in,json=notIn,proto3" json:"not_in,omitempty"` // Additional filter condition to combine with this one using OR logic. - Or *StringFilterOption `protobuf:"bytes,3,opt,name=or,proto3" json:"or,omitempty"` + Or []*StringFilterOption `protobuf:"bytes,3,rep,name=or,proto3" json:"or,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -427,7 +427,7 @@ func (x *StringFilterOption) GetNotIn() []string { return nil } -func (x *StringFilterOption) GetOr() *StringFilterOption { +func (x *StringFilterOption) GetOr() []*StringFilterOption { if x != nil { return x.Or } @@ -1080,11 +1080,11 @@ const file_pkg_grpc_fetch_api_proto_rawDesc = "" + "\fcontains_all\x18\x02 \x03(\tR\vcontainsAll\x12(\n" + "\x10not_contains_any\x18\x03 \x03(\tR\x0enotContainsAny\x12(\n" + "\x10not_contains_all\x18\x04 \x03(\tR\x0enotContainsAll\x12'\n" + - "\x02or\x18\x05 \x01(\v2\x17.grpc.ArrayFilterOptionR\x02or\"e\n" + + "\x02or\x18\x05 \x03(\v2\x17.grpc.ArrayFilterOptionR\x02or\"e\n" + "\x12StringFilterOption\x12\x0e\n" + "\x02in\x18\x01 \x03(\tR\x02in\x12\x15\n" + "\x06not_in\x18\x02 \x03(\tR\x05notIn\x12(\n" + - "\x02or\x18\x03 \x01(\v2\x18.grpc.StringFilterOptionR\x02or\"m\n" + + "\x02or\x18\x03 \x03(\v2\x18.grpc.StringFilterOptionR\x02or\"m\n" + "\x0fCloudEventIndex\x124\n" + "\x06header\x18\x01 \x01(\v2\x1c.cloudevent.CloudEventHeaderR\x06header\x12$\n" + "\x04data\x18\x02 \x01(\v2\x10.grpc.ObjectInfoR\x04data\"\x1e\n" + diff --git a/pkg/grpc/fetch-api.proto b/pkg/grpc/fetch-api.proto index e659c7c..72842c7 100644 --- a/pkg/grpc/fetch-api.proto +++ b/pkg/grpc/fetch-api.proto @@ -113,7 +113,7 @@ message ArrayFilterOption { repeated string not_contains_all = 4; // Additional filter condition to combine with this one using OR logic. - ArrayFilterOption or = 5; + repeated ArrayFilterOption or = 5; } @@ -125,7 +125,7 @@ message StringFilterOption { repeated string not_in = 2; // Additional filter condition to combine with this one using OR logic. - StringFilterOption or = 3; + repeated StringFilterOption or = 3; } message CloudEventIndex {