From 0ea89f0cb7ffee7a730ea9913619873904e94c76 Mon Sep 17 00:00:00 2001 From: srinivasr Date: Mon, 24 Aug 2026 17:50:10 +0530 Subject: [PATCH] Page datastore queries that fetched all applications ListApplications for piped, ListApplications for web, and the application lookup in CreateDeploymentChain ran one query with no limit and returned every matching row. A project with thousands of applications pays for all of them on every sync even though the RPC responses have no cursor field and cannot page. These now fetch 100 rows at a time and combine the pages before responding. Paging by cursor needs a fixed sort order, so the queries sort by Id. ListEvents also pages now. It defaults to newest first when a request leaves the order unset, which previously errored on the second page. (Also repairs the ListEvents pagination test to use the correct mock). Added the composite indexes these query shapes need to firestoreindexensurer; Firestore does not create them automatically. Fixes #7051 Signed-off-by: srinivasr --- .../ops/firestoreindexensurer/indexes.json | 178 +++++++++++++++++ .../ops/firestoreindexensurer/indexes_test.go | 180 +++++++++++++++++- pkg/app/server/grpcapi/application_lister.go | 52 +++++ pkg/app/server/grpcapi/event_lister.go | 52 +++++ .../server/grpcapi/list_applications_test.go | 135 +++++++++++++ pkg/app/server/grpcapi/list_events_test.go | 100 ++++++++++ pkg/app/server/grpcapi/piped_api.go | 45 ++++- pkg/app/server/grpcapi/web_api.go | 3 +- 8 files changed, 737 insertions(+), 8 deletions(-) create mode 100644 pkg/app/server/grpcapi/application_lister.go create mode 100644 pkg/app/server/grpcapi/event_lister.go create mode 100644 pkg/app/server/grpcapi/list_applications_test.go create mode 100644 pkg/app/server/grpcapi/list_events_test.go diff --git a/pkg/app/ops/firestoreindexensurer/indexes.json b/pkg/app/ops/firestoreindexensurer/indexes.json index e1c0a5f325..dff0b19e6e 100644 --- a/pkg/app/ops/firestoreindexensurer/indexes.json +++ b/pkg/app/ops/firestoreindexensurer/indexes.json @@ -1,4 +1,114 @@ [ + { + "collectionGroup": "Application", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "PipedId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Disabled", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, + { + "collectionGroup": "Application", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, + { + "collectionGroup": "Application", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Name", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, + { + "collectionGroup": "Application", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Kind", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, + { + "collectionGroup": "Application", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Name", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Kind", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, { "collectionGroup": "Application", "queryScope": "COLLECTION", @@ -141,6 +251,27 @@ } ] }, + { + "collectionGroup": "Application", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Disabled", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, { "collectionGroup": "Command", "queryScope": "COLLECTION", @@ -550,6 +681,53 @@ } ] }, + { + "collectionGroup": "Event", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "CreatedAt", + "order": "DESCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, + { + "collectionGroup": "Event", + "queryScope": "COLLECTION", + "fields": [ + { + "fieldPath": "ProjectId", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Status", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "CreatedAt", + "order": "ASCENDING", + "arrayConfig": "" + }, + { + "fieldPath": "Id", + "order": "ASCENDING", + "arrayConfig": "" + } + ] + }, { "collectionGroup": "DeploymentChain", "queryScope": "COLLECTION", diff --git a/pkg/app/ops/firestoreindexensurer/indexes_test.go b/pkg/app/ops/firestoreindexensurer/indexes_test.go index 35892ee4a8..fb6fd6a3c9 100644 --- a/pkg/app/ops/firestoreindexensurer/indexes_test.go +++ b/pkg/app/ops/firestoreindexensurer/indexes_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The PipeCD Authors. +// Copyright 2026 The PipeCD Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,6 +23,116 @@ import ( func TestParseIndexes(t *testing.T) { want := []index{ + { + CollectionGroup: "Application", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "PipedId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Disabled", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, + { + CollectionGroup: "Application", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, + { + CollectionGroup: "Application", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Name", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, + { + CollectionGroup: "Application", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Kind", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, + { + CollectionGroup: "Application", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Name", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Kind", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, { CollectionGroup: "Application", QueryScope: "COLLECTION", @@ -165,6 +275,27 @@ func TestParseIndexes(t *testing.T) { }, }, }, + { + CollectionGroup: "Application", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Disabled", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, { CollectionGroup: "Command", QueryScope: "COLLECTION", @@ -574,6 +705,53 @@ func TestParseIndexes(t *testing.T) { }, }, }, + { + CollectionGroup: "Event", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "CreatedAt", + Order: "DESCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, + { + CollectionGroup: "Event", + QueryScope: "COLLECTION", + Fields: []field{ + { + FieldPath: "ProjectId", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Status", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "CreatedAt", + Order: "ASCENDING", + ArrayConfig: "", + }, + { + FieldPath: "Id", + Order: "ASCENDING", + ArrayConfig: "", + }, + }, + }, { CollectionGroup: "DeploymentChain", QueryScope: "COLLECTION", diff --git a/pkg/app/server/grpcapi/application_lister.go b/pkg/app/server/grpcapi/application_lister.go new file mode 100644 index 0000000000..0422d2b78e --- /dev/null +++ b/pkg/app/server/grpcapi/application_lister.go @@ -0,0 +1,52 @@ +// Copyright 2026 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcapi + +import ( + "context" + + "github.com/pipe-cd/pipecd/pkg/datastore" + "github.com/pipe-cd/pipecd/pkg/model" +) + +// listApplicationsPageSize is the number of applications fetched per datastore +// query while aggregating the full application list. +const listApplicationsPageSize = 100 + +// applicationLister is the subset of the application stores needed to page +// through applications. +type applicationLister interface { + List(ctx context.Context, opts datastore.ListOptions) ([]*model.Application, string, error) +} + +// listAllApplications pages through the datastore so that fetching every +// application matching opts cannot be done by one unbounded query. Cursor +// paging requires a stable order, so callers must set Orders on opts; the +// returned slice aggregates all pages because neither RPC response carries a +// cursor field. +func listAllApplications(ctx context.Context, store applicationLister, opts datastore.ListOptions) ([]*model.Application, error) { + apps := make([]*model.Application, 0, listApplicationsPageSize) + for { + page, cursor, err := store.List(ctx, opts) + if err != nil { + return nil, err + } + apps = append(apps, page...) + if cursor == "" { + return apps, nil + } + opts.Cursor = cursor + } +} diff --git a/pkg/app/server/grpcapi/event_lister.go b/pkg/app/server/grpcapi/event_lister.go new file mode 100644 index 0000000000..03c761659b --- /dev/null +++ b/pkg/app/server/grpcapi/event_lister.go @@ -0,0 +1,52 @@ +// Copyright 2026 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcapi + +import ( + "context" + + "github.com/pipe-cd/pipecd/pkg/datastore" + "github.com/pipe-cd/pipecd/pkg/model" +) + +// listEventsPageSize is the number of events fetched per datastore query +// while aggregating the full list. +const listEventsPageSize = 100 + +// eventLister is the subset of the event stores needed to page +// through events. +type eventLister interface { + List(ctx context.Context, opts datastore.ListOptions) ([]*model.Event, string, error) +} + +// listAllEvents pages through the datastore so that fetching every +// event matching opts cannot be done by one unbounded query. Cursor +// paging requires a stable order, so callers must set Orders on opts; the +// returned slice aggregates all pages because neither RPC response carries a +// cursor field. +func listAllEvents(ctx context.Context, store eventLister, opts datastore.ListOptions) ([]*model.Event, error) { + events := make([]*model.Event, 0, listEventsPageSize) + for { + page, cursor, err := store.List(ctx, opts) + if err != nil { + return nil, err + } + events = append(events, page...) + if cursor == "" { + return events, nil + } + opts.Cursor = cursor + } +} diff --git a/pkg/app/server/grpcapi/list_applications_test.go b/pkg/app/server/grpcapi/list_applications_test.go new file mode 100644 index 0000000000..3f3a15823c --- /dev/null +++ b/pkg/app/server/grpcapi/list_applications_test.go @@ -0,0 +1,135 @@ +// Copyright 2026 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcapi + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice" + "github.com/pipe-cd/pipecd/pkg/datastore" + "github.com/pipe-cd/pipecd/pkg/datastore/datastoretest" + "github.com/pipe-cd/pipecd/pkg/model" + "github.com/pipe-cd/pipecd/pkg/rpc/rpcauth" +) + +// fakePipedTokenVerifier accepts every piped token. +type fakePipedTokenVerifier struct{} + +func (fakePipedTokenVerifier) Verify(_ context.Context, _, _, _ string) error { return nil } + +// pipedAuthContext returns a context carrying a valid piped token, built by +// running the real PipedTokenUnaryServerInterceptor over a stub handler so the +// test exercises only exported rpcauth APIs. +func pipedAuthContext(t *testing.T) context.Context { + t.Helper() + + token := rpcauth.MakePipedToken("projectID", "pipedID", "pipedKey") + md := metadata.Pairs("authorization", fmt.Sprintf("%s %s", rpcauth.PipedTokenCredentials, token)) + ctx := metadata.NewIncomingContext(context.Background(), md) + + verifier := fakePipedTokenVerifier{} + var authed context.Context + interceptor := rpcauth.PipedTokenUnaryServerInterceptor(verifier, nil) + _, err := interceptor(ctx, nil, &grpc.UnaryServerInfo{FullMethod: "TestListApplications"}, func(c context.Context, _ interface{}) (interface{}, error) { + authed = c + return nil, nil + }) + if err != nil { + t.Fatalf("failed to build authorized context: %v", err) + } + return authed +} + +func TestListApplicationsPagination(t *testing.T) { + ctx := pipedAuthContext(t) + + t.Run("single page", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + apps := []*model.Application{ + {Id: "app-1", PipedId: "pipedID"}, + {Id: "app-2", PipedId: "pipedID"}, + } + s := datastoretest.NewMockApplicationStore(ctrl) + s.EXPECT(). + List(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, opts datastore.ListOptions) ([]*model.Application, string, error) { + assert.Equal(t, listApplicationsPageSize, opts.Limit) + assert.NotEmpty(t, opts.Orders, "cursor paging requires a stable order") + return apps, "", nil + }) + + api := &PipedAPI{applicationStore: s} + resp, err := api.ListApplications(ctx, &pipedservice.ListApplicationsRequest{}) + assert.NoError(t, err) + assert.Equal(t, apps, resp.Applications) + }) + + t.Run("multiple pages are aggregated", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + page1 := []*model.Application{{Id: "app-1", PipedId: "pipedID"}} + page2 := []*model.Application{{Id: "app-2", PipedId: "pipedID"}} + + s := datastoretest.NewMockApplicationStore(ctrl) + firstCall := s.EXPECT(). + List(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, opts datastore.ListOptions) ([]*model.Application, string, error) { + assert.Empty(t, opts.Cursor, "first call must not carry a cursor") + return page1, "cursor-1", nil + }) + secondCall := s.EXPECT(). + List(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, opts datastore.ListOptions) ([]*model.Application, string, error) { + assert.Equal(t, "cursor-1", opts.Cursor, "second call must continue from the first cursor") + return page2, "", nil + }) + gomock.InOrder(firstCall, secondCall) + + api := &PipedAPI{applicationStore: s} + resp, err := api.ListApplications(ctx, &pipedservice.ListApplicationsRequest{}) + assert.NoError(t, err) + want := make([]*model.Application, 0, len(page1)+len(page2)) + want = append(want, page1...) + want = append(want, page2...) + assert.Equal(t, want, resp.Applications) + }) + + t.Run("store error is returned", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + s := datastoretest.NewMockApplicationStore(ctrl) + s.EXPECT(). + List(gomock.Any(), gomock.Any()). + Return(nil, "", assert.AnError) + + api := &PipedAPI{applicationStore: s} + resp, err := api.ListApplications(ctx, &pipedservice.ListApplicationsRequest{}) + assert.Nil(t, resp) + assert.Equal(t, codes.Internal, status.Code(err)) + }) +} diff --git a/pkg/app/server/grpcapi/list_events_test.go b/pkg/app/server/grpcapi/list_events_test.go new file mode 100644 index 0000000000..d836dd0513 --- /dev/null +++ b/pkg/app/server/grpcapi/list_events_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcapi + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + service "github.com/pipe-cd/pipecd/pkg/app/server/service/pipedservice" + "github.com/pipe-cd/pipecd/pkg/datastore" + "github.com/pipe-cd/pipecd/pkg/model" +) + +type fakeEventStore struct { + pipedAPIEventStore + pages [][]*model.Event + gotOpts []datastore.ListOptions +} + +func (f *fakeEventStore) List(ctx context.Context, opts datastore.ListOptions) ([]*model.Event, string, error) { + f.gotOpts = append(f.gotOpts, opts) + if len(f.pages) == 0 { + return nil, "", nil + } + page := f.pages[0] + f.pages = f.pages[1:] + cursor := "" + if len(f.pages) > 0 { + cursor = "next" + } + return page, cursor, nil +} + +func TestListEventsPagination(t *testing.T) { + ctx := pipedAuthContext(t) + + t.Run("defaults to newest first when order is unset", func(t *testing.T) { + store := &fakeEventStore{ + pages: [][]*model.Event{ + {{Id: "event-1"}}, + }, + } + + api := &PipedAPI{eventStore: store} + resp, err := api.ListEvents(ctx, &service.ListEventsRequest{}) + assert.NoError(t, err) + assert.Len(t, resp.Events, 1) + + opts := store.gotOpts[0] + assert.NotEmpty(t, opts.Orders, "paging requires a stable order") + assert.Equal(t, datastore.Desc, opts.Orders[0].Direction) + assert.Equal(t, listEventsPageSize, opts.Limit) + }) + + t.Run("unknown order value is rejected", func(t *testing.T) { + store := &fakeEventStore{} + + api := &PipedAPI{eventStore: store} + resp, err := api.ListEvents(ctx, &service.ListEventsRequest{Order: service.ListOrder(99)}) + assert.Nil(t, resp) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + }) + + t.Run("multiple pages are aggregated", func(t *testing.T) { + store := &fakeEventStore{ + pages: [][]*model.Event{ + {{Id: "event-1"}}, + {{Id: "event-2"}}, + }, + } + + api := &PipedAPI{eventStore: store} + resp, err := api.ListEvents(ctx, &service.ListEventsRequest{Order: service.ListOrder_ASC}) + assert.NoError(t, err) + assert.Len(t, resp.Events, 2) + assert.Equal(t, "event-1", resp.Events[0].Id) + assert.Equal(t, "event-2", resp.Events[1].Id) + + // First call must not carry a cursor; the second must continue from + // the cursor returned by the first. + assert.Empty(t, store.gotOpts[0].Cursor) + assert.NotEmpty(t, store.gotOpts[1].Cursor) + }) +} diff --git a/pkg/app/server/grpcapi/piped_api.go b/pkg/app/server/grpcapi/piped_api.go index 121268e848..bc593ea84d 100644 --- a/pkg/app/server/grpcapi/piped_api.go +++ b/pkg/app/server/grpcapi/piped_api.go @@ -220,6 +220,12 @@ func (a *PipedAPI) ListApplications(ctx context.Context, req *pipedservice.ListA return nil, err } opts := datastore.ListOptions{ + Orders: []datastore.Order{ + { + Field: "Id", + Direction: datastore.Asc, + }, + }, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -237,9 +243,12 @@ func (a *PipedAPI) ListApplications(ctx context.Context, req *pipedservice.ListA Value: false, }, }, + Limit: listApplicationsPageSize, } - // TODO: Support pagination in ListApplications - apps, _, err := a.applicationStore.List(ctx, opts) + // Page through the datastore so a large project cannot be served by one + // unbounded query. The RPC response has no cursor field, so all pages are + // aggregated here before returning. + apps, err := listAllApplications(ctx, a.applicationStore, opts) if err != nil { return nil, gRPCStoreError(err, "fetch applications") } @@ -720,7 +729,6 @@ func (a *PipedAPI) GetLatestEvent(ctx context.Context, req *pipedservice.GetLate // Try to fetch the most recently registered event that has the given parameters. opts := datastore.ListOptions{ - Limit: 1, Filters: []datastore.ListFilter{ { Field: "ProjectId", @@ -748,6 +756,7 @@ func (a *PipedAPI) GetLatestEvent(ctx context.Context, req *pipedservice.GetLate Direction: datastore.Asc, }, }, + Limit: 1, } events, _, err := a.eventStore.List(ctx, opts) if err != nil { @@ -815,6 +824,19 @@ func (a *PipedAPI) ListEvents(ctx context.Context, req *pipedservice.ListEventsR }) } switch req.Order { + case pipedservice.ListOrder_NONE: + // Cursor paging requires a stable order; default to newest first + // when the request does not specify one. + opts.Orders = []datastore.Order{ + { + Field: "CreatedAt", + Direction: datastore.Desc, + }, + { + Field: "Id", + Direction: datastore.Asc, + }, + } case pipedservice.ListOrder_ASC: opts.Orders = []datastore.Order{ { @@ -837,9 +859,12 @@ func (a *PipedAPI) ListEvents(ctx context.Context, req *pipedservice.ListEventsR Direction: datastore.Asc, }, } + default: + return nil, status.Errorf(codes.InvalidArgument, "unknown order %v given", req.Order) } + opts.Limit = listEventsPageSize - events, _, err := a.eventStore.List(ctx, opts) + events, err := listAllEvents(ctx, a.eventStore, opts) if err != nil { return nil, gRPCStoreError(err, "list events") } @@ -1002,9 +1027,17 @@ func (a *PipedAPI) CreateDeploymentChain(ctx context.Context, req *pipedservice. // TODO: Support find node apps by appLabels. - apps, _, err := a.applicationStore.List(ctx, datastore.ListOptions{ + opts := datastore.ListOptions{ Filters: filters, - }) + Orders: []datastore.Order{ + { + Field: "Id", + Direction: datastore.Asc, + }, + }, + Limit: listApplicationsPageSize, + } + apps, err := listAllApplications(ctx, a.applicationStore, opts) if err != nil { return nil, nil, err } diff --git a/pkg/app/server/grpcapi/web_api.go b/pkg/app/server/grpcapi/web_api.go index 4fdbedbe71..e91c933494 100644 --- a/pkg/app/server/grpcapi/web_api.go +++ b/pkg/app/server/grpcapi/web_api.go @@ -677,9 +677,10 @@ func (a *WebAPI) ListApplications(ctx context.Context, req *webservice.ListAppli } } - apps, _, err := a.applicationStore.List(ctx, datastore.ListOptions{ + apps, err := listAllApplications(ctx, a.applicationStore, datastore.ListOptions{ Filters: filters, Orders: orders, + Limit: listApplicationsPageSize, }) if err != nil { return nil, gRPCStoreError(err, "list applications")