From b6c1827aa1db27225a6f29a5b7db1cf2664031ac Mon Sep 17 00:00:00 2001 From: Florin Mirosnicencu Date: Thu, 23 Jul 2026 15:21:12 +0100 Subject: [PATCH 1/6] chore: extract file filtering logic outside of bundle manager --- bundle/bundle.go | 1 - bundle/bundle_manager.go | 56 +------- bundle/bundle_manager_test.go | 82 ------------ internal/util/supportedfiles/filter.go | 94 +++++++++++++ internal/util/supportedfiles/filter_test.go | 138 ++++++++++++++++++++ 5 files changed, 239 insertions(+), 132 deletions(-) create mode 100644 internal/util/supportedfiles/filter.go create mode 100644 internal/util/supportedfiles/filter_test.go diff --git a/bundle/bundle.go b/bundle/bundle.go index 0c3d6f93..4845ad42 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -120,7 +120,6 @@ func (b *deepCodeBundle) extendBundle(ctx context.Context, requestId string, upl } const ( - maxFileSize = 1024 * 1024 maxUploadBatchSize = 1024*1024*4 - 1024 // subtract 1k for potential headers jsonOverheadRequest = "{\"files\":{}}" jsonOverHeadRequestLength = len(jsonOverheadRequest) diff --git a/bundle/bundle_manager.go b/bundle/bundle_manager.go index 3a18b9e2..98a5bc48 100644 --- a/bundle/bundle_manager.go +++ b/bundle/bundle_manager.go @@ -19,10 +19,9 @@ package bundle import ( "context" "os" - "path/filepath" - "github.com/puzpuzpuz/xsync" "github.com/rs/zerolog" + "github.com/snyk/code-client-go/internal/util/supportedfiles" "github.com/snyk/code-client-go/internal/deepcode" "github.com/snyk/code-client-go/internal/util" @@ -38,8 +37,7 @@ type bundleManager struct { errorReporter observability.ErrorReporter logger *zerolog.Logger trackerFactory scan.TrackerFactory - supportedExtensions *xsync.MapOf[string, bool] - supportedConfigFiles *xsync.MapOf[string, bool] + supportedFilesFilter *supportedfiles.SupportedFilesFilter } type BundleManager interface { @@ -78,8 +76,7 @@ func NewBundleManager( errorReporter: errorReporter, logger: logger, trackerFactory: trackerFactory, - supportedExtensions: xsync.NewMapOf[bool](), - supportedConfigFiles: xsync.NewMapOf[bool](), + supportedFilesFilter: supportedfiles.NewSupportedFilesFilter(deepcodeClient, logger), } } @@ -125,22 +122,12 @@ func (b *bundleManager) create( if ctx.Err() != nil { return bundle, err // The cancellation error should be handled by the calling function } - var supported bool - supported, err = b.IsSupported(span.Context(), absoluteFilePath) - if err != nil { - return bundle, err - } - if !supported { - continue - } - fileInfo, fileErr := os.Stat(absoluteFilePath) - if fileErr != nil { - b.logger.Error().Err(err).Str("filePath", absoluteFilePath).Msg("Failed to read file info") - continue + supported, filterErr := b.supportedFilesFilter.IsFileSupported(span.Context(), absoluteFilePath) + if filterErr != nil { + return bundle, filterErr } - - if fileInfo.Size() == 0 || fileInfo.Size() > maxFileSize { + if !supported { continue } @@ -290,32 +277,3 @@ func (b *bundleManager) groupInBatches( } return batches } - -func (b *bundleManager) IsSupported(ctx context.Context, file string) (bool, error) { - if b.supportedExtensions.Size() == 0 && b.supportedConfigFiles.Size() == 0 { - filters, err := b.deepcodeClient.GetFilters(ctx) - if err != nil { - b.logger.Error().Err(err).Msg("could not get filters") - return false, err - } - - for _, ext := range filters.Extensions { - b.supportedExtensions.Store(ext, true) - } - for _, configFile := range filters.ConfigFiles { - // .gitignore and .dcignore should not be uploaded - // (https://github.com/snyk/code-client/blob/d6f6a2ce4c14cb4b05aa03fb9f03533d8cf6ca4a/src/files.ts#L138) - if configFile == ".gitignore" || configFile == ".dcignore" { - continue - } - b.supportedConfigFiles.Store(configFile, true) - } - } - - fileExtension := filepath.Ext(file) - fileName := filepath.Base(file) // Config files are compared to the file name, not just the extensions - _, isSupportedExtension := b.supportedExtensions.Load(fileExtension) - _, isSupportedConfigFile := b.supportedConfigFiles.Load(fileName) - - return isSupportedExtension || isSupportedConfigFile, nil -} diff --git a/bundle/bundle_manager_test.go b/bundle/bundle_manager_test.go index 8ab01243..f74822bc 100644 --- a/bundle/bundle_manager_test.go +++ b/bundle/bundle_manager_test.go @@ -455,88 +455,6 @@ func createTempFileInDir(t *testing.T, name string, size int, temporaryDir strin return documentURI, deepcode.BundleFile{Hash: hash, ContentSize: size} } -func Test_IsSupported_Extensions(t *testing.T) { - ctrl := gomock.NewController(t) - mockSnykCodeClient := deepcodeMocks.NewMockDeepcodeClient(ctrl) - mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ - ConfigFiles: []string{}, - Extensions: []string{".java"}, - }, nil) - mockInstrumentor := mocks.NewMockInstrumentor(ctrl) - mockErrorReporter := mocks.NewMockErrorReporter(ctrl) - mockTrackerFactory := trackerMocks.NewMockTrackerFactory(ctrl) - - bundler := bundle.NewBundleManager(mockSnykCodeClient, newLogger(t), mockInstrumentor, mockErrorReporter, mockTrackerFactory) - - t.Run("should return true for supported languages", func(t *testing.T) { - supported, _ := bundler.IsSupported(t.Context(), "C:\\some\\path\\Test.java") - assert.True(t, supported) - }) - - t.Run("should return false for unsupported languages", func(t *testing.T) { - supported, _ := bundler.IsSupported(t.Context(), "C:\\some\\path\\Test.rs") - assert.False(t, supported) - }) - - t.Run("should cache supported extensions", func(t *testing.T) { - path := "C:\\some\\path\\Test.rs" - _, _ = bundler.IsSupported(t.Context(), path) - _, _ = bundler.IsSupported(t.Context(), path) - }) -} - -func Test_IsSupported_ConfigFiles(t *testing.T) { - configFilesFromFiltersEndpoint := []string{ - ".supportedConfigFile", - ".snyk", - ".dcignore", - ".gitignore", - } - expectedConfigFiles := []string{ // .dcignore and .gitignore should be excluded - ".supportedConfigFile", - ".snyk", - } - - ctrl := gomock.NewController(t) - mockSnykCodeClient := deepcodeMocks.NewMockDeepcodeClient(ctrl) - mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ - ConfigFiles: configFilesFromFiltersEndpoint, - Extensions: []string{}, - }, nil) - mockInstrumentor := mocks.NewMockInstrumentor(ctrl) - mockErrorReporter := mocks.NewMockErrorReporter(ctrl) - mockTrackerFactory := trackerMocks.NewMockTrackerFactory(ctrl) - - bundler := bundle.NewBundleManager(mockSnykCodeClient, newLogger(t), mockInstrumentor, mockErrorReporter, mockTrackerFactory) - dir, _ := os.Getwd() - - t.Run("should return true for supported config files", func(t *testing.T) { - for _, file := range expectedConfigFiles { - path := filepath.Join(dir, file) - supported, _ := bundler.IsSupported(t.Context(), path) - assert.True(t, supported) - } - }) - t.Run("should exclude .gitignore and .dcignore", func(t *testing.T) { - for _, file := range []string{".gitignore", ".dcignore"} { - path := filepath.Join(dir, file) - supported, _ := bundler.IsSupported(t.Context(), path) - assert.False(t, supported) - } - }) - t.Run("should return false for unsupported config files", func(t *testing.T) { - path := "C:\\some\\path\\.unsupported" - supported, _ := bundler.IsSupported(t.Context(), path) - assert.False(t, supported) - }) - - t.Run("should cache supported extensions", func(t *testing.T) { - path := "C:\\some\\path\\Test.rs" - _, _ = bundler.IsSupported(t.Context(), path) - _, _ = bundler.IsSupported(t.Context(), path) - }) -} - func setup(t *testing.T) string { t.Helper() dir := t.TempDir() diff --git a/internal/util/supportedfiles/filter.go b/internal/util/supportedfiles/filter.go new file mode 100644 index 00000000..d628a67c --- /dev/null +++ b/internal/util/supportedfiles/filter.go @@ -0,0 +1,94 @@ +/* + * © 2022-2024 Snyk Limited All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package supportedfiles + +import ( + "context" + "os" + "path/filepath" + + "github.com/puzpuzpuz/xsync" + "github.com/rs/zerolog" + "github.com/snyk/code-client-go/internal/deepcode" +) + +const maxFileSize = 1024 * 1024 + +type SupportedFilesFilter struct { + client deepcode.DeepcodeClient + logger *zerolog.Logger + supportedExtensions *xsync.MapOf[string, bool] + supportedConfigFiles *xsync.MapOf[string, bool] +} + +func NewSupportedFilesFilter(client deepcode.DeepcodeClient, logger *zerolog.Logger) *SupportedFilesFilter { + return &SupportedFilesFilter{ + client: client, + logger: logger, + supportedExtensions: xsync.NewMapOf[bool](), + supportedConfigFiles: xsync.NewMapOf[bool](), + } +} + +func (s *SupportedFilesFilter) isPathSupported(ctx context.Context, path string) (bool, error) { + if s.supportedExtensions.Size() == 0 && s.supportedConfigFiles.Size() == 0 { + filters, err := s.client.GetFilters(ctx) + if err != nil { + s.logger.Error().Err(err).Msg("could not get filters") + return false, err + } + + for _, ext := range filters.Extensions { + s.supportedExtensions.Store(ext, true) + } + for _, configFile := range filters.ConfigFiles { + // .gitignore and .dcignore should not be uploaded + // (https://github.com/snyk/code-client/blob/d6f6a2ce4c14cb4b05aa03fb9f03533d8cf6ca4a/src/files.ts#L138) + if configFile == ".gitignore" || configFile == ".dcignore" { + continue + } + s.supportedConfigFiles.Store(configFile, true) + } + } + + fileExtension := filepath.Ext(path) + fileName := filepath.Base(path) // Config files are compared to the file name, not just the extensions + _, isSupportedExtension := s.supportedExtensions.Load(fileExtension) + _, isSupportedConfigFile := s.supportedConfigFiles.Load(fileName) + + return isSupportedExtension || isSupportedConfigFile, nil +} + +func (s *SupportedFilesFilter) IsFileSupported(ctx context.Context, path string) (bool, error) { + supported, err := s.isPathSupported(ctx, path) + if err != nil { + return false, err + } + if !supported { + return false, nil + } + + fileInfo, fileErr := os.Stat(path) + if fileErr != nil { + s.logger.Error().Err(fileErr).Str("filePath", path).Msg("Failed to read file info") + return false, nil + } + + if fileInfo.Size() == 0 || fileInfo.Size() > maxFileSize { + return false, nil + } + return true, nil +} diff --git a/internal/util/supportedfiles/filter_test.go b/internal/util/supportedfiles/filter_test.go new file mode 100644 index 00000000..9dbb2fd4 --- /dev/null +++ b/internal/util/supportedfiles/filter_test.go @@ -0,0 +1,138 @@ +/* + * © 2024 Snyk Limited All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package supportedfiles_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/snyk/code-client-go/internal/deepcode" + deepcodeMocks "github.com/snyk/code-client-go/internal/deepcode/mocks" + "github.com/snyk/code-client-go/internal/util/supportedfiles" +) + +func Test_IsFileSupported_Extensions(t *testing.T) { + ctrl := gomock.NewController(t) + mockSnykCodeClient := deepcodeMocks.NewMockDeepcodeClient(ctrl) + mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ + ConfigFiles: []string{}, + Extensions: []string{".java"}, + }, nil) + + filter := supportedfiles.NewSupportedFilesFilter(mockSnykCodeClient, newLogger(t)) + dir := t.TempDir() + + t.Run("should return true for supported languages", func(t *testing.T) { + supported, _ := filter.IsFileSupported(t.Context(), createFile(t, dir, "Test.java")) + assert.True(t, supported) + }) + + t.Run("should return false for unsupported languages", func(t *testing.T) { + supported, _ := filter.IsFileSupported(t.Context(), createFile(t, dir, "Test.rs")) + assert.False(t, supported) + }) +} + +func Test_IsFileSupported_ConfigFiles(t *testing.T) { + configFilesFromFiltersEndpoint := []string{ + ".supportedConfigFile", + ".snyk", + ".dcignore", + ".gitignore", + } + expectedConfigFiles := []string{ // .dcignore and .gitignore should be excluded + ".supportedConfigFile", + ".snyk", + } + + ctrl := gomock.NewController(t) + mockSnykCodeClient := deepcodeMocks.NewMockDeepcodeClient(ctrl) + getFiltersCalls := 0 + mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).DoAndReturn(func(context.Context) (deepcode.FiltersResponse, error) { + getFiltersCalls++ + return deepcode.FiltersResponse{ + ConfigFiles: configFilesFromFiltersEndpoint, + Extensions: []string{}, + }, nil + }) + + filter := supportedfiles.NewSupportedFilesFilter(mockSnykCodeClient, newLogger(t)) + dir := t.TempDir() + + t.Run("should return true for supported config files", func(t *testing.T) { + for _, file := range expectedConfigFiles { + supported, _ := filter.IsFileSupported(t.Context(), createFile(t, dir, file)) + assert.True(t, supported) + } + }) + t.Run("should exclude .gitignore and .dcignore", func(t *testing.T) { + for _, file := range []string{".gitignore", ".dcignore"} { + supported, _ := filter.IsFileSupported(t.Context(), createFile(t, dir, file)) + assert.False(t, supported) + } + }) + t.Run("should return false for unsupported config files", func(t *testing.T) { + supported, _ := filter.IsFileSupported(t.Context(), createFile(t, dir, ".unsupported")) + assert.False(t, supported) + }) +} + +func Test_IsFileSupported_FileSize(t *testing.T) { + ctrl := gomock.NewController(t) + mockSnykCodeClient := deepcodeMocks.NewMockDeepcodeClient(ctrl) + mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ + ConfigFiles: []string{}, + Extensions: []string{".java"}, + }, nil) + + filter := supportedfiles.NewSupportedFilesFilter(mockSnykCodeClient, newLogger(t)) + dir := t.TempDir() + + t.Run("should return false for empty files", func(t *testing.T) { + path := filepath.Join(dir, "empty.java") + require.NoError(t, os.WriteFile(path, []byte{}, 0600)) + supported, _ := filter.IsFileSupported(t.Context(), path) + assert.False(t, supported) + }) + + t.Run("should return false for files over the max size", func(t *testing.T) { + path := filepath.Join(dir, "big.java") + require.NoError(t, os.WriteFile(path, make([]byte, 1024*1024+1), 0600)) + supported, _ := filter.IsFileSupported(t.Context(), path) + assert.False(t, supported) + }) +} + +func createFile(t *testing.T, dir, name string) string { + t.Helper() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte("some content so the file won't be skipped"), 0600)) + return path +} + +func newLogger(t *testing.T) *zerolog.Logger { + t.Helper() + logger := zerolog.New(zerolog.NewTestWriter(t)) + return &logger +} From 9fbdf3fbeb0543a53a1fee13b56e09d4bc28eb36 Mon Sep 17 00:00:00 2001 From: Florin Mirosnicencu Date: Thu, 23 Jul 2026 15:56:28 +0100 Subject: [PATCH 2/6] chore: update test service client to accept upload_revisions --- internal/api/test/2025-04-07/helper.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/api/test/2025-04-07/helper.go b/internal/api/test/2025-04-07/helper.go index 1390d90f..b63018d6 100644 --- a/internal/api/test/2025-04-07/helper.go +++ b/internal/api/test/2025-04-07/helper.go @@ -34,6 +34,21 @@ func WithInputBundle(id string, localFilePath string, repoUrl *string, limitTest } } +func WithInputUploadRevision(id string, localFilePath string, repoUrl *string) CreateTestOption { + return func(body *CreateTestApplicationVndAPIPlusJSONRequestBody) { + revisionInput := v20250407.TestInputUploadRevision{ + RevisionId: id, + Type: v20250407.UploadRevision, + Metadata: &struct { + LocalFilePath *string `json:"local_file_path,omitempty"` + RepoUrl *string `json:"repo_url,omitempty"` + }{LocalFilePath: &localFilePath, RepoUrl: repoUrl}, + } + + body.Data.Attributes.Input.FromTestInputUploadRevision(revisionInput) + } +} + func WithInputLegacyScmProject(project v20250407.TestInputLegacyScmProject) CreateTestOption { return func(body *CreateTestApplicationVndAPIPlusJSONRequestBody) { body.Data.Attributes.Input.FromTestInputLegacyScmProject(project) From 50f522bd19e9465ed3d7f46f256444987f7715cd Mon Sep 17 00:00:00 2001 From: Florin Mirosnicencu Date: Fri, 24 Jul 2026 10:39:27 +0100 Subject: [PATCH 3/6] feat: integrate with FUA client from GAF --- go.mod | 14 +- go.sum | 34 +-- .../uploadrevision/mocks/upload_revision.go | 52 +++++ internal/uploadrevision/upload_revision.go | 152 ++++++++++++ .../uploadrevision/upload_revision_test.go | 219 ++++++++++++++++++ .../uploadrevision/utf8_file_internal_test.go | 86 +++++++ 6 files changed, 534 insertions(+), 23 deletions(-) create mode 100644 internal/uploadrevision/mocks/upload_revision.go create mode 100644 internal/uploadrevision/upload_revision.go create mode 100644 internal/uploadrevision/upload_revision_test.go create mode 100644 internal/uploadrevision/utf8_file_internal_test.go diff --git a/go.mod b/go.mod index a41ffb6f..a1e4f2d9 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/puzpuzpuz/xsync v1.5.2 github.com/rs/zerolog v1.34.0 github.com/snyk/error-catalog-golang-public v0.0.0-20260205094614-116c03822905 - github.com/snyk/go-application-framework v0.0.0-20260504124159-e299fb4a44a1 + github.com/snyk/go-application-framework v0.8.1-0.20260727102853-16db50c1cb56 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.11.1 golang.org/x/net v0.55.0 @@ -100,13 +100,13 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.52.0 // indirect - golang.org/x/mod v0.35.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/mod v0.36.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.45.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250404141209-ee84b53bf3d0 // indirect google.golang.org/grpc v1.71.0 // indirect google.golang.org/protobuf v1.36.6 // indirect diff --git a/go.sum b/go.sum index cc27d7e4..b1d3ba6f 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,10 @@ github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnB github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/snyk/error-catalog-golang-public v0.0.0-20260205094614-116c03822905 h1:pUe6iOWHEOFY0t4u4ssXeTqpMmZBu1xq06VBFI9zUik= github.com/snyk/error-catalog-golang-public v0.0.0-20260205094614-116c03822905/go.mod h1:Ytttq7Pw4vOCu9NtRQaOeDU2dhBYUyNBe6kX4+nIIQ4= -github.com/snyk/go-application-framework v0.0.0-20260504124159-e299fb4a44a1 h1:yiG0VOwT292qxs+g6HTnr6O8is/sKEI+9XTUGf2tFDU= -github.com/snyk/go-application-framework v0.0.0-20260504124159-e299fb4a44a1/go.mod h1:yTGCJKf6RmqdwrNs5B9zmukL9x1D8EhfSK8mzaPB1Rk= +github.com/snyk/go-application-framework v0.8.1-0.20260727094148-7e9b94e9b8ba h1:Lo7An3bw612z3cQbGvrgcaXdrAUsKNWrsXjoYDPa+78= +github.com/snyk/go-application-framework v0.8.1-0.20260727094148-7e9b94e9b8ba/go.mod h1:0YC7xCETnFTdz6rq8OQPL0aWekMLOkBQu1FE4/cReMA= +github.com/snyk/go-application-framework v0.8.1-0.20260727102853-16db50c1cb56 h1:eY/MAu7AIqdActA/3vpnit4GYkZGv/VHXlgtMpZwU8c= +github.com/snyk/go-application-framework v0.8.1-0.20260727102853-16db50c1cb56/go.mod h1:0YC7xCETnFTdz6rq8OQPL0aWekMLOkBQu1FE4/cReMA= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530 h1:s9PHNkL6ueYRiAKNfd8OVxlUOqU3qY0VDbgCD1f6WQY= github.com/snyk/go-httpauth v0.0.0-20231117135515-eb445fea7530/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -327,14 +329,14 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -354,8 +356,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -381,24 +383,24 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/uploadrevision/mocks/upload_revision.go b/internal/uploadrevision/mocks/upload_revision.go new file mode 100644 index 00000000..1bc8105c --- /dev/null +++ b/internal/uploadrevision/mocks/upload_revision.go @@ -0,0 +1,52 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: upload_revision.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + uploadrevision "github.com/snyk/code-client-go/internal/uploadrevision" + scan "github.com/snyk/code-client-go/scan" +) + +// MockUploadRevision is a mock of UploadRevision interface. +type MockUploadRevision struct { + ctrl *gomock.Controller + recorder *MockUploadRevisionMockRecorder +} + +// MockUploadRevisionMockRecorder is the mock recorder for MockUploadRevision. +type MockUploadRevisionMockRecorder struct { + mock *MockUploadRevision +} + +// NewMockUploadRevision creates a new mock instance. +func NewMockUploadRevision(ctrl *gomock.Controller) *MockUploadRevision { + mock := &MockUploadRevision{ctrl: ctrl} + mock.recorder = &MockUploadRevisionMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockUploadRevision) EXPECT() *MockUploadRevisionMockRecorder { + return m.recorder +} + +// Upload mocks base method. +func (m *MockUploadRevision) Upload(ctx context.Context, requestId string, target scan.Target, files <-chan string) (uploadrevision.RevisionID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upload", ctx, requestId, target, files) + ret0, _ := ret[0].(uploadrevision.RevisionID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Upload indicates an expected call of Upload. +func (mr *MockUploadRevisionMockRecorder) Upload(ctx, requestId, target, files interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upload", reflect.TypeOf((*MockUploadRevision)(nil).Upload), ctx, requestId, target, files) +} diff --git a/internal/uploadrevision/upload_revision.go b/internal/uploadrevision/upload_revision.go new file mode 100644 index 00000000..7135126b --- /dev/null +++ b/internal/uploadrevision/upload_revision.go @@ -0,0 +1,152 @@ +/* + * © 2024 Snyk Limited All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uploadrevision + +import ( + "bytes" + "context" + "errors" + "io" + "io/fs" + "net/http" + + "github.com/rs/zerolog" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" + + "github.com/snyk/code-client-go/bundle" + "github.com/snyk/code-client-go/internal/deepcode" + "github.com/snyk/code-client-go/internal/util" + "github.com/snyk/code-client-go/internal/util/supportedfiles" + "github.com/snyk/code-client-go/scan" +) + +//go:generate go tool github.com/golang/mock/mockgen -destination=mocks/upload_revision.go -source=upload_revision.go -package mocks + +type RevisionID string + +type UploadRevision interface { + Upload(ctx context.Context, requestId string, target scan.Target, files <-chan string) (RevisionID, error) +} + +type uploadRevision struct { + client fileupload.Client + supportedFilesFilter *supportedfiles.SupportedFilesFilter + logger *zerolog.Logger +} + +var _ UploadRevision = (*uploadRevision)(nil) + +func NewUploadRevision(httpClient *http.Client, cfg fileupload.Config, deepcodeClient deepcode.DeepcodeClient, logger *zerolog.Logger) *uploadRevision { + client := fileupload.NewClient( + httpClient, + cfg, + fileupload.WithPathEncoder(util.EncodePath), + fileupload.WithContentTranscoder(newUTF8File), + ) + return &uploadRevision{ + client: client, + supportedFilesFilter: supportedfiles.NewSupportedFilesFilter(deepcodeClient, logger), + logger: logger, + } +} + +// utf8File holds a file's content converted to UTF-8, so that the size it reports is the size +// of the content that gets streamed rather than the size of the content on disk. The upload +// client closes the file this content was read from. +type utf8File struct { + info fs.FileInfo + reader *bytes.Reader +} + +var _ fs.File = (*utf8File)(nil) + +func newUTF8File(f fs.File) (fs.File, error) { + content, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + info, err := f.Stat() + if err != nil { + return nil, err + } + + // Falls back to the raw content like util.Hash does, so that a file which cannot be + // converted is still uploaded. + utf8Content, err := util.ConvertToUTF8(content) + if err != nil { + utf8Content = content + } + + return &utf8File{info: info, reader: bytes.NewReader(utf8Content)}, nil +} + +func (u *utf8File) Read(p []byte) (int, error) { + return u.reader.Read(p) +} + +func (u *utf8File) Close() error { + return nil +} + +func (u *utf8File) Stat() (fs.FileInfo, error) { + return utf8FileInfo{FileInfo: u.info, size: u.reader.Size()}, nil +} + +type utf8FileInfo struct { + fs.FileInfo + size int64 +} + +func (i utf8FileInfo) Size() int64 { + return i.size +} + +func (u *uploadRevision) Upload(ctx context.Context, requestId string, target scan.Target, files <-chan string) (RevisionID, error) { + var supported []string + noFiles := true + for path := range files { + noFiles = false + isSupported, err := u.supportedFilesFilter.IsFileSupported(ctx, path) + if err != nil { + return "", err + } + if isSupported { + supported = append(supported, path) + } + } + + if noFiles { + return "", bundle.NoFilesError{} + } + + supportedFiles := make(chan string, len(supported)) + for _, path := range supported { + supportedFiles <- path + } + close(supportedFiles) + + res, err := u.client.CreateRevisionFromChan(ctx, supportedFiles, target.GetPath()) + if err != nil { + if errors.Is(err, fileupload.ErrNoFilesProvided) { + return "", bundle.NoFilesError{} + } + return "", err + } + + return RevisionID(res.RevisionID.String()), nil +} diff --git a/internal/uploadrevision/upload_revision_test.go b/internal/uploadrevision/upload_revision_test.go new file mode 100644 index 00000000..31980f5f --- /dev/null +++ b/internal/uploadrevision/upload_revision_test.go @@ -0,0 +1,219 @@ +/* + * © 2024 Snyk Limited All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uploadrevision_test + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + "github.com/rs/zerolog" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" + "github.com/stretchr/testify/suite" + + "github.com/snyk/code-client-go/bundle" + "github.com/snyk/code-client-go/internal/deepcode" + deepcodeMocks "github.com/snyk/code-client-go/internal/deepcode/mocks" + "github.com/snyk/code-client-go/internal/uploadrevision" + "github.com/snyk/code-client-go/scan" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func response(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Body: io.NopCloser(strings.NewReader(body)), + Header: make(http.Header), + } +} + +type uploadRevisionSuite struct { + suite.Suite + deepcodeClient *deepcodeMocks.MockDeepcodeClient + uploader uploadrevision.UploadRevision + uploaded map[string]string + uploadCall map[string]int + revID uuid.UUID +} + +func TestUploadRevisionSuite(t *testing.T) { + suite.Run(t, new(uploadRevisionSuite)) +} + +func (s *uploadRevisionSuite) SetupTest() { + ctrl := gomock.NewController(s.T()) + s.deepcodeClient = deepcodeMocks.NewMockDeepcodeClient(ctrl) + s.uploaded = map[string]string{} + s.uploadCall = map[string]int{} + s.revID = uuid.New() + populateCall := 0 + + createResp := fmt.Sprintf(`{"data":{"id":%q,"type":"upload_revision","attributes":{"revision_type":"snapshot","sealed":false}}}`, s.revID) + sealResp := fmt.Sprintf(`{"data":{"id":%q,"type":"upload_revision","attributes":{"revision_type":"snapshot","sealed":true}}}`, s.revID) + + transport := roundTripFunc(func(r *http.Request) (*http.Response, error) { + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files"): + populateCall++ + gz, err := gzip.NewReader(r.Body) + if err != nil { + return nil, err + } + _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + return nil, err + } + mr := multipart.NewReader(gz, params["boundary"]) + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + content, _ := io.ReadAll(part) + s.uploaded[part.FormName()] = string(content) + s.uploadCall[part.FormName()] = populateCall + } + return response(http.StatusNoContent, ""), nil + case r.Method == http.MethodPost: + return response(http.StatusCreated, createResp), nil + case r.Method == http.MethodPatch: + return response(http.StatusOK, sealResp), nil + default: + return nil, fmt.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + } + }) + + logger := zerolog.Nop() + s.uploader = uploadrevision.NewUploadRevision( + &http.Client{Transport: transport}, + fileupload.Config{BaseURL: "https://example.com", OrgID: uuid.New()}, + s.deepcodeClient, + &logger, + ) +} + +func (s *uploadRevisionSuite) TestUpload_NoFiles() { + files := make(chan string) + close(files) + + revisionID, err := s.uploader.Upload(context.Background(), "requestId", scan.RepositoryTarget{LocalFilePath: "/path"}, files) + + s.Empty(revisionID) + s.True(bundle.IsNoFilesError(err)) + s.Empty(s.uploaded) +} + +func (s *uploadRevisionSuite) TestUpload_SingleFileExcludedByFilters() { + s.deepcodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ + ConfigFiles: []string{}, + Extensions: []string{".java"}, + }, nil) + + files := make(chan string, 1) + files <- "/path/file.txt" + close(files) + + revisionID, err := s.uploader.Upload(context.Background(), "requestId", scan.RepositoryTarget{LocalFilePath: "/path"}, files) + + s.True(bundle.IsNoFilesError(err)) + s.Empty(revisionID) + s.Empty(s.uploaded) +} + +func (s *uploadRevisionSuite) TestUpload_SingleFile() { + s.deepcodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ + ConfigFiles: []string{}, + Extensions: []string{".go"}, + }, nil) + + dir := s.T().TempDir() + s.writeFile(dir, "main.go", []byte("package main")) + + files := make(chan string, 1) + files <- filepath.Join(dir, "main.go") + close(files) + + revisionID, err := s.uploader.Upload(context.Background(), "requestId", scan.RepositoryTarget{LocalFilePath: dir}, files) + + s.Require().NoError(err) + s.Equal(uploadrevision.RevisionID(s.revID.String()), revisionID) + s.Require().Len(s.uploaded, 1) + s.Equal("package main", s.uploaded["main.go"]) + s.Equal(1, s.uploadCall["main.go"]) +} + +func (s *uploadRevisionSuite) TestUpload_EncodesPaths() { + s.deepcodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ + ConfigFiles: []string{}, + Extensions: []string{".go"}, + }, nil) + + dir := s.T().TempDir() + s.writeFile(dir, filepath.Join("sub dir", "a b.go"), []byte("package a")) + + files := make(chan string, 1) + files <- filepath.Join(dir, "sub dir", "a b.go") + close(files) + + _, err := s.uploader.Upload(context.Background(), "requestId", scan.RepositoryTarget{LocalFilePath: dir}, files) + + s.Require().NoError(err) + s.Require().Len(s.uploaded, 1) + s.Equal("package a", s.uploaded["sub%20dir/a%20b.go"]) +} + +func (s *uploadRevisionSuite) TestUpload_TranscodesContent() { + s.deepcodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{ + ConfigFiles: []string{}, + Extensions: []string{".go"}, + }, nil) + + dir := s.T().TempDir() + s.writeFile(dir, "main.go", append([]byte("package main"), 0xff)) + + files := make(chan string, 1) + files <- filepath.Join(dir, "main.go") + close(files) + + _, err := s.uploader.Upload(context.Background(), "requestId", scan.RepositoryTarget{LocalFilePath: dir}, files) + + s.Require().NoError(err) + s.Require().Len(s.uploaded, 1) + s.Equal("package main�", s.uploaded["main.go"]) +} + +func (s *uploadRevisionSuite) writeFile(dir, relPath string, content []byte) { + fullPath := filepath.Join(dir, relPath) + s.Require().NoError(os.MkdirAll(filepath.Dir(fullPath), 0o755)) + s.Require().NoError(os.WriteFile(fullPath, content, 0o600)) +} diff --git a/internal/uploadrevision/utf8_file_internal_test.go b/internal/uploadrevision/utf8_file_internal_test.go new file mode 100644 index 00000000..765a1a7e --- /dev/null +++ b/internal/uploadrevision/utf8_file_internal_test.go @@ -0,0 +1,86 @@ +/* + * © 2024 Snyk Limited All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The transcoder is tested from inside the package because the upload client it is passed to +// enforces its size limits in megabytes, which makes the reported size unobservable from a test +// going through UploadRevision. +package uploadrevision + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_newUTF8File_reportsTheConvertedSize(t *testing.T) { + // Each byte that is not valid UTF-8 is converted to U+FFFD, which takes three bytes. + f := openFile(t, append([]byte("package main"), 0xff)) + + transcoded, err := newUTF8File(f) + require.NoError(t, err) + + info, err := transcoded.Stat() + require.NoError(t, err) + assert.Equal(t, int64(15), info.Size()) + + content, err := io.ReadAll(transcoded) + require.NoError(t, err) + assert.Equal(t, "package main�", string(content)) + assert.Len(t, content, int(info.Size())) +} + +func Test_newUTF8File_reportsTheSameSizeAfterReading(t *testing.T) { + f := openFile(t, append([]byte("package main"), 0xff)) + + transcoded, err := newUTF8File(f) + require.NoError(t, err) + + _, err = io.ReadAll(transcoded) + require.NoError(t, err) + + info, err := transcoded.Stat() + require.NoError(t, err) + assert.Equal(t, int64(15), info.Size()) +} + +func Test_newUTF8File_keepsTheRemainingFileInfo(t *testing.T) { + f := openFile(t, []byte("package main")) + + transcoded, err := newUTF8File(f) + require.NoError(t, err) + + info, err := transcoded.Stat() + require.NoError(t, err) + assert.Equal(t, "main.go", info.Name()) + assert.True(t, info.Mode().IsRegular()) +} + +func openFile(t *testing.T, content []byte) *os.File { + t.Helper() + + path := filepath.Join(t.TempDir(), "main.go") + require.NoError(t, os.WriteFile(path, content, 0o600)) + + f, err := os.Open(path) + require.NoError(t, err) + t.Cleanup(func() { f.Close() }) + + return f +} From 9c0db59f1de51439bd9180b80a0b3d881c9bffde Mon Sep 17 00:00:00 2001 From: Florin Mirosnicencu Date: Fri, 24 Jul 2026 14:03:00 +0100 Subject: [PATCH 4/6] feat: trigger FUA upload based on FF and analysis option --- internal/analysis/analysis.go | 33 +++++-- internal/analysis/analysis_test.go | 97 +++++++++++++++++++ internal/analysis/mocks/analysis.go | 8 +- .../commands/code_workflow/native_workflow.go | 6 ++ .../code_workflow/native_workflow_test.go | 74 +++++++++++++- pkg/code/code.go | 3 + scan.go | 59 +++++++++-- scan_test.go | 7 +- 8 files changed, 260 insertions(+), 27 deletions(-) diff --git a/internal/analysis/analysis.go b/internal/analysis/analysis.go index 0449c203..d557030c 100644 --- a/internal/analysis/analysis.go +++ b/internal/analysis/analysis.go @@ -47,19 +47,22 @@ import ( //go:generate go tool github.com/golang/mock/mockgen -destination=mocks/analysis.go -source=analysis.go -package mocks type AnalysisOrchestrator interface { - RunTest(ctx context.Context, orgId string, b bundle.Bundle, target scan.Target, reportingOptions AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) + RunTest(ctx context.Context, orgId string, b bundle.Bundle, revisionId *string, target scan.Target, reportingOptions AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) RunTestRemote(ctx context.Context, orgId string, reportingOptions AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) RunLegacyTest(ctx context.Context, bundleHash string, shardKey string, limitToFiles []string, severity int) (*sarif.SarifResponse, scan.LegacyScanStatus, error) } +var ErrMissingTestInput = errors.New("either a bundle or an upload revision id is required") + type AnalysisConfig struct { - Report bool - ProjectName *string - ProjectTags *[]string - TargetName *string - TargetReference *string - ProjectId *uuid.UUID - CommitId *string + Report bool + ProjectName *string + ProjectTags *[]string + TargetName *string + TargetReference *string + ProjectId *uuid.UUID + CommitId *string + UploadFileContentToFileUploadApi bool } type analysisOrchestrator struct { @@ -233,7 +236,7 @@ func (a *analysisOrchestrator) createTestAndGetResults(ctx context.Context, orgI return result, metadata, err } -func (a *analysisOrchestrator) RunTest(ctx context.Context, orgId string, b bundle.Bundle, target scan.Target, reportingConfig AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) { +func (a *analysisOrchestrator) RunTest(ctx context.Context, orgId string, b bundle.Bundle, revisionId *string, target scan.Target, reportingConfig AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) { var commitId *string = nil var repoUrl *string = nil var branchName *string = nil @@ -252,8 +255,18 @@ func (a *analysisOrchestrator) RunTest(ctx context.Context, orgId string, b bund } } + var input testApi.CreateTestOption + switch { + case b != nil: + input = testApi.WithInputBundle(b.GetBundleHash(), target.GetPath(), repoUrl, b.GetLimitToFiles(), commitId, branchName) + case revisionId != nil: + input = testApi.WithInputUploadRevision(*revisionId, target.GetPath(), repoUrl) + default: + return nil, nil, ErrMissingTestInput + } + body := testApi.NewCreateTestApplicationBody( - testApi.WithInputBundle(b.GetBundleHash(), target.GetPath(), repoUrl, b.GetLimitToFiles(), commitId, branchName), + input, testApi.WithScanType(a.testType), testApi.WithProjectName(reportingConfig.ProjectName), testApi.WithProjectTags(reportingConfig.ProjectTags), diff --git a/internal/analysis/analysis_test.go b/internal/analysis/analysis_test.go index 33e794f7..915a1e69 100644 --- a/internal/analysis/analysis_test.go +++ b/internal/analysis/analysis_test.go @@ -338,6 +338,7 @@ func TestAnalysis_RunTest(t *testing.T) { t.Context(), orgId, inputBundle, + nil, targetId, analysis.AnalysisConfig{ Report: report, @@ -409,6 +410,7 @@ func TestAnalysis_RunTest_WithBranchName(t *testing.T) { t.Context(), orgId, inputBundle, + nil, targetId, analysis.AnalysisConfig{ Report: report, @@ -468,6 +470,7 @@ func TestAnalysis_RunTest_WithProjectTags(t *testing.T) { t.Context(), orgId, inputBundle, + nil, targetId, analysis.AnalysisConfig{ Report: report, @@ -484,6 +487,100 @@ func TestAnalysis_RunTest_WithProjectTags(t *testing.T) { assert.Equal(t, sarifResponse.Version, result.Sarif.Version) } +func mockTestCreatedResponseWithRevisionValidation(t *testing.T, mockHTTPClient *httpmocks.MockHTTPClient, testId uuid.UUID, orgId string, expectedRevisionId string, responseCode int) { + t.Helper() + response := v20250407.NewTestResponse() + response.Data.Id = testId + responseBodyBytes, err := json.Marshal(response) + assert.NoError(t, err) + expectedTestCreatedUrl := fmt.Sprintf("http://localhost/hidden/orgs/%s/tests?version=%s", orgId, v20250407.ApiVersion) + mockHTTPClient.EXPECT().Do(mock.MatchedBy(func(i interface{}) bool { + req := i.(*http.Request) + body, _ := io.ReadAll(req.Body) + var testRequestBody v20250407Models.CreateTestRequestBody + err := json.Unmarshal(body, &testRequestBody) + assert.NoError(t, err) + revision, err := testRequestBody.Data.Attributes.Input.AsTestInputUploadRevision() + assert.NoError(t, err) + assert.Equal(t, expectedRevisionId, revision.RevisionId) + + return req.URL.String() == expectedTestCreatedUrl && req.Method == http.MethodPost + })).Times(1).Return(&http.Response{ + StatusCode: responseCode, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(bytes.NewReader(responseBodyBytes)), + }, mockDeriveErrorFromStatusCode(responseCode)) +} + +func TestAnalysis_RunTest_WithUploadRevision(t *testing.T) { + mockConfig, mockHTTPClient, mockInstrumentor, mockErrorReporter, mockTracker, mockTrackerFactory, logger := setup(t, nil) + mockTracker.EXPECT().Begin(gomock.Eq("Snyk Code analysis for ../mypath/"), gomock.Eq("Retrieving results...")).Return() + mockTracker.EXPECT().End(gomock.Eq("Analysis completed.")).Return() + + orgId := "4a72d1db-b465-4764-99e1-ecedad03b06a" + projectId := uuid.New() + snapshotId := uuid.New() + testId := uuid.New() + revisionId := uuid.NewString() + targetId, err := scan.NewRepositoryTarget("../mypath/") + assert.NoError(t, err) + + mockTestCreatedResponseWithRevisionValidation(t, mockHTTPClient, testId, orgId, revisionId, http.StatusCreated) + mockTestStatusResponse(t, mockHTTPClient, orgId, testId, http.StatusOK) + + expectedWebuilink := "" + expectedDocumentPath := "/1234" + mockResultCompletedResponse(t, mockHTTPClient, expectedWebuilink, projectId, snapshotId, orgId, testId, expectedDocumentPath, http.StatusOK) + + sarifResponse := sarif.SarifDocument{Version: "42.0"} + mockGetComponentResponse(t, sarifResponse, expectedDocumentPath, mockHTTPClient, http.StatusOK) + + analysisOrchestrator := analysis.NewAnalysisOrchestrator( + mockConfig, + mockHTTPClient, + analysis.WithLogger(&logger), + analysis.WithInstrumentor(mockInstrumentor), + analysis.WithTrackerFactory(mockTrackerFactory), + analysis.WithErrorReporter(mockErrorReporter), + ) + + result, resultMetadata, err := analysisOrchestrator.RunTest( + t.Context(), + orgId, + nil, + &revisionId, + targetId, + analysis.AnalysisConfig{}, + ) + + require.NoError(t, err) + assert.NotNil(t, result) + assert.NotNil(t, resultMetadata) + assert.Equal(t, sarifResponse.Version, result.Sarif.Version) +} + +func TestAnalysis_RunTest_NoInput(t *testing.T) { + mockConfig, mockHTTPClient, mockInstrumentor, mockErrorReporter, _, mockTrackerFactory, logger := setup(t, nil) + + targetId, err := scan.NewRepositoryTarget("../mypath/") + assert.NoError(t, err) + + analysisOrchestrator := analysis.NewAnalysisOrchestrator( + mockConfig, + mockHTTPClient, + analysis.WithLogger(&logger), + analysis.WithInstrumentor(mockInstrumentor), + analysis.WithTrackerFactory(mockTrackerFactory), + analysis.WithErrorReporter(mockErrorReporter), + ) + + _, _, err = analysisOrchestrator.RunTest(t.Context(), "org", nil, nil, targetId, analysis.AnalysisConfig{}) + + assert.ErrorIs(t, err, analysis.ErrMissingTestInput) +} + func TestAnalysis_RunTestRemote(t *testing.T) { mockConfig, mockHTTPClient, mockInstrumentor, mockErrorReporter, mockTracker, mockTrackerFactory, logger := setup(t, nil) mockTracker.EXPECT().Begin(gomock.Eq("Snyk Code analysis for remote project"), gomock.Eq("Retrieving results...")).Return() diff --git a/internal/analysis/mocks/analysis.go b/internal/analysis/mocks/analysis.go index 9f7a796c..0c19ebd8 100644 --- a/internal/analysis/mocks/analysis.go +++ b/internal/analysis/mocks/analysis.go @@ -55,9 +55,9 @@ func (mr *MockAnalysisOrchestratorMockRecorder) RunLegacyTest(ctx, bundleHash, s } // RunTest mocks base method. -func (m *MockAnalysisOrchestrator) RunTest(ctx context.Context, orgId string, b bundle.Bundle, target scan.Target, reportingOptions analysis.AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) { +func (m *MockAnalysisOrchestrator) RunTest(ctx context.Context, orgId string, b bundle.Bundle, revisionId *string, target scan.Target, reportingOptions analysis.AnalysisConfig) (*sarif.SarifResponse, *scan.ResultMetaData, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RunTest", ctx, orgId, b, target, reportingOptions) + ret := m.ctrl.Call(m, "RunTest", ctx, orgId, b, revisionId, target, reportingOptions) ret0, _ := ret[0].(*sarif.SarifResponse) ret1, _ := ret[1].(*scan.ResultMetaData) ret2, _ := ret[2].(error) @@ -65,9 +65,9 @@ func (m *MockAnalysisOrchestrator) RunTest(ctx context.Context, orgId string, b } // RunTest indicates an expected call of RunTest. -func (mr *MockAnalysisOrchestratorMockRecorder) RunTest(ctx, orgId, b, target, reportingOptions interface{}) *gomock.Call { +func (mr *MockAnalysisOrchestratorMockRecorder) RunTest(ctx, orgId, b, revisionId, target, reportingOptions interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunTest", reflect.TypeOf((*MockAnalysisOrchestrator)(nil).RunTest), ctx, orgId, b, target, reportingOptions) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunTest", reflect.TypeOf((*MockAnalysisOrchestrator)(nil).RunTest), ctx, orgId, b, revisionId, target, reportingOptions) } // RunTestRemote mocks base method. diff --git a/internal/commands/code_workflow/native_workflow.go b/internal/commands/code_workflow/native_workflow.go index a94c05a9..5983dd9b 100644 --- a/internal/commands/code_workflow/native_workflow.go +++ b/internal/commands/code_workflow/native_workflow.go @@ -42,6 +42,8 @@ const ( ConfigurationSastSettings = "internal_sast_settings" ConfigurationSlceEnabled = "internal_snyk_scle_enabled" + ConfigurationUploadToFileUploadApi = "internal_upload_to_fua" + MetadataBundleHash = "Snyk-Bundle-Hash" ) @@ -293,6 +295,10 @@ func defaultAnalyzeFunction(ctx context.Context, path string, httpClientFunc fun return analyzeWithLegacyEngine(ctx, codeScanner, requestId, target, files, changedFiles, logger) } + if config.GetBool(ConfigurationUploadToFileUploadApi) { + analysisOptions = append(analysisOptions, codeclient.WithUploadToFileUploadApi()) + } + result, bundleHash, resultMetaData, err = codeScanner.UploadAndAnalyzeWithOptions(ctx, requestId, target, files, changedFiles, analysisOptions...) return result, bundleHash, resultMetaData, err diff --git a/internal/commands/code_workflow/native_workflow_test.go b/internal/commands/code_workflow/native_workflow_test.go index 9345d161..46a19e3f 100644 --- a/internal/commands/code_workflow/native_workflow_test.go +++ b/internal/commands/code_workflow/native_workflow_test.go @@ -2,6 +2,7 @@ package code_workflow import ( "context" + "fmt" "net/http" "net/http/httptest" "os" @@ -10,6 +11,7 @@ import ( "sync" "testing" + "github.com/google/uuid" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,7 +55,7 @@ func Test_defaultAnalyzeFunction_usesLocalEngineLegacyEndpoints(t *testing.T) { requests = append(requests, r.Method+" "+r.URL.Path) mu.Unlock() - assert.Equal(t, "test-org", r.Header.Get("snyk-org-name")) + assert.Equal(t, "4a72d1db-b465-4764-99e1-ecedad03b06a", r.Header.Get("snyk-org-name")) switch { case r.Method == http.MethodGet && r.URL.Path == "/filters": @@ -86,7 +88,7 @@ func Test_defaultAnalyzeFunction_usesLocalEngineLegacyEndpoints(t *testing.T) { config := configuration.NewWithOpts() config.Set(configuration.API_URL, "https://api.snyk.io") - config.Set(configuration.ORGANIZATION, "test-org") + config.Set(configuration.ORGANIZATION, "4a72d1db-b465-4764-99e1-ecedad03b06a") config.Set(configuration.MAX_THREADS, 1) config.Set(configuration.FLAG_REMOTE_REPO_URL, "https://github.com/snyk/nodejs-goof") config.Set(ConfigurationSlceEnabled, true) @@ -123,6 +125,74 @@ func Test_defaultAnalyzeFunction_usesLocalEngineLegacyEndpoints(t *testing.T) { }, requests) } +func Test_defaultAnalyzeFunction_usesFileUploadApi(t *testing.T) { + logger := zerolog.Nop() + revID := uuid.NewString() + + var ( + mu sync.Mutex + filtersHit, createHit, uploadHit, sealHit, testHit bool + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + switch { + case r.Method == http.MethodGet && r.URL.Path == "/filters": + filtersHit = true + _, _ = w.Write([]byte(`{"configFiles":[],"extensions":[".js"]}`)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files"): + uploadHit = true + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/upload_revisions"): + createHit = true + w.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(w, `{"data":{"id":%q,"type":"upload_revision","attributes":{"revision_type":"snapshot","sealed":false}}}`, revID) + case r.Method == http.MethodPatch && strings.Contains(r.URL.Path, "/upload_revisions/"): + sealHit = true + _, _ = fmt.Fprintf(w, `{"data":{"id":%q,"type":"upload_revision","attributes":{"revision_type":"snapshot","sealed":true}}}`, revID) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/tests"): + // The test service is invoked against the uploaded revision; its happy + // path is covered by the analysis package tests, so it is stubbed here. + testHit = true + w.WriteHeader(http.StatusBadRequest) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + path := t.TempDir() + writeFile(t, filepath.Join(path, "app.js")) + + config := configuration.NewWithOpts() + config.Set(configuration.API_URL, server.URL) + config.Set(configuration.ORGANIZATION, uuid.NewString()) + config.Set(configuration.MAX_THREADS, 1) + config.Set(configuration.FLAG_REMOTE_REPO_URL, "https://github.com/snyk/nodejs-goof") + config.Set(ConfigurationUploadToFileUploadApi, true) + + result, _, _, err := defaultAnalyzeFunction( + context.Background(), + path, + func() *http.Client { return server.Client() }, + &logger, + config, + ui.DefaultUi(), + ) + + require.NoError(t, err) + assert.Nil(t, result) + + mu.Lock() + defer mu.Unlock() + assert.True(t, filtersHit) + assert.True(t, createHit) + assert.True(t, uploadHit) + assert.True(t, sealHit) + assert.True(t, testHit) +} + type fakeLegacyCodeScanner struct { called bool shardKey string diff --git a/pkg/code/code.go b/pkg/code/code.go index 083d668e..54548f36 100644 --- a/pkg/code/code.go +++ b/pkg/code/code.go @@ -202,6 +202,9 @@ func Init(engine workflow.Engine) error { engine.GetConfiguration().AddDefaultValue(code_workflow.ConfigurationTestFLowName, configuration.StandardDefaultValueFunction("cli_test")) config_utils.AddFeatureFlagToConfig(engine, configuration.FF_CODE_CONSISTENT_IGNORES, "snykCodeConsistentIgnores") config_utils.AddFeatureFlagToConfig(engine, configuration.FF_CODE_NATIVE_IMPLEMENTATION, FfNameNativeImplementation) + config_utils.AddFeatureFlagsToConfig(engine, map[string]string{ + code_workflow.ConfigurationUploadToFileUploadApi: "code-cli-upload-file-content-to-fua", + }) return err } diff --git a/scan.go b/scan.go index 5f1935ce..02ba67e8 100644 --- a/scan.go +++ b/scan.go @@ -20,11 +20,13 @@ package codeclient import ( "context" "fmt" + "net/http" "time" "github.com/google/uuid" "github.com/pkg/errors" "github.com/rs/zerolog" + "github.com/snyk/go-application-framework/pkg/apiclients/fileupload" "github.com/snyk/code-client-go/bundle" "github.com/snyk/code-client-go/config" @@ -32,6 +34,7 @@ import ( "github.com/snyk/code-client-go/internal/analysis" testModels "github.com/snyk/code-client-go/internal/api/test/2025-04-07/models" "github.com/snyk/code-client-go/internal/deepcode" + "github.com/snyk/code-client-go/internal/uploadrevision" "github.com/snyk/code-client-go/observability" "github.com/snyk/code-client-go/sarif" "github.com/snyk/code-client-go/scan" @@ -41,6 +44,7 @@ type codeScanner struct { httpClient codeClientHTTP.HTTPClient bundleManager bundle.BundleManager analysisOrchestrator analysis.AnalysisOrchestrator + uploadRevision uploadrevision.UploadRevision instrumentor observability.Instrumentor errorReporter observability.ErrorReporter trackerFactory scan.TrackerFactory @@ -49,6 +53,14 @@ type codeScanner struct { resultTypes testModels.ResultType } +type httpClientRoundTripper struct { + httpClient codeClientHTTP.HTTPClient +} + +func (rt httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return rt.httpClient.Do(req) +} + type CodeScanner interface { Upload( ctx context.Context, @@ -143,6 +155,12 @@ func ReportRemoteTest(projectId uuid.UUID, commitId string) AnalysisOption { } } +func WithUploadToFileUploadApi() AnalysisOption { + return func(c *analysis.AnalysisConfig) { + c.UploadFileContentToFileUploadApi = true + } +} + // NewCodeScanner creates a Code Scanner which can be used to trigger Snyk Code on a folder. func NewCodeScanner( config config.Config, @@ -183,6 +201,14 @@ func NewCodeScanner( ) scanner.analysisOrchestrator = analysisOrchestrator + orgID := uuid.MustParse(scanner.config.Organization()) + scanner.uploadRevision = uploadrevision.NewUploadRevision( + &http.Client{Transport: httpClientRoundTripper{httpClient: httpClient}}, + fileupload.Config{BaseURL: scanner.config.SnykApi(), OrgID: orgID}, + deepcodeClient, + scanner.logger, + ) + return scanner } @@ -338,25 +364,40 @@ func (c *codeScanner) UploadAndAnalyzeWithOptions( changedFiles map[string]bool, options ...AnalysisOption, ) (*sarif.SarifResponse, string, *scan.ResultMetaData, error) { - uploadedBundle, err := c.Upload(ctx, requestId, target, files, changedFiles) - - if err != nil || uploadedBundle == nil || uploadedBundle.GetBundleHash() == "" { - c.logger.Debug().Msg("empty bundle, no Snyk Code analysis") - return nil, "", nil, err - } - cfg := analysis.AnalysisConfig{} for _, opt := range options { opt(&cfg) } - response, metadata, err := c.analysisOrchestrator.RunTest(ctx, c.config.Organization(), uploadedBundle, target, cfg) + var uploadedBundle bundle.Bundle + var revisionId *string + var scanIdentifier string + var err error + if cfg.UploadFileContentToFileUploadApi { + revision, uploadErr := c.uploadRevision.Upload(ctx, requestId, target, files) + if uploadErr != nil { + c.logger.Debug().Msg("upload to file-upload-api failed") + return nil, "", nil, uploadErr + } + revisionString := string(revision) + revisionId = &revisionString + scanIdentifier = revisionString + } else { + uploadedBundle, err = c.Upload(ctx, requestId, target, files, changedFiles) + if err != nil || uploadedBundle == nil || uploadedBundle.GetBundleHash() == "" { + c.logger.Debug().Msg("empty bundle, no Snyk Code analysis") + return nil, "", nil, err + } + scanIdentifier = uploadedBundle.GetBundleHash() + } + + response, metadata, err := c.analysisOrchestrator.RunTest(ctx, c.config.Organization(), uploadedBundle, revisionId, target, cfg) err = c.checkCancellationOrLogError(ctx, target.GetPath(), err, "error running analysis...") if err != nil { return nil, "", nil, err } - return response, uploadedBundle.GetBundleHash(), metadata, err + return response, scanIdentifier, metadata, err } func (c *codeScanner) AnalyzeRemote(ctx context.Context, options ...AnalysisOption) (*sarif.SarifResponse, *scan.ResultMetaData, error) { diff --git a/scan_test.go b/scan_test.go index 02872130..5d79f6a0 100644 --- a/scan_test.go +++ b/scan_test.go @@ -137,6 +137,7 @@ func Test_UploadAndAnalyze(t *testing.T) { gomock.Any(), testOrgId, gomock.Any(), + gomock.Nil(), gomock.Any(), gomock.Any(), ).Return(&sarif.SarifResponse{Status: "COMPLETE"}, &scan.ResultMetaData{}, nil) @@ -174,6 +175,7 @@ func Test_UploadAndAnalyze(t *testing.T) { gomock.Any(), testOrgId, gomock.Any(), + gomock.Nil(), gomock.Any(), gomock.Any(), ).Return(&sarif.SarifResponse{Status: "COMPLETE"}, &scan.ResultMetaData{}, nil) @@ -201,7 +203,8 @@ func TestAnalyzeRemote(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockConfig := confMocks.NewMockConfig(ctrl) - mockConfig.EXPECT().Organization().AnyTimes().Return("mockOrgId") + mockConfig.EXPECT().Organization().AnyTimes().Return("4a72d1db-b465-4764-99e1-ecedad03b06a") + mockConfig.EXPECT().SnykApi().AnyTimes().Return("") mockHTTPClient := httpmocks.NewMockHTTPClient(ctrl) mockInstrumentor := mocks.NewMockInstrumentor(ctrl) @@ -226,7 +229,7 @@ func TestAnalyzeRemote(t *testing.T) { t.Run("returns valid response", func(t *testing.T) { mockAnalysisOrchestrator.EXPECT().RunTestRemote( gomock.Any(), - "mockOrgId", + "4a72d1db-b465-4764-99e1-ecedad03b06a", gomock.Any(), ).Return(&sarif.SarifResponse{Status: "COMPLETE"}, &scan.ResultMetaData{}, nil) From 94084fd6afb43fb3de8fd2d04721a1de146cf02b Mon Sep 17 00:00:00 2001 From: Florin Mirosnicencu Date: Tue, 28 Jul 2026 15:36:04 +0100 Subject: [PATCH 5/6] chore: update test-api spec to version that supports commitId and branch name for upload_revision as testInput --- internal/api/test/2025-04-07/client.go | 197 ++++++++++++++++ internal/api/test/2025-04-07/models/tests.go | 214 ++++++++++++++++-- .../api/test/2025-04-07/models/tests.yaml | 178 ++++++++++++++- internal/api/test/2025-04-07/spec.yaml | 49 ++++ scripts/download-test-api.py | 2 +- 5 files changed, 610 insertions(+), 30 deletions(-) diff --git a/internal/api/test/2025-04-07/client.go b/internal/api/test/2025-04-07/client.go index a2c2d04f..3edbdc5d 100644 --- a/internal/api/test/2025-04-07/client.go +++ b/internal/api/test/2025-04-07/client.go @@ -25,6 +25,12 @@ type CreateTestParams struct { Version externalRef0.Version `form:"version" json:"version"` } +// CreateBatchTestsParams defines parameters for CreateBatchTests. +type CreateBatchTestsParams struct { + // Version The requested version of the endpoint to process the request + Version externalRef0.Version `form:"version" json:"version"` +} + // GetTestResultParams defines parameters for GetTestResult. type GetTestResultParams struct { // Version The requested version of the endpoint to process the request @@ -55,6 +61,9 @@ type GetTestInitialConfigurationParams struct { // CreateTestApplicationVndAPIPlusJSONRequestBody defines body for CreateTest for application/vnd.api+json ContentType. type CreateTestApplicationVndAPIPlusJSONRequestBody = externalRef1.CreateTestRequestBody +// CreateBatchTestsJSONRequestBody defines body for CreateBatchTests for application/json ContentType. +type CreateBatchTestsJSONRequestBody = externalRef1.CreateBatchTestRequestBody + // RequestEditorFn is the function signature for the RequestEditor callback function type RequestEditorFn func(ctx context.Context, req *http.Request) error @@ -133,6 +142,11 @@ type ClientInterface interface { CreateTestWithApplicationVndAPIPlusJSONBody(ctx context.Context, orgId externalRef2.OrgId, params *CreateTestParams, body CreateTestApplicationVndAPIPlusJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // CreateBatchTestsWithBody request with any body + CreateBatchTestsWithBody(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + CreateBatchTests(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, body CreateBatchTestsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetTestResult request GetTestResult(ctx context.Context, orgId externalRef2.OrgId, testId externalRef2.TestId, params *GetTestResultParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -167,6 +181,30 @@ func (c *Client) CreateTestWithApplicationVndAPIPlusJSONBody(ctx context.Context return c.Client.Do(req) } +func (c *Client) CreateBatchTestsWithBody(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateBatchTestsRequestWithBody(c.Server, orgId, params, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreateBatchTests(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, body CreateBatchTestsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateBatchTestsRequest(c.Server, orgId, params, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetTestResult(ctx context.Context, orgId externalRef2.OrgId, testId externalRef2.TestId, params *GetTestResultParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetTestResultRequest(c.Server, orgId, testId, params) if err != nil { @@ -268,6 +306,71 @@ func NewCreateTestRequestWithBody(server string, orgId externalRef2.OrgId, param return req, nil } +// NewCreateBatchTestsRequest calls the generic CreateBatchTests builder with application/json body +func NewCreateBatchTestsRequest(server string, orgId externalRef2.OrgId, params *CreateBatchTestsParams, body CreateBatchTestsJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewCreateBatchTestsRequestWithBody(server, orgId, params, "application/json", bodyReader) +} + +// NewCreateBatchTestsRequestWithBody generates requests for CreateBatchTests with any type of body +func NewCreateBatchTestsRequestWithBody(server string, orgId externalRef2.OrgId, params *CreateBatchTestsParams, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "org_id", runtime.ParamLocationPath, orgId) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/orgs/%s/tests/batch", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "version", runtime.ParamLocationQuery, params.Version); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewGetTestResultRequest generates requests for GetTestResult func NewGetTestResultRequest(server string, orgId externalRef2.OrgId, testId externalRef2.TestId, params *GetTestResultParams) (*http.Request, error) { var err error @@ -541,6 +644,11 @@ type ClientWithResponsesInterface interface { CreateTestWithApplicationVndAPIPlusJSONBodyWithResponse(ctx context.Context, orgId externalRef2.OrgId, params *CreateTestParams, body CreateTestApplicationVndAPIPlusJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateTestResponse, error) + // CreateBatchTestsWithBodyWithResponse request with any body + CreateBatchTestsWithBodyWithResponse(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateBatchTestsResponse, error) + + CreateBatchTestsWithResponse(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, body CreateBatchTestsJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateBatchTestsResponse, error) + // GetTestResultWithResponse request GetTestResultWithResponse(ctx context.Context, orgId externalRef2.OrgId, testId externalRef2.TestId, params *GetTestResultParams, reqEditors ...RequestEditorFn) (*GetTestResultResponse, error) @@ -579,6 +687,31 @@ func (r CreateTestResponse) StatusCode() int { return 0 } +type CreateBatchTestsResponse struct { + Body []byte + HTTPResponse *http.Response + ApplicationvndApiJSON400 *externalRef0.N400 + ApplicationvndApiJSON401 *externalRef0.N401 + ApplicationvndApiJSON403 *externalRef0.N403 + ApplicationvndApiJSON500 *externalRef0.N500 +} + +// Status returns HTTPResponse.Status +func (r CreateBatchTestsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r CreateBatchTestsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetTestResultResponse struct { Body []byte HTTPResponse *http.Response @@ -679,6 +812,23 @@ func (c *ClientWithResponses) CreateTestWithApplicationVndAPIPlusJSONBodyWithRes return ParseCreateTestResponse(rsp) } +// CreateBatchTestsWithBodyWithResponse request with arbitrary body returning *CreateBatchTestsResponse +func (c *ClientWithResponses) CreateBatchTestsWithBodyWithResponse(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateBatchTestsResponse, error) { + rsp, err := c.CreateBatchTestsWithBody(ctx, orgId, params, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreateBatchTestsResponse(rsp) +} + +func (c *ClientWithResponses) CreateBatchTestsWithResponse(ctx context.Context, orgId externalRef2.OrgId, params *CreateBatchTestsParams, body CreateBatchTestsJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateBatchTestsResponse, error) { + rsp, err := c.CreateBatchTests(ctx, orgId, params, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreateBatchTestsResponse(rsp) +} + // GetTestResultWithResponse request returning *GetTestResultResponse func (c *ClientWithResponses) GetTestResultWithResponse(ctx context.Context, orgId externalRef2.OrgId, testId externalRef2.TestId, params *GetTestResultParams, reqEditors ...RequestEditorFn) (*GetTestResultResponse, error) { rsp, err := c.GetTestResult(ctx, orgId, testId, params, reqEditors...) @@ -774,6 +924,53 @@ func ParseCreateTestResponse(rsp *http.Response) (*CreateTestResponse, error) { return response, nil } +// ParseCreateBatchTestsResponse parses an HTTP response from a CreateBatchTestsWithResponse call +func ParseCreateBatchTestsResponse(rsp *http.Response) (*CreateBatchTestsResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &CreateBatchTestsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest externalRef0.N400 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationvndApiJSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest externalRef0.N401 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationvndApiJSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest externalRef0.N403 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationvndApiJSON403 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest externalRef0.N500 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationvndApiJSON500 = &dest + + } + + return response, nil +} + // ParseGetTestResultResponse parses an HTTP response from a GetTestResultWithResponse call func ParseGetTestResultResponse(rsp *http.Response) (*GetTestResultResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/internal/api/test/2025-04-07/models/tests.go b/internal/api/test/2025-04-07/models/tests.go index 1fb2833d..fbb04818 100644 --- a/internal/api/test/2025-04-07/models/tests.go +++ b/internal/api/test/2025-04-07/models/tests.go @@ -13,6 +13,22 @@ import ( externalRef0 "github.com/snyk/code-client-go/internal/api/test/2025-04-07/common" ) +// Defines values for CreateBatchTestAttributesInitiator. +const ( + CreateBatchTestAttributesInitiatorRecurringTest CreateBatchTestAttributesInitiator = "recurring_test" +) + +// Defines values for CreateBatchTestAttributesScanner. +const ( + CreateBatchTestAttributesScannerSca CreateBatchTestAttributesScanner = "sca" + CreateBatchTestAttributesScannerSecrets CreateBatchTestAttributesScanner = "secrets" +) + +// Defines values for CreateBatchTestRequestBodyDataType. +const ( + BatchTest CreateBatchTestRequestBodyDataType = "batch_test" +) + // Defines values for CreateTestRequestBodyDataType. const ( CreateTestRequestBodyDataTypeTest CreateTestRequestBodyDataType = "test" @@ -20,15 +36,47 @@ const ( // Defines values for OutputConfigInitiator. const ( - ApiTest OutputConfigInitiator = "api_test" - AutoImport OutputConfigInitiator = "auto_import" - CliTest OutputConfigInitiator = "cli_test" - EssentialsImport OutputConfigInitiator = "essentials_import" - IdeTest OutputConfigInitiator = "ide_test" - Import OutputConfigInitiator = "import" - ManualTest OutputConfigInitiator = "manual_test" - PrCheck OutputConfigInitiator = "pr_check" - RecurringTest OutputConfigInitiator = "recurring_test" + OutputConfigInitiatorApiImport OutputConfigInitiator = "api_import" + OutputConfigInitiatorApiTest OutputConfigInitiator = "api_test" + OutputConfigInitiatorAutoImport OutputConfigInitiator = "auto_import" + OutputConfigInitiatorCliTest OutputConfigInitiator = "cli_test" + OutputConfigInitiatorContainerImport OutputConfigInitiator = "container_import" + OutputConfigInitiatorEssentialsImport OutputConfigInitiator = "essentials_import" + OutputConfigInitiatorIdeTest OutputConfigInitiator = "ide_test" + OutputConfigInitiatorImport OutputConfigInitiator = "import" + OutputConfigInitiatorManualRetest OutputConfigInitiator = "manual_retest" + OutputConfigInitiatorManualTest OutputConfigInitiator = "manual_test" + OutputConfigInitiatorPrCheck OutputConfigInitiator = "pr_check" + OutputConfigInitiatorPushTest OutputConfigInitiator = "push_test" + OutputConfigInitiatorRecurringTest OutputConfigInitiator = "recurring_test" +) + +// Defines values for OutputConfigProjectBusinessCriticality. +const ( + Critical OutputConfigProjectBusinessCriticality = "critical" + High OutputConfigProjectBusinessCriticality = "high" + Low OutputConfigProjectBusinessCriticality = "low" + Medium OutputConfigProjectBusinessCriticality = "medium" +) + +// Defines values for OutputConfigProjectEnvironment. +const ( + Backend OutputConfigProjectEnvironment = "backend" + Distributed OutputConfigProjectEnvironment = "distributed" + External OutputConfigProjectEnvironment = "external" + Frontend OutputConfigProjectEnvironment = "frontend" + Hosted OutputConfigProjectEnvironment = "hosted" + Internal OutputConfigProjectEnvironment = "internal" + Mobile OutputConfigProjectEnvironment = "mobile" + Onprem OutputConfigProjectEnvironment = "onprem" + Saas OutputConfigProjectEnvironment = "saas" +) + +// Defines values for OutputConfigProjectLifecycle. +const ( + Development OutputConfigProjectLifecycle = "development" + Production OutputConfigProjectLifecycle = "production" + Sandbox OutputConfigProjectLifecycle = "sandbox" ) // Defines values for ResultType. @@ -39,10 +87,12 @@ const ( // Defines values for ScanConfigScanners. const ( - LegacyScanners ScanConfigScanners = "legacy_scanners" - Sast ScanConfigScanners = "sast" - Sca ScanConfigScanners = "sca" - Secrets ScanConfigScanners = "secrets" + ScanConfigScannersAllScanners ScanConfigScanners = "all_scanners" + ScanConfigScannersContainer ScanConfigScanners = "container" + ScanConfigScannersLegacyScanners ScanConfigScanners = "legacy_scanners" + ScanConfigScannersSast ScanConfigScanners = "sast" + ScanConfigScannersSca ScanConfigScanners = "sca" + ScanConfigScannersSecrets ScanConfigScanners = "secrets" ) // Defines values for TestAcceptedStateStatus. @@ -76,6 +126,11 @@ const ( TestInitialConfigurationResponseDataTypeTest TestInitialConfigurationResponseDataType = "test" ) +// Defines values for TestInputContainerAnalysisRevisionType. +const ( + ContainerAnalysisRevision TestInputContainerAnalysisRevisionType = "container_analysis_revision" +) + // Defines values for TestInputDiffTargetType. const ( DiffScmTarget TestInputDiffTargetType = "diff_scm_target" @@ -146,6 +201,44 @@ const ( TestResultDataTypeTest TestResultDataType = "test" ) +// BatchInput Reference to the batch file in object storage +type BatchInput struct { + // Bucket Name of the bucket where the batch file is stored + Bucket string `json:"bucket"` + + // Path Path to the batch file within the bucket + Path string `json:"path"` +} + +// CreateBatchTestAttributes defines model for CreateBatchTestAttributes. +type CreateBatchTestAttributes struct { + // Initiator The type of test flow that initiated the batch + Initiator CreateBatchTestAttributesInitiator `json:"initiator"` + + // Input Reference to the batch file in object storage + Input BatchInput `json:"input"` + + // Scanner The scanner type to use for all tests in the batch + Scanner CreateBatchTestAttributesScanner `json:"scanner"` +} + +// CreateBatchTestAttributesInitiator The type of test flow that initiated the batch +type CreateBatchTestAttributesInitiator string + +// CreateBatchTestAttributesScanner The scanner type to use for all tests in the batch +type CreateBatchTestAttributesScanner string + +// CreateBatchTestRequestBody defines model for CreateBatchTestRequestBody. +type CreateBatchTestRequestBody struct { + Data struct { + Attributes CreateBatchTestAttributes `json:"attributes"` + Type CreateBatchTestRequestBodyDataType `json:"type"` + } `json:"data"` +} + +// CreateBatchTestRequestBodyDataType defines model for CreateBatchTestRequestBody.Data.Type. +type CreateBatchTestRequestBodyDataType string + // CreateTestRequestBody defines model for CreateTestRequestBody. type CreateTestRequestBody struct { Data struct { @@ -162,21 +255,40 @@ type CreateTestRequestBodyDataType string // OutputConfig defines model for OutputConfig. type OutputConfig struct { + // AssetId The id of the asset to associate with the test. + AssetId *openapi_types.UUID `json:"asset_id,omitempty"` + + // AssetName The name used for the asset. + AssetName *string `json:"asset_name,omitempty"` + // Initiator The type of test flow or system that initiated the test Initiator *OutputConfigInitiator `json:"initiator,omitempty"` - // Label Arbitrary value up to the user + // Label Arbitrary value up to the user. Deprecated: use `labels` instead. + // Deprecated: Label *string `json:"label,omitempty"` // Labels Arbitrary string values up to the user, up to 10 keys Labels *map[string]string `json:"labels,omitempty"` + // Monitor Determines if a project should be created (true) or not (false). + Monitor *bool `json:"monitor,omitempty"` + // Origin The source control management system or platform origin - Origin *string `json:"origin,omitempty"` - ProjectId *openapi_types.UUID `json:"project_id,omitempty"` - ProjectName *string `json:"project_name,omitempty"` + Origin *string `json:"origin,omitempty"` + + // ProjectBusinessCriticality Project business criticality attributes + ProjectBusinessCriticality *[]OutputConfigProjectBusinessCriticality `json:"project_business_criticality,omitempty"` + + // ProjectEnvironment Project environment attributes + ProjectEnvironment *[]OutputConfigProjectEnvironment `json:"project_environment,omitempty"` + ProjectId *openapi_types.UUID `json:"project_id,omitempty"` - // ProjectTags Project tags to assign when reporting. Each entry is a key=value string. + // ProjectLifecycle Project lifecycle attributes + ProjectLifecycle *[]OutputConfigProjectLifecycle `json:"project_lifecycle,omitempty"` + ProjectName *string `json:"project_name,omitempty"` + + // ProjectTags Project tags as an array of key=value pairs. Each tag must be in the format 'key=value'. Example: ["department=finance", "team=alpha"] ProjectTags *[]string `json:"project_tags,omitempty"` // Report Determines if the test is stateless (false) or stateful (true) @@ -190,6 +302,15 @@ type OutputConfig struct { // OutputConfigInitiator The type of test flow or system that initiated the test type OutputConfigInitiator string +// OutputConfigProjectBusinessCriticality defines model for OutputConfig.ProjectBusinessCriticality. +type OutputConfigProjectBusinessCriticality string + +// OutputConfigProjectEnvironment defines model for OutputConfig.ProjectEnvironment. +type OutputConfigProjectEnvironment string + +// OutputConfigProjectLifecycle defines model for OutputConfig.ProjectLifecycle. +type OutputConfigProjectLifecycle string + // ResultType defines model for ResultType. type ResultType string @@ -198,10 +319,16 @@ type ScanConfig struct { // ExclusionGlobs A list of file paths and directories to exclude from the workspace. If empty default exclusion globs apply, according to the coordinate type. ExclusionGlobs *[]string `json:"exclusion_globs,omitempty"` + // InclusionGlobs A list of file paths to include in the workspace. If empty, all files are included (subject to exclusion globs). + InclusionGlobs *[]string `json:"inclusion_globs,omitempty"` + // LimitTestToFiles A list of file paths to use in the scan. If empty the whole target is tested. - LimitTestToFiles *[]string `json:"limit_test_to_files,omitempty"` - ResultType *ResultType `json:"result_type,omitempty"` - Scanners *[]ScanConfigScanners `json:"scanners,omitempty"` + LimitTestToFiles *[]string `json:"limit_test_to_files,omitempty"` + + // LimitTestToProjects A list of project IDs to limit the test to. If empty, all projects are tested. + LimitTestToProjects *[]openapi_types.UUID `json:"limit_test_to_projects,omitempty"` + ResultType *ResultType `json:"result_type,omitempty"` + Scanners *[]ScanConfigScanners `json:"scanners,omitempty"` } // ScanConfigScanners defines model for ScanConfig.Scanners. @@ -300,6 +427,16 @@ type TestInitialConfigurationResponse struct { // TestInitialConfigurationResponseDataType defines model for TestInitialConfigurationResponse.Data.Type. type TestInitialConfigurationResponseDataType string +// TestInputContainerAnalysisRevision defines model for TestInputContainerAnalysisRevision. +type TestInputContainerAnalysisRevision struct { + // RevisionId A Snyk revision id referencing the uploaded container analysis + RevisionId string `json:"revision_id"` + Type TestInputContainerAnalysisRevisionType `json:"type"` +} + +// TestInputContainerAnalysisRevisionType defines model for TestInputContainerAnalysisRevision.Type. +type TestInputContainerAnalysisRevisionType string + // TestInputDiffTarget defines model for TestInputDiffTarget. type TestInputDiffTarget struct { // BaseVersion SHA of the last commit existing in base branch @@ -308,6 +445,9 @@ type TestInputDiffTarget struct { // HeadVersion SHA of the commit to be tested HeadVersion string `json:"head_version"` + // IntegrationId A Snyk integration id that has access to the target's repository. + IntegrationId *openapi_types.UUID `json:"integration_id,omitempty"` + // TargetId Id of the target to be tested TargetId openapi_types.UUID `json:"target_id"` Type TestInputDiffTargetType `json:"type"` @@ -492,6 +632,12 @@ type TestInputTargetType string type TestInputUploadRevision struct { // Metadata Metadata of the input to be tested Metadata *struct { + // Branch The name of the branch being tested + Branch *string `json:"branch,omitempty"` + + // CommitId SHA of the commit being tested + CommitId *string `json:"commit_id,omitempty"` + // LocalFilePath This can be a file path or a folder id for IDE LocalFilePath *string `json:"local_file_path,omitempty"` @@ -866,6 +1012,32 @@ func (t *TestAttributes_Input) MergeTestInputSBOMSourceRevisions(v TestInputSBOM return err } +// AsTestInputContainerAnalysisRevision returns the union data inside the TestAttributes_Input as a TestInputContainerAnalysisRevision +func (t TestAttributes_Input) AsTestInputContainerAnalysisRevision() (TestInputContainerAnalysisRevision, error) { + var body TestInputContainerAnalysisRevision + err := json.Unmarshal(t.union, &body) + return body, err +} + +// FromTestInputContainerAnalysisRevision overwrites any union data inside the TestAttributes_Input as the provided TestInputContainerAnalysisRevision +func (t *TestAttributes_Input) FromTestInputContainerAnalysisRevision(v TestInputContainerAnalysisRevision) error { + b, err := json.Marshal(v) + t.union = b + return err +} + +// MergeTestInputContainerAnalysisRevision performs a merge with any union data inside the TestAttributes_Input, using the provided TestInputContainerAnalysisRevision +func (t *TestAttributes_Input) MergeTestInputContainerAnalysisRevision(v TestInputContainerAnalysisRevision) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + + merged, err := runtime.JSONMerge(t.union, b) + t.union = merged + return err +} + func (t TestAttributes_Input) MarshalJSON() ([]byte, error) { b, err := t.union.MarshalJSON() return b, err diff --git a/internal/api/test/2025-04-07/models/tests.yaml b/internal/api/test/2025-04-07/models/tests.yaml index 624a1797..fa4f9cfa 100644 --- a/internal/api/test/2025-04-07/models/tests.yaml +++ b/internal/api/test/2025-04-07/models/tests.yaml @@ -13,10 +13,12 @@ components: items: type: string enum: + - all_scanners - sast - sca - legacy_scanners - secrets + - container result_type: $ref: '#/components/schemas/ResultType' limit_test_to_files: @@ -31,6 +33,19 @@ components: type: array items: type: string + inclusion_globs: + description: A list of file paths to include in the workspace. If empty, + all files are included (subject to exclusion globs). + type: array + items: + type: string + limit_test_to_projects: + description: A list of project IDs to limit the test to. If empty, all projects + are tested. + type: array + items: + type: string + format: uuid additionalProperties: false OutputConfig: type: object @@ -38,6 +53,9 @@ components: report: type: boolean description: Determines if the test is stateless (false) or stateful (true) + monitor: + type: boolean + description: Determines if a project should be created (true) or not (false). project_name: type: string pattern: ^[?#@&+=%a-zA-Z0-9_.~\/\\:\s-]+$ @@ -47,6 +65,14 @@ components: target_name: type: string pattern: ^[?#@&+=%a-zA-Z0-9_.~\/\\:\s-]+$ + asset_name: + type: string + description: The name used for the asset. + pattern: ^[?#@&+=%a-zA-Z0-9_.~\/\\:\s-]+$ + asset_id: + type: string + format: uuid + description: The id of the asset to associate with the test. target_reference: type: string description: A reference for the target's version - can be a git branch @@ -54,7 +80,8 @@ components: example: main label: type: string - description: Arbitrary value up to the user + description: 'Arbitrary value up to the user. Deprecated: use `labels` instead.' + deprecated: true pattern: ^[:/?#@&+=%a-zA-Z0-9_.~-]+$ labels: type: object @@ -63,11 +90,6 @@ components: additionalProperties: type: string pattern: ^[:/?#@&+=%a-zA-Z0-9_.~-]+$ - project_tags: - type: array - description: Project tags to assign when reporting. Each entry is a key=value string. - items: - type: string origin: type: string description: The source control management system or platform origin @@ -86,7 +108,62 @@ components: - import - auto_import - essentials_import + - api_import + - push_test + - manual_retest + - container_import example: api_test + project_environment: + type: array + description: Project environment attributes + items: + type: string + enum: + - frontend + - backend + - internal + - external + - mobile + - saas + - onprem + - hosted + - distributed + example: + - frontend + - backend + project_business_criticality: + type: array + description: Project business criticality attributes + items: + type: string + enum: + - critical + - high + - medium + - low + example: + - high + project_lifecycle: + type: array + description: Project lifecycle attributes + items: + type: string + enum: + - production + - development + - sandbox + example: + - production + project_tags: + type: array + description: 'Project tags as an array of key=value pairs. Each tag must + be in the format ''key=value''. Example: ["department=finance", "team=alpha"]' + items: + type: string + pattern: ^[a-zA-Z0-9_-]{1,30}=[:/?#@&+=%a-zA-Z0-9_.~-]{1,256}$ + example: + - department=finance + - team=alpha additionalProperties: false TestConfiguration: type: object @@ -113,6 +190,7 @@ components: - $ref: '#/components/schemas/TestInputUploadRevision' - $ref: '#/components/schemas/TestInputSBOMRevision' - $ref: '#/components/schemas/TestInputSBOMSourceRevisions' + - $ref: '#/components/schemas/TestInputContainerAnalysisRevision' configuration: $ref: '#/components/schemas/TestConfiguration' additionalProperties: false @@ -365,8 +443,8 @@ components: repo_url: description: A repository url for which a test will run. type: string - format: uri - example: https://github/com/snyk/goof + maxLength: 1000 + example: https://github.com/snyk/goof revision: description: SHA of the commit to be tested. type: string @@ -389,6 +467,11 @@ components: example: 275af21f-e92b-40aa-8604-ef9b00c9bd8d format: uuid type: string + integration_id: + description: A Snyk integration id that has access to the target's repository. + type: string + format: uuid + example: 275af21f-e92b-40aa-8604-ef9b00c9bd8d base_version: description: SHA of the last commit existing in base branch type: string @@ -499,6 +582,30 @@ components: git@github.com:test-repo/test-repo.git ' + commit_id: + description: SHA of the commit being tested + type: string + example: 024db817148169e6ca9e7f33408cca01002c1dce + branch: + type: string + maxLength: 1000 + description: The name of the branch being tested + required: + - type + - revision_id + TestInputContainerAnalysisRevision: + type: object + properties: + type: + type: string + enum: + - container_analysis_revision + example: container_analysis_revision + revision_id: + description: A Snyk revision id referencing the uploaded container analysis + type: string + maxLength: 100 + example: 275af21f-e92b-40aa-8604-ef9b00c9bd8d required: - type - revision_id @@ -708,3 +815,58 @@ components: required: - created_at - status + CreateBatchTestRequestBody: + type: object + properties: + data: + type: object + properties: + type: + type: string + enum: + - batch_test + example: batch_test + attributes: + $ref: '#/components/schemas/CreateBatchTestAttributes' + required: + - type + - attributes + required: + - data + CreateBatchTestAttributes: + type: object + properties: + scanner: + type: string + enum: + - secrets + - sca + description: The scanner type to use for all tests in the batch + example: secrets + input: + $ref: '#/components/schemas/BatchInput' + initiator: + type: string + description: The type of test flow that initiated the batch + enum: + - recurring_test + example: recurring_test + required: + - scanner + - input + - initiator + BatchInput: + type: object + description: Reference to the batch file in object storage + properties: + bucket: + type: string + description: Name of the bucket where the batch file is stored + example: my-batch-bucket + path: + type: string + description: Path to the batch file within the bucket + example: batches/secrets-2024-01-01.json + required: + - bucket + - path diff --git a/internal/api/test/2025-04-07/spec.yaml b/internal/api/test/2025-04-07/spec.yaml index bbc5d305..1f3ca9a4 100644 --- a/internal/api/test/2025-04-07/spec.yaml +++ b/internal/api/test/2025-04-07/spec.yaml @@ -311,3 +311,52 @@ paths: "403": { $ref: "./common/common.yaml#/components/responses/403" } "404": { $ref: "./common/common.yaml#/components/responses/404" } "500": { $ref: "./common/common.yaml#/components/responses/500" } + /orgs/{org_id}/tests/batch: + post: + summary: Create multiple tests from a batch of targets + operationId: createBatchTests + description: Creates multiple tests for the specified targets. + x-cerberus: + authentication: + strategies: + - SharedSecret: + env: AUTHN_SHARED_SECRET + enableAccessAudit: true + tags: + - Test + parameters: + - { $ref: "./common/common.yaml#/components/parameters/Version" } + - { $ref: "./parameters/orgs.yaml#/components/parameters/OrgId" } + requestBody: + description: Batch of targets to test + content: + application/json: + schema: + { $ref: "./models/tests.yaml#/components/schemas/CreateBatchTestRequestBody" } + responses: + "204": + description: Batch tests created successfully + headers: + snyk-version-requested: + { + $ref: "./common/common.yaml#/components/headers/VersionRequestedResponseHeader", + } + snyk-version-served: + { + $ref: "./common/common.yaml#/components/headers/VersionServedResponseHeader", + } + snyk-request-id: + { + $ref: "./common/common.yaml#/components/headers/RequestIdResponseHeader", + } + snyk-version-lifecycle-stage: + { + $ref: "./common/common.yaml#/components/headers/VersionStageResponseHeader", + } + deprecation: + { $ref: "./common/common.yaml#/components/headers/DeprecationHeader" } + sunset: { $ref: "./common/common.yaml#/components/headers/SunsetHeader" } + "400": { $ref: "./common/common.yaml#/components/responses/400" } + "401": { $ref: "./common/common.yaml#/components/responses/401" } + "403": { $ref: "./common/common.yaml#/components/responses/403" } + "500": { $ref: "./common/common.yaml#/components/responses/500" } diff --git a/scripts/download-test-api.py b/scripts/download-test-api.py index b8129ced..118d3940 100755 --- a/scripts/download-test-api.py +++ b/scripts/download-test-api.py @@ -11,7 +11,7 @@ from utils import replaceInFile API_VERSION = "2025-04-07" -COMMIT_SHA = "7d92349ab7315554fe5e59fb3381a0babd34e628" +COMMIT_SHA = "d78b3de859e07f1a49b5627efe7bfe03316a7f28" SERVICE = "test-service" FOLDER = "test" BASELOCALDIR = f"./internal/api/{FOLDER}/{API_VERSION}" From db59a23af27fd027f733e6cb510c1dc40330648b Mon Sep 17 00:00:00 2001 From: Florin Mirosnicencu Date: Tue, 28 Jul 2026 15:37:01 +0100 Subject: [PATCH 6/6] fix: send commitId and branch name with uploadRevision as test input to allow CLI localReport mode to continue working. --- internal/analysis/analysis.go | 2 +- internal/analysis/analysis_test.go | 16 +++++++++++++--- internal/api/test/2025-04-07/helper.go | 6 ++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/analysis/analysis.go b/internal/analysis/analysis.go index d557030c..28c12400 100644 --- a/internal/analysis/analysis.go +++ b/internal/analysis/analysis.go @@ -260,7 +260,7 @@ func (a *analysisOrchestrator) RunTest(ctx context.Context, orgId string, b bund case b != nil: input = testApi.WithInputBundle(b.GetBundleHash(), target.GetPath(), repoUrl, b.GetLimitToFiles(), commitId, branchName) case revisionId != nil: - input = testApi.WithInputUploadRevision(*revisionId, target.GetPath(), repoUrl) + input = testApi.WithInputUploadRevision(*revisionId, target.GetPath(), repoUrl, commitId, branchName) default: return nil, nil, ErrMissingTestInput } diff --git a/internal/analysis/analysis_test.go b/internal/analysis/analysis_test.go index 915a1e69..a29f78b9 100644 --- a/internal/analysis/analysis_test.go +++ b/internal/analysis/analysis_test.go @@ -487,7 +487,7 @@ func TestAnalysis_RunTest_WithProjectTags(t *testing.T) { assert.Equal(t, sarifResponse.Version, result.Sarif.Version) } -func mockTestCreatedResponseWithRevisionValidation(t *testing.T, mockHTTPClient *httpmocks.MockHTTPClient, testId uuid.UUID, orgId string, expectedRevisionId string, responseCode int) { +func mockTestCreatedResponseWithRevisionValidation(t *testing.T, mockHTTPClient *httpmocks.MockHTTPClient, testId uuid.UUID, orgId string, expectedRevisionId string, expectedCommitId string, expectedBranch string, responseCode int) { t.Helper() response := v20250407.NewTestResponse() response.Data.Id = testId @@ -503,6 +503,9 @@ func mockTestCreatedResponseWithRevisionValidation(t *testing.T, mockHTTPClient revision, err := testRequestBody.Data.Attributes.Input.AsTestInputUploadRevision() assert.NoError(t, err) assert.Equal(t, expectedRevisionId, revision.RevisionId) + require.NotNil(t, revision.Metadata) + assert.Equal(t, expectedCommitId, *revision.Metadata.CommitId) + assert.Equal(t, expectedBranch, *revision.Metadata.Branch) return req.URL.String() == expectedTestCreatedUrl && req.Method == http.MethodPost })).Times(1).Return(&http.Response{ @@ -524,10 +527,17 @@ func TestAnalysis_RunTest_WithUploadRevision(t *testing.T) { snapshotId := uuid.New() testId := uuid.New() revisionId := uuid.NewString() - targetId, err := scan.NewRepositoryTarget("../mypath/") + expectedCommitId := "abc123" + expectedBranch := "feature/my-branch" + targetId, err := scan.NewRepositoryTarget( + "../mypath/", + scan.WithRepositoryUrl("https://github.com/test/repo"), + scan.WithCommitId(expectedCommitId), + scan.WithBranchName(expectedBranch), + ) assert.NoError(t, err) - mockTestCreatedResponseWithRevisionValidation(t, mockHTTPClient, testId, orgId, revisionId, http.StatusCreated) + mockTestCreatedResponseWithRevisionValidation(t, mockHTTPClient, testId, orgId, revisionId, expectedCommitId, expectedBranch, http.StatusCreated) mockTestStatusResponse(t, mockHTTPClient, orgId, testId, http.StatusOK) expectedWebuilink := "" diff --git a/internal/api/test/2025-04-07/helper.go b/internal/api/test/2025-04-07/helper.go index b63018d6..ff29e899 100644 --- a/internal/api/test/2025-04-07/helper.go +++ b/internal/api/test/2025-04-07/helper.go @@ -34,15 +34,17 @@ func WithInputBundle(id string, localFilePath string, repoUrl *string, limitTest } } -func WithInputUploadRevision(id string, localFilePath string, repoUrl *string) CreateTestOption { +func WithInputUploadRevision(id string, localFilePath string, repoUrl *string, commitId *string, branch *string) CreateTestOption { return func(body *CreateTestApplicationVndAPIPlusJSONRequestBody) { revisionInput := v20250407.TestInputUploadRevision{ RevisionId: id, Type: v20250407.UploadRevision, Metadata: &struct { + Branch *string `json:"branch,omitempty"` + CommitId *string `json:"commit_id,omitempty"` LocalFilePath *string `json:"local_file_path,omitempty"` RepoUrl *string `json:"repo_url,omitempty"` - }{LocalFilePath: &localFilePath, RepoUrl: repoUrl}, + }{LocalFilePath: &localFilePath, RepoUrl: repoUrl, CommitId: commitId, Branch: branch}, } body.Data.Attributes.Input.FromTestInputUploadRevision(revisionInput)