diff --git a/README.md b/README.md index 8160da67..04e76b21 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,12 @@ Flags: --tree Recursively push all SBOMs in external reference tree ``` +An SBOM may be pushed as a package to a GitLab repository through the [Generic Package Registry web API](https://docs.gitlab.com/ee/user/packages/generic_packages) by using the following URL format. Authorization for this command is configured by assigning the value of your GitLab token to the `BOMCTL_GITLAB_TOKEN` environment variable. + +```shell +bomctl push SBOM_ID_OR_ALIAS https://www.gitlab.com/PROJECT/REPOSITORY#PACKAGE_NAME@PACKAGE_VERSION +``` + ### Tag Edit the tags of an SBOM document. diff --git a/internal/pkg/client/gitlab/client.go b/internal/pkg/client/gitlab/client.go index 2fc62565..61f38317 100644 --- a/internal/pkg/client/gitlab/client.go +++ b/internal/pkg/client/gitlab/client.go @@ -21,6 +21,8 @@ package gitlab import ( "fmt" + "io" + "net/http" "regexp" gitlab "gitlab.com/gitlab-org/api/client-go" @@ -28,14 +30,73 @@ import ( "github.com/bomctl/bomctl/internal/pkg/netutil" ) -type Client struct { - ProjectProvider - BranchProvider - CommitProvider - DependencyListExporter - Export *gitlab.DependencyListExport - GitLabToken string -} +type ( + projectProvider interface { + GetProject( + any, + *gitlab.GetProjectOptions, + ...gitlab.RequestOptionFunc, + ) (*gitlab.Project, *gitlab.Response, error) + } + + branchProvider interface { + GetBranch( + any, + string, + ...gitlab.RequestOptionFunc, + ) (*gitlab.Branch, *gitlab.Response, error) + } + + commitProvider interface { + GetCommit( + any, + string, + *gitlab.GetCommitOptions, + ...gitlab.RequestOptionFunc, + ) (*gitlab.Commit, *gitlab.Response, error) + } + + dependencyListExporter interface { + CreateDependencyListExport( + int, + *gitlab.CreateDependencyListExportOptions, + ...gitlab.RequestOptionFunc, + ) (*gitlab.DependencyListExport, *gitlab.Response, error) + GetDependencyListExport( + int, + ...gitlab.RequestOptionFunc, + ) (*gitlab.DependencyListExport, *gitlab.Response, error) + DownloadDependencyListExport(int, ...gitlab.RequestOptionFunc) (io.Reader, *gitlab.Response, error) + } + + genericPackagePublisher interface { + PublishPackageFile( + any, + string, + string, + string, + io.Reader, + *gitlab.PublishPackageFileOptions, + ...gitlab.RequestOptionFunc, + ) (*gitlab.GenericPackagesFile, *gitlab.Response, error) + } + + sbomFile struct { + Contents string + Name string + } + + Client struct { + projectProvider + branchProvider + commitProvider + dependencyListExporter + genericPackagePublisher + GitLabToken string + Export *gitlab.DependencyListExport + PushQueue []*sbomFile + } +) func (*Client) Name() string { return "GitLab" @@ -44,8 +105,8 @@ func (*Client) Name() string { func (*Client) RegExp() *regexp.Regexp { return regexp.MustCompile(fmt.Sprintf("(?i)^%s%s%s$", `(?Phttps?|git|ssh):\/\/`, - `(?P[^@\/?#:]+gitlab[^@\/?#:]+)(?::(?P\d+))?/`, - `(?P[^@#]+)@(?P\S+)`)) + `(?P[^@\/?#:]*gitlab[^@\/?#:]+)(?::(?P\d+))?/`, + `(?P[^@?#]+)(?:@(?P[^?#]+))?(?:\?(?P[^#]+))?(?:#(?P.+))?`)) } func (client *Client) Parse(rawURL string) *netutil.URL { @@ -58,7 +119,7 @@ func (client *Client) Parse(rawURL string) *netutil.URL { } // Ensure required map fields are present. - for _, required := range []string{"scheme", "hostname", "path", "gitRef"} { + for _, required := range []string{"scheme", "hostname", "path"} { if value, ok := results[required]; !ok || value == "" { return nil } @@ -70,5 +131,15 @@ func (client *Client) Parse(rawURL string) *netutil.URL { Port: results["port"], Path: results["path"], GitRef: results["gitRef"], + Query: results["query"], + Fragment: results["fragment"], } } + +func validateHTTPStatusCode(statusCode int) error { + if statusCode < http.StatusOK || statusCode >= http.StatusMultipleChoices { + return fmt.Errorf("%w. HTTP status code: %d", errFailedWebRequest, statusCode) + } + + return nil +} diff --git a/internal/pkg/client/gitlab/client_test.go b/internal/pkg/client/gitlab/client_test.go index fa1f6479..f2fb15ca 100644 --- a/internal/pkg/client/gitlab/client_test.go +++ b/internal/pkg/client/gitlab/client_test.go @@ -21,21 +21,35 @@ package gitlab_test import ( "bytes" + "context" "fmt" "io" "net/http" + "os" + "regexp" + "strings" "testing" + "github.com/protobom/protobom/pkg/sbom" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" gogitlab "gitlab.com/gitlab-org/api/client-go" "github.com/bomctl/bomctl/internal/pkg/client/gitlab" + "github.com/bomctl/bomctl/internal/pkg/db" + "github.com/bomctl/bomctl/internal/pkg/options" + "github.com/bomctl/bomctl/internal/pkg/outpututil" + "github.com/bomctl/bomctl/internal/testutil" ) type ( gitLabClientSuite struct { suite.Suite + tmpDir string + *options.Options + *db.Backend + documents []*sbom.Document + documentInfo []testutil.DocumentInfo } mockProjectProvider struct { @@ -53,14 +67,31 @@ type ( mockDependencyListExporter struct { mock.Mock } + + mockGenericPackagePublisher struct { + mock.Mock + } ) -//revive:disable:unchecked-type-assertion +//revive:disable:unchecked-type-assertion,import-shadowing + +func (mpp *mockGenericPackagePublisher) PublishPackageFile( + pid any, + packageName, packageVersion, fileName string, + content io.Reader, + opt *gogitlab.PublishPackageFileOptions, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic +) (*gogitlab.GenericPackagesFile, *gogitlab.Response, error) { + args := mpp.Called(pid, packageName, packageVersion, fileName, content, opt, options) + + //nolint:errcheck,wrapcheck + return args.Get(0).(*gogitlab.GenericPackagesFile), args.Get(1).(*gogitlab.Response), args.Error(2) +} func (mpp *mockProjectProvider) GetProject( pid any, opt *gogitlab.GetProjectOptions, - options ...gogitlab.RequestOptionFunc, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic ) (*gogitlab.Project, *gogitlab.Response, error) { args := mpp.Called(pid, opt, options) @@ -71,7 +102,7 @@ func (mpp *mockProjectProvider) GetProject( func (mbp *mockBranchProvider) GetBranch( pid any, branch string, - options ...gogitlab.RequestOptionFunc, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic ) (*gogitlab.Branch, *gogitlab.Response, error) { args := mbp.Called(pid, branch, options) @@ -83,7 +114,7 @@ func (mcp *mockCommitProvider) GetCommit( pid any, sha string, opt *gogitlab.GetCommitOptions, - options ...gogitlab.RequestOptionFunc, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic ) (*gogitlab.Commit, *gogitlab.Response, error) { args := mcp.Called(pid, sha, opt, options) @@ -94,7 +125,7 @@ func (mcp *mockCommitProvider) GetCommit( func (mdle *mockDependencyListExporter) CreateDependencyListExport( pipelineID int, opt *gogitlab.CreateDependencyListExportOptions, - options ...gogitlab.RequestOptionFunc, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic ) (*gogitlab.DependencyListExport, *gogitlab.Response, error) { args := mdle.Called(pipelineID, opt, options) @@ -104,7 +135,7 @@ func (mdle *mockDependencyListExporter) CreateDependencyListExport( func (mdle *mockDependencyListExporter) GetDependencyListExport( id int, - options ...gogitlab.RequestOptionFunc, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic ) (*gogitlab.DependencyListExport, *gogitlab.Response, error) { args := mdle.Called(id, options) @@ -114,7 +145,7 @@ func (mdle *mockDependencyListExporter) GetDependencyListExport( func (mdle *mockDependencyListExporter) DownloadDependencyListExport( id int, - options ...gogitlab.RequestOptionFunc, + options ...gogitlab.RequestOptionFunc, //nolint:gocritic ) (io.Reader, *gogitlab.Response, error) { args := mdle.Called(id, options) @@ -122,7 +153,38 @@ func (mdle *mockDependencyListExporter) DownloadDependencyListExport( return args.Get(0).(io.Reader), args.Get(1).(*gogitlab.Response), args.Error(2) } -//revive:enable:unchecked-type-assertion +//revive:enable:unchecked-type-assertion,import-shadowing + +func (glcs *gitLabClientSuite) SetupTest() { + var err error + + glcs.tmpDir, err = os.MkdirTemp("", "gitlab-push-test") + glcs.Require().NoError(err, "Failed to create temporary directory") + + glcs.Backend, err = testutil.NewTestBackend() + glcs.Require().NoError(err, "failed database backend creation") + + glcs.documentInfo, err = testutil.AddTestDocuments(glcs.Backend) + glcs.Require().NoError(err, "failed database backend setup") + + for _, docInfo := range glcs.documentInfo { + glcs.documents = append(glcs.documents, docInfo.Document) + } + + glcs.Options = options.New(). + WithCacheDir(glcs.tmpDir). + WithContext(context.WithValue(context.Background(), db.BackendKey{}, glcs.Backend)) +} + +func (glcs *gitLabClientSuite) TearDownTest() { + glcs.Backend.CloseClient() + glcs.documents = nil + glcs.documentInfo = nil + + if err := os.RemoveAll(glcs.tmpDir); err != nil { + glcs.T().Fatalf("Error removing temp directory %s", glcs.tmpDir) + } +} var successGitLabResponse = &gogitlab.Response{ Response: &http.Response{ @@ -233,13 +295,12 @@ func (glcs *gitLabClientSuite) TestClient_Fetch() { []gogitlab.RequestOptionFunc(nil), ).Return(bytes.NewBuffer(expectedSbomData), successGitLabResponse, nil) - client := &gitlab.Client{ - ProjectProvider: mockedProjectProvider, - BranchProvider: mockedBranchProvider, - CommitProvider: mockedCommitProvider, - DependencyListExporter: mockedDependencyListExporter, - Export: nil, - } + client := gitlab.NewFetchClient( + mockedProjectProvider, + mockedBranchProvider, + mockedCommitProvider, + mockedDependencyListExporter, + ) glcs.Run("Fetch", func() { _, err := client.Fetch(dummyFetchURL, nil) @@ -252,6 +313,87 @@ func (glcs *gitLabClientSuite) TestClient_Fetch() { }) } +func (glcs *gitLabClientSuite) TestClient_Push() { + dummyHost := "gitlab.dummy" + dummyProjectID := 1234 + dummyProjectName := "TESTING/TEST" + dummyPackageName := "SBOM" + dummyPackageVersion := "1.0.0" + + dummyURL := fmt.Sprintf( + "https://%s/%s#%s@%s", + dummyHost, + dummyProjectName, + dummyPackageName, + dummyPackageVersion, + ) + + uuidRegex := regexp.MustCompile(`urn:uuid:([\w-]+)`) + uuidMatch := uuidRegex.FindStringSubmatch(glcs.documents[0].GetMetadata().GetId()) + expectedSbomFileName := uuidMatch[1] + ".json" + + sbomWriter := &gitlab.StringWriter{&strings.Builder{}} + err := outpututil.WriteStream(glcs.documents[0], "original", glcs.Options, sbomWriter) + glcs.Require().NoError(err, "failed to serialize SBOM: %v", err) + + expectedSbom := sbomWriter.String() + + mockedProjectProvider := &mockProjectProvider{} + mockedGenericPackagePublisher := &mockGenericPackagePublisher{} + + mockedProjectProvider.On( + "GetProject", + dummyProjectName, + (*gogitlab.GetProjectOptions)(nil), + []gogitlab.RequestOptionFunc(nil), + ).Return( + &gogitlab.Project{ + ID: dummyProjectID, + Name: dummyProjectName, + }, + successGitLabResponse, + nil, + ) + + mockedGenericPackagePublisher.On( + "PublishPackageFile", + dummyProjectID, + dummyPackageName, + dummyPackageVersion, + expectedSbomFileName, + mock.Anything, + (*gogitlab.PublishPackageFileOptions)(nil), + []gogitlab.RequestOptionFunc(nil), + ).Return( + &gogitlab.GenericPackagesFile{}, + successGitLabResponse, + nil, + ) + + client := gitlab.NewPushClient(mockedProjectProvider, mockedGenericPackagePublisher) + + glcs.Run("Push", func() { + err = client.AddFile( + dummyURL, + glcs.documents[0].GetMetadata().GetId(), + &options.PushOptions{ + Options: glcs.Options, + Format: "original", + }, + ) + glcs.Require().NoError(err, "failed to add file: %v", err) + glcs.Require().Len(client.PushQueue, 1) + glcs.Require().Equal(expectedSbomFileName, client.PushQueue[0].Name) + glcs.Require().Equal(expectedSbom, client.PushQueue[0].Contents) + + err = client.Push(dummyURL, nil) + glcs.Require().NoError(err, "failed to create dependency list export: %v", err) + + mockedProjectProvider.AssertExpectations(glcs.T()) + mockedGenericPackagePublisher.AssertExpectations(glcs.T()) + }) +} + func TestGithubClientSuite(t *testing.T) { t.Parallel() suite.Run(t, new(gitLabClientSuite)) diff --git a/internal/pkg/client/gitlab/fetch.go b/internal/pkg/client/gitlab/fetch.go index 65413b1a..afa9e409 100644 --- a/internal/pkg/client/gitlab/fetch.go +++ b/internal/pkg/client/gitlab/fetch.go @@ -33,60 +33,12 @@ import ( "github.com/bomctl/bomctl/internal/pkg/options" ) -type ( - ProjectProvider interface { - GetProject( - any, - *gitlab.GetProjectOptions, - ...gitlab.RequestOptionFunc, - ) (*gitlab.Project, *gitlab.Response, error) - } - - BranchProvider interface { - GetBranch( - any, - string, - ...gitlab.RequestOptionFunc, - ) (*gitlab.Branch, *gitlab.Response, error) - } - - CommitProvider interface { - GetCommit( - any, - string, - *gitlab.GetCommitOptions, - ...gitlab.RequestOptionFunc, - ) (*gitlab.Commit, *gitlab.Response, error) - } - - DependencyListExporter interface { - CreateDependencyListExport( - int, - *gitlab.CreateDependencyListExportOptions, - ...gitlab.RequestOptionFunc, - ) (*gitlab.DependencyListExport, *gitlab.Response, error) - GetDependencyListExport( - int, - ...gitlab.RequestOptionFunc, - ) (*gitlab.DependencyListExport, *gitlab.Response, error) - DownloadDependencyListExport(int, ...gitlab.RequestOptionFunc) (io.Reader, *gitlab.Response, error) - } -) - var ( errInvalidGitLabURL = errors.New("invalid URL for GitLab fetching") errFailedWebRequest = errors.New("web request failed") errForbiddenAccess = errors.New("the supplied token is missing the read_dependency permission") ) -func validateHTTPStatusCode(statusCode int) error { - if statusCode < http.StatusOK || http.StatusMultipleChoices <= statusCode { - return fmt.Errorf("%w. HTTP status code: %d", errFailedWebRequest, statusCode) - } - - return nil -} - func (client *Client) createExport(projectName, branchName string) error { project, response, err := client.GetProject(projectName, nil) if err != nil { @@ -191,10 +143,10 @@ func (client *Client) PrepareFetch(url *netutil.URL, _auth *netutil.BasicAuth, _ } client.GitLabToken = gitLabToken - client.ProjectProvider = gitLabClient.Projects - client.BranchProvider = gitLabClient.Branches - client.CommitProvider = gitLabClient.Commits - client.DependencyListExporter = gitLabClient.DependencyListExport + client.projectProvider = gitLabClient.Projects + client.branchProvider = gitLabClient.Branches + client.commitProvider = gitLabClient.Commits + client.dependencyListExporter = gitLabClient.DependencyListExport return nil } diff --git a/internal/pkg/client/gitlab/internal_test.go b/internal/pkg/client/gitlab/internal_test.go new file mode 100644 index 00000000..066a1989 --- /dev/null +++ b/internal/pkg/client/gitlab/internal_test.go @@ -0,0 +1,46 @@ +// ----------------------------------------------------------------------------- +// SPDX-FileCopyrightText: Copyright © 2025 bomctl a Series of LF Projects, LLC +// SPDX-FileName: internal/pkg/client/gitlab/internal_test.go +// SPDX-FileType: SOURCE +// SPDX-License-Identifier: Apache-2.0 +// ----------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ----------------------------------------------------------------------------- + +package gitlab + +type StringWriter = stringWriter + +func NewFetchClient( + projectProvider projectProvider, + branchProvider branchProvider, + commitProvider commitProvider, + dependencyListExporter dependencyListExporter, +) *Client { + return &Client{ + projectProvider: projectProvider, + branchProvider: branchProvider, + commitProvider: commitProvider, + dependencyListExporter: dependencyListExporter, + } +} + +func NewPushClient( + projectProvider projectProvider, + genericPackagePublisher genericPackagePublisher, +) *Client { + return &Client{ + projectProvider: projectProvider, + genericPackagePublisher: genericPackagePublisher, + } +} diff --git a/internal/pkg/client/gitlab/push.go b/internal/pkg/client/gitlab/push.go index 3bf1cc0e..73545bd2 100644 --- a/internal/pkg/client/gitlab/push.go +++ b/internal/pkg/client/gitlab/push.go @@ -19,16 +19,158 @@ package gitlab -import "github.com/bomctl/bomctl/internal/pkg/options" +import ( + "errors" + "fmt" + "os" + "regexp" + "strings" -func (*Client) AddFile(_name, _id string, _opts *options.PushOptions) error { + gitlab "gitlab.com/gitlab-org/api/client-go" + + "github.com/bomctl/bomctl/internal/pkg/db" + "github.com/bomctl/bomctl/internal/pkg/options" + "github.com/bomctl/bomctl/internal/pkg/outpututil" +) + +type stringWriter struct { + *strings.Builder +} + +var ( + errInvalidSbomID = errors.New("invalid SBOM ID") + errMissingPackageInfo = errors.New("missing package name or version") +) + +func (*stringWriter) Close() error { return nil } -func (*Client) PreparePush(_pushURL string, _opts *options.PushOptions) error { +func (client *Client) PreparePush(pushURL string, _opts *options.PushOptions) error { + gitLabToken := os.Getenv("BOMCTL_GITLAB_TOKEN") + + url := client.Parse(pushURL) + + host := url.Hostname + + if url.Port != "" { + host = fmt.Sprintf("%s:%s", host, url.Port) + } + + baseURL := fmt.Sprintf("https://%s/api/v4", host) + + gitLabClient, err := gitlab.NewClient(gitLabToken, gitlab.WithBaseURL(baseURL)) + if err != nil { + return fmt.Errorf("%w", err) + } + + client.GitLabToken = gitLabToken + client.projectProvider = gitLabClient.Projects + client.genericPackagePublisher = gitLabClient.GenericPackages + + client.PushQueue = make([]*sbomFile, 0) + return nil } -func (*Client) Push(_pushURL string, _opts *options.PushOptions) error { +func (client *Client) AddFile(_pushURL, id string, opts *options.PushOptions) error { + opts.Logger.Info("Adding file", "id", id) + + backend, err := db.BackendFromContext(opts.Context()) + if err != nil { + return fmt.Errorf("%w", err) + } + + sbom, err := backend.GetDocumentByIDOrAlias(id) + if err != nil { + return fmt.Errorf("%w", err) + } + + sbomFormat := sbom.GetMetadata().GetSourceData().GetFormat() + + isCycloneDX := strings.Contains(sbomFormat, "cyclonedx") + + var uuidRegex *regexp.Regexp + + if isCycloneDX { + uuidRegex = regexp.MustCompile(`^urn:uuid:([\w-]+)$`) + } else { + uuidRegex = regexp.MustCompile(`^.+/([^/#]+)(?:#\w+)?`) + } + + uuidMatch := uuidRegex.FindStringSubmatch(id) + + if len(uuidMatch) == 0 { + return fmt.Errorf("%w: %s", errInvalidSbomID, id) + } + + sbomFilename := uuidMatch[1] + + xmlFormatRegex := regexp.MustCompile(`\bxml\b`) + xmlFormatMatch := xmlFormatRegex.FindStringSubmatch(string(opts.Format)) + + if len(xmlFormatMatch) == 0 { + sbomFilename += ".json" + } else { + sbomFilename += ".xml" + } + + sbomWriter := &stringWriter{&strings.Builder{}} + if err := outpututil.WriteStream(sbom, opts.Format, opts.Options, sbomWriter); err != nil { + return fmt.Errorf("failed to serialize SBOM %s: %w", id, err) + } + + client.PushQueue = append(client.PushQueue, &sbomFile{ + Name: sbomFilename, + Contents: sbomWriter.String(), + }) + + return nil +} + +func (client *Client) Push(pushURL string, _opts *options.PushOptions) error { + url := client.Parse(pushURL) + if url == nil { + return fmt.Errorf("%w: %s", errInvalidGitLabURL, pushURL) + } + + project, response, err := client.GetProject(url.Path, nil) + if err != nil { + return fmt.Errorf("failed to get project info: %w", err) + } + + if err := validateHTTPStatusCode(response.StatusCode); err != nil { + return err + } + + packageInfo := strings.Split(url.Fragment, "@") + + if packageInfoExpectedLength := 2; len(packageInfo) != packageInfoExpectedLength { + return fmt.Errorf("%w: %s", errMissingPackageInfo, url.Fragment) + } + + packageName := packageInfo[0] + packageVersion := packageInfo[1] + + for _, sbomFile := range client.PushQueue { + sbomReader := strings.NewReader(sbomFile.Contents) + + _, response, err := client.genericPackagePublisher.PublishPackageFile( + project.ID, + packageName, + packageVersion, + sbomFile.Name, + sbomReader, + nil, + ) + if err != nil { + return fmt.Errorf("failed to push sbom: %w", err) + } + + if err := validateHTTPStatusCode(response.StatusCode); err != nil { + return err + } + } + return nil } diff --git a/internal/pkg/push/push.go b/internal/pkg/push/push.go index 6c2197e7..b72635dd 100644 --- a/internal/pkg/push/push.go +++ b/internal/pkg/push/push.go @@ -29,6 +29,7 @@ import ( "github.com/bomctl/bomctl/internal/pkg/client" "github.com/bomctl/bomctl/internal/pkg/client/git" "github.com/bomctl/bomctl/internal/pkg/client/github" + "github.com/bomctl/bomctl/internal/pkg/client/gitlab" "github.com/bomctl/bomctl/internal/pkg/client/http" "github.com/bomctl/bomctl/internal/pkg/client/oci" "github.com/bomctl/bomctl/internal/pkg/db" @@ -38,7 +39,7 @@ import ( ) func NewPusher(url string) (client.Pusher, error) { - clients := []client.Pusher{&github.Client{}, &git.Client{}, &http.Client{}, &oci.Client{}} + clients := []client.Pusher{&gitlab.Client{}, &github.Client{}, &git.Client{}, &http.Client{}, &oci.Client{}} pusher, err := sliceutil.Next(clients, func(f client.Pusher) bool { return f.Parse(url) != nil }) if err != nil {