8000 refactor: pass file arguments as stream readers/writers instead of strings by jhoward-lm · Pull Request #76 · bomctl/bomctl · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

refactor: pass file arguments as stream readers/writers instead of strings #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? 8000 Sign in to your account

Merged
merged 3 commits into from
Jun 11, 2024
Merged

refactor: pass file arguments as stream readers/writers instead of strings #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2894259
refactor: pass file arguments as stream readers/writers instead of st…
jhoward-lm Jun 11, 2024
c910bda
chore: address PR comment
jhoward-lm Jun 11, 2024
192b969
fix: limit input SBOM URLs to a single argument when --output-file is…
jhoward-lm Jun 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
< 8000 /form>
Diff view
20 changes: 18 additions & 2 deletions cmd/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,27 @@ func fetchCmd() *cobra.Command {
Short: "Fetch SBOM file(s) from HTTP(S), OCI, or Git URLs",
Long: "Fetch SBOM file(s) from HTTP(S), OCI, or Git URLs",
Run: func(_ *cobra.Command, _ []string) {
var err error
var (
err error
output *os.File
)

logger = utils.NewLogger("fetch")

if string(outputFile) != "" {
if len(sbomURLs) > 1 {
logger.Fatal("The --output-file option cannot be used when more than one URL is provided.")
}

if output, err = os.Create(string(outputFile)); err != nil {
logger.Fatal("error creating output file", "outputFile", outputFile)
}

defer output.Close()
}

for _, url := range sbomURLs {
if err = fetch.Exec(url, outputFile.String(), useNetRC); err != nil {
if err = fetch.Fetch(url, output, useNetRC); err != nil {
logger.Error(err)
}
}
Expand Down
100 changes: 73 additions & 27 deletions internal/pkg/fetch/fetch.go
View file
Open in desktop
10000
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,17 @@
package fetch

import (
"bytes"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strconv"

"github.com/jdx/go-netrc"
"github.com/protobom/protobom/pkg/sbom"
"github.com/protobom/protobom/pkg/reader"

"github.com/bomctl/bomctl/internal/pkg/db"
"github.com/bomctl/bomctl/internal/pkg/fetch/git"
Expand All @@ -40,14 +43,14 @@ var errUnsupportedURL = errors.New("unsupported URL scheme")

type Fetcher interface {
url.Parser
Fetch(*url.ParsedURL, *url.BasicAuth) (*sbom.Document, error)
Fetch(*url.ParsedURL, *url.BasicAuth) ([]byte, error)
Name() string
}

func Exec(sbomURL, outputFile string, useNetRC bool) error {
func Fetch(sbomURL string, outputFile *os.File, useNetRC bool) error { //nolint:cyclop,funlen
logger := utils.NewLogger("fetch")

fetcher, err := getFetcher(sbomURL, outputFile)
fetcher, err := NewFetcher(sbomURL)
if err != nil {
return err
}
Expand All @@ -63,53 +66,96 @@ func Exec(sbomURL, outputFile string, useNetRC bool) error {
}
}

document, err := fetcher.Fetch(parsedURL, auth)
sbomData, err := fetcher.Fetch(parsedURL, auth)
if err != nil {
return fmt.Errorf("%w", err)
}

if outputFile != nil {
// Write the SBOM document bytes to file.
if _, err = io.Copy(outputFile, bytes.NewReader(sbomData)); err != nil {
return fmt.Errorf("failed to write %s: %w", outputFile.Name(), err)
}
}

sbomReader := reader.New()

document, err := sbomReader.ParseStream(bytes.NewReader(sbomData))
if err != nil {
return fmt.Errorf("error parsing SBOM file content: %w", err)
}

// Insert fetched document data into database.
err = db.AddDocument(document)
if err != nil {
return fmt.Errorf("%w", err)
}

if outputFile == nil || outputFile.Name() == "" {
return nil
}

// Fetch externally referenced BOMs
var idx uint8
for _, ref := range utils.GetBOMReferences(document) {
idx++

if outputFile != "" {
// Matches base filename, excluding extension
baseFilename := regexp.MustCompile(`^([^\.]+)?`).FindString(filepath.Base(outputFile))

outputFile = fmt.Sprintf("%s-%d.%s",
filepath.Join(filepath.Dir(outputFile), baseFilename),
idx,
filepath.Ext(outputFile),
)
refOutput, err := getRefFile(outputFile)
if err != nil {
return err
}

err := Exec(ref.Url, outputFile, useNetRC)
if err != nil {
return fmt.Errorf("%w", err)
defer refOutput.Close()

if err := Fetch(ref.Url, refOutput, useNetRC); err != nil {
return err
}
}

return nil
}

func getFetcher(sbomURL, outputFile string) (Fetcher, error) {
switch {
case (&oci.Fetcher{}).Parse(sbomURL) != nil:
return &oci.Fetcher{}, nil
case (&git.Fetcher{}).Parse(sbomURL) != nil:
return &git.Fetcher{}, nil
case (&http.Fetcher{}).Parse(sbomURL) != nil:
return &http.Fetcher{OutputFile: outputFile}, nil
default:
return nil, fmt.Errorf("%w", errUnsupportedURL)
func NewFetcher(sbomURL string) (Fetcher, error) {
for _, fetcher := range []Fetcher{&git.Fetcher{}, &http.Fetcher{}, &oci.Fetcher{}} {
if parsedURL := fetcher.Parse(sbomURL); parsedURL != nil {
return fetcher, nil
}
}

return nil, fmt.Errorf("%w: %s", errUnsupportedURL, sbomURL)
}

func getRefFile(parentFile *os.File) (*os.File, error) {
idx := 0

// Matches base filename, excluding extension
baseFilename := regexp.MustCompile(`^([^\.]+)?`).FindString(filepath.Base(parentFile.Name()))

suffix := regexp.MustCompile(`^.*-(\d+)`).FindString(baseFilename)

if suffix != "" {
var err error

idx, err = strconv.Atoi(suffix)
if err != nil {
return nil, fmt.Errorf("%w", err)
}
}

idx++

outputFile := fmt.Sprintf("%s-%d.%s",
filepath.Join(filepath.Dir(parentFile.Name()), baseFilename),
idx,
filepath.Ext(parentFile.Name()),
)

refOutput, err := os.Create(outputFile)
if err != nil {
return nil, fmt.Errorf("%w", err)
}

return refOutput, nil
}

func setNetRCAuth(hostname string, auth *url.BasicAuth) error {
Expand Down
16 changes: 4 additions & 12 deletions internal/pkg/fetch/git/git.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,8 @@ import (

"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/protobom/protobom/pkg/sbom"

"github.com/bomctl/bomctl/internal/pkg/url"
"github.com/bomctl/bomctl/internal/pkg/utils"
)

type Fetcher struct{}
Expand Down Expand Up @@ -83,8 +81,8 @@ func (fetcher *Fetcher) Parse(fetchURL string) *url.ParsedURL {
}
}

func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) (*sbom.Document, error) {
// Create temp directory to clone into
func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) ([]byte, error) {
// Create temp directory to clone into.
tmpDir, err := os.MkdirTemp(os.TempDir(), "repo")
if err != nil {
return nil, fmt.Errorf("failed to create temp directory: %w", err)
Expand Down Expand Up @@ -118,16 +116,10 @@ func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) (*s
}

// Read the file specified in the URL fragment
sbomBytes, err := os.ReadFile(filepath.Join(tmpDir, parsedURL.Fragment))
sbomData, err := os.ReadFile(filepath.Join(tmpDir, parsedURL.Fragment))
if err != nil {
return nil, fmt.Errorf("failed to open file %s: %w", parsedURL.Fragment, err)
}

// Parse the file content
document, err := utils.ParseSBOMData(sbomBytes)
if err != nil {
return nil, fmt.Errorf("error parsing SBOM file content: %w", err)
}

return document, nil
return sbomData, nil
}
46 changes: 8 additions & 38 deletions internal/pkg/fetch/http/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,12 @@ import (
"fmt"
"io"
"net/http"
"os"
"regexp"

"github.com/protobom/protobom/pkg/sbom"

"github.com/bomctl/bomctl/internal/pkg/url"
"github.com/bomctl/bomctl/internal/pkg/utils"
)

var client = http.DefaultClient

type Fetcher struct {
OutputFile string
}
type Fetcher struct{}

func (fetcher *Fetcher) Name() string {
return "HTTP"
Expand Down Expand Up @@ -81,47 +73,25 @@ func (fetcher *Fetcher) Parse(fetchURL string) *url.ParsedURL {
}
}

func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) (*sbom.Document, error) {
func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) ([]byte, error) {
req, err := http.NewRequestWithContext(context.Background(), "GET", parsedURL.String(), nil)
if err != nil {
return nil, fmt.Errorf("%w", err)
return nil, fmt.Errorf("failed creating request to %s: %w", parsedURL.String(), err)
}

auth.SetAuth(req)

// Get the data
resp, err := client.Do(req)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("%w", err)
return nil, fmt.Errorf("failed request to %s: %w", parsedURL.String(), err)
}

defer resp.Body.Close()

respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("%w", err)
}

// Create the file if specified at the command line
if fetcher.OutputFile != "" {
out, err := os.Create(fetcher.OutputFile)
if err != nil {
return nil, fmt.Errorf("%w", err)
}

defer out.Close()

// Write the response body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return nil, fmt.Errorf("%w", err)
}
}

document, err := utils.ParseSBOMData(respBytes)
sbomData, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("%w", err)
return nil, fmt.Errorf("error reading response: %w", err)
}

return document, nil
return sbomData, nil
}
11 changes: 2 additions & 9 deletions internal/pkg/fetch/oci/oci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"strings"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/protobom/protobom/pkg/sbom"
oras "oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/memory"
Expand All @@ -36,7 +35,6 @@ import (
"oras.land/oras-go/v2/registry/remote/retry"

"github.com/bomctl/bomctl/internal/pkg/url"
"github.com/bomctl/bomctl/internal/pkg/utils"
)

var (
Expand Down Expand Up @@ -107,9 +105,8 @@ func (fetcher *Fetcher) Parse(fetchURL string) *url.ParsedURL {
}
}

func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) (*sbom.Document, error) {
func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) ([]byte, error) {
var (
document *sbom.Document
err error
manifestDescriptor, sbomDescriptor *ocispec.Descriptor
repo *remote.Repository
Expand Down Expand Up @@ -137,11 +134,7 @@ func (fetcher *Fetcher) Fetch(parsedURL *url.ParsedURL, auth *url.BasicAuth) (*s
return nil, err
}

if document, err = utils.ParseSBOMData(sbomData); err != nil {
return nil, fmt.Errorf("error parsing SBOM file content: %w", err)
}

return document, nil
return sbomData, nil
}

func createRepository(parsedURL *url.ParsedURL, auth *url.BasicAuth) (*remote.Repository, error) {
Expand Down
Loading
0