From 07906cfb8d041688fa28b8e8de7be028e73e655c Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 9 Apr 2025 14:13:59 -0400 Subject: [PATCH] [wip] run matchers in parallel Signed-off-by: Alex Goodman --- cmd/grype/cli/commands/root.go | 12 ++++-- grype/deprecated.go | 3 +- grype/vulnerability_matcher.go | 76 +++++++++++++++++++++++----------- 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/cmd/grype/cli/commands/root.go b/cmd/grype/cli/commands/root.go index 750522be5df..c8c23fc58fb 100644 --- a/cmd/grype/cli/commands/root.go +++ b/cmd/grype/cli/commands/root.go @@ -1,8 +1,10 @@ package commands import ( + "context" "errors" "fmt" + "github.com/anchore/go-sync" "strings" "github.com/spf13/cobra" @@ -74,12 +76,12 @@ You can also pipe in Syft JSON directly: Args: validateRootArgs, SilenceUsage: true, SilenceErrors: true, - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { userInput := "" if len(args) > 0 { userInput = args[0] } - return runGrype(app, opts, userInput) + return runGrype(cmd.Context(), app, opts, userInput) }, ValidArgsFunction: dockerImageValidArgsFunction, }, opts) @@ -107,7 +109,7 @@ var ignoreLinuxKernelHeaders = []match.IgnoreRule{ } //nolint:funlen -func runGrype(app clio.Application, opts *options.Grype, userInput string) (errs error) { +func runGrype(ctx context.Context, app clio.Application, opts *options.Grype, userInput string) (errs error) { writer, err := format.MakeScanResultWriter(opts.Outputs, opts.File, format.PresentationConfig{ TemplateFilePath: opts.OutputTemplateFile, ShowSuppressed: opts.ShowSuppressed, @@ -192,7 +194,9 @@ func runGrype(app clio.Application, opts *options.Grype, userInput string) (errs }), } - remainingMatches, ignoredMatches, err := vulnMatcher.FindMatches(packages, pkgContext) + ctx = sync.SetContextExecutor(ctx, cataloging.ExecutorCPU, sync.NewExecutor(50)) + + remainingMatches, ignoredMatches, err := vulnMatcher.FindMatches(ctx, packages, pkgContext) if err != nil { if !errors.Is(err, grypeerr.ErrAboveSeverityThreshold) { return err diff --git a/grype/deprecated.go b/grype/deprecated.go index fc8ec72b29f..ad7c4fb6673 100644 --- a/grype/deprecated.go +++ b/grype/deprecated.go @@ -1,6 +1,7 @@ package grype import ( + "context" "github.com/anchore/grype/grype/match" "github.com/anchore/grype/grype/matcher" "github.com/anchore/grype/grype/pkg" @@ -42,7 +43,7 @@ func FindVulnerabilitiesForPackage(store vulnerability.Provider, d *linux.Releas NormalizeByCVE: false, } - actualResults, _, err := runner.FindMatches(packages, pkg.Context{ + actualResults, _, err := runner.FindMatches(context.Background(), packages, pkg.Context{ Distro: d, }) if err != nil || actualResults == nil { diff --git a/grype/vulnerability_matcher.go b/grype/vulnerability_matcher.go index 6c5266930fa..029f27eb262 100644 --- a/grype/vulnerability_matcher.go +++ b/grype/vulnerability_matcher.go @@ -1,8 +1,11 @@ package grype import ( + "context" "errors" "fmt" + "github.com/anchore/go-sync" + "github.com/anchore/syft/syft/cataloging" "strings" "github.com/wagoodman/go-partybus" @@ -53,7 +56,7 @@ func (m *VulnerabilityMatcher) WithIgnoreRules(ignoreRules []match.IgnoreRule) * return m } -func (m *VulnerabilityMatcher) FindMatches(pkgs []pkg.Package, context pkg.Context) (remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, err error) { +func (m *VulnerabilityMatcher) FindMatches(ctx context.Context, pkgs []pkg.Package, pkgCtx pkg.Context) (remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, err error) { progressMonitor := trackMatcher(len(pkgs)) defer func() { @@ -64,13 +67,13 @@ func (m *VulnerabilityMatcher) FindMatches(pkgs []pkg.Package, context pkg.Conte } }() - remainingMatches, ignoredMatches, err = m.findDBMatches(pkgs, context, progressMonitor) + remainingMatches, ignoredMatches, err = m.findDBMatches(ctx, pkgs, pkgCtx, progressMonitor) if err != nil { err = fmt.Errorf("unable to find matches against vulnerability database: %w", err) return remainingMatches, ignoredMatches, err } - remainingMatches, ignoredMatches, err = m.findVEXMatches(context, remainingMatches, ignoredMatches, progressMonitor) + remainingMatches, ignoredMatches, err = m.findVEXMatches(pkgCtx, remainingMatches, ignoredMatches, progressMonitor) if err != nil { err = fmt.Errorf("unable to find matches against VEX sources: %w", err) return remainingMatches, ignoredMatches, err @@ -88,11 +91,11 @@ func (m *VulnerabilityMatcher) FindMatches(pkgs []pkg.Package, context pkg.Conte return remainingMatches, ignoredMatches, nil } -func (m *VulnerabilityMatcher) findDBMatches(pkgs []pkg.Package, context pkg.Context, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) { +func (m *VulnerabilityMatcher) findDBMatches(ctx context.Context, pkgs []pkg.Package, pkgCtx pkg.Context, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) { var ignoredMatches []match.IgnoredMatch log.Trace("finding matches against DB") - matches, err := m.searchDBForMatches(context.Distro, pkgs, progressMonitor) + matches, err := m.searchDBForMatches(ctx, pkgCtx.Distro, pkgs, progressMonitor) if err != nil { if match.IsFatalError(err) { return nil, nil, err @@ -138,12 +141,12 @@ func (m *VulnerabilityMatcher) mergeIgnoredMatches(allIgnoredMatches ...[]match. //nolint:funlen func (m *VulnerabilityMatcher) searchDBForMatches( + ctx context.Context, release *linux.Release, packages []pkg.Package, progressMonitor *monitorWriter, ) (match.Matches, error) { - var allMatches []match.Match - var allIgnored []match.IgnoredMatch + matcherIndex, defaultMatcher := newMatcherIndex(m.Matchers) var d *distro.Distro @@ -163,8 +166,13 @@ func (m *VulnerabilityMatcher) searchDBForMatches( defaultMatcher = stock.NewStockMatcher(stock.MatcherConfig{UseCPEs: true}) } - var matcherErrs []error - for _, p := range packages { + type result struct { + matches []match.Match + ignored []match.IgnoredMatch + distro distro.Distro + } + + processor := func(p pkg.Package) (result, error) { progressMonitor.PackagesProcessed.Increment() log.WithFields("package", displayPackage(p)).Trace("searching for vulnerability matches") @@ -178,38 +186,56 @@ func (m *VulnerabilityMatcher) searchDBForMatches( if !ok { matchAgainst = []match.Matcher{defaultMatcher} } + var matcherErrs []error + var allMatches []match.Match + var allIgnoredMatches []match.IgnoredMatch for _, theMatcher := range matchAgainst { matches, ignoredMatches, err := theMatcher.Match(m.VulnerabilityProvider, p) if err != nil { if match.IsFatalError(err) { - return match.Matches{}, err + return result{}, err } log.WithFields("error", err, "package", displayPackage(p)).Warn("matcher returned error") matcherErrs = append(matcherErrs, err) } + allMatches = append(allMatches, matches...) + allIgnoredMatches = append(allIgnoredMatches, ignoredMatches...) + } + return result{matches: allMatches, ignored: allIgnoredMatches, distro: *p.Distro}, errors.Join(matcherErrs...) + } - allIgnored = append(allIgnored, ignoredMatches...) + var allMatches []match.Match + var allIgnored []match.IgnoredMatch + accumulator := func(p pkg.Package, res result) { + matches := res.matches + ignoredMatches := res.ignored - // Filter out matches based on records in the database exclusion table and hard-coded rules - filtered, dropped := match.ApplyExplicitIgnoreRules(m.ExclusionProvider, match.NewMatches(matches...)) + allIgnored = append(allIgnored, ignoredMatches...) - additionalMatches := filtered.Sorted() - logPackageMatches(p, additionalMatches) - logExplicitDroppedPackageMatches(p, dropped) - allMatches = append(allMatches, additionalMatches...) + // filter out matches based on records in the database exclusion table and hard-coded rules + filtered, dropped := match.ApplyExplicitIgnoreRules(m.ExclusionProvider, match.NewMatches(matches...)) - progressMonitor.MatchesDiscovered.Add(int64(len(additionalMatches))) + additionalMatches := filtered.Sorted() + logPackageMatches(p, additionalMatches) + logExplicitDroppedPackageMatches(p, dropped) + allMatches = append(allMatches, additionalMatches...) - // note: there is a difference between "ignore" and "dropped" matches. - // ignored: matches that are filtered out due to user-provided ignore rules - // dropped: matches that are filtered out due to hard-coded rules - updateVulnerabilityList(progressMonitor, additionalMatches, nil, dropped, m.VulnerabilityProvider) - } + progressMonitor.MatchesDiscovered.Add(int64(len(additionalMatches))) - p.Distro = orig + // note: there is a difference between "ignore" and "dropped" matches. + // ignored: matches that are filtered out due to user-provided ignore rules + // dropped: matches that are filtered out due to hard-coded rules + updateVulnerabilityList(progressMonitor, additionalMatches, nil, dropped, m.VulnerabilityProvider) } + matcherErr := sync.Collect(&ctx, + cataloging.ExecutorFile, + sync.ToSeq(packages), + processor, + accumulator, + ) + // apply ignores based on matchers returning ignore rules filtered, dropped := match.ApplyIgnoreFilters(allMatches, ignoredMatchFilter(allIgnored)) logIgnoredMatches(dropped) @@ -220,7 +246,7 @@ func (m *VulnerabilityMatcher) searchDBForMatches( // update the total discovered matches after removing all duplicates and ignores progressMonitor.MatchesDiscovered.Set(int64(res.Count())) - return res, errors.Join(matcherErrs...) + return res, matcherErr } func (m *VulnerabilityMatcher) findVEXMatches(context pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) {