8000 fix: thread exhaustion error by cx-rogerio-dalot-x · Pull Request #297 · Checkmarx/2ms · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

fix: thread exhaustion error #297

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 17, 2025
+196 −231
Merged

fix: thread exhaustion error #297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8689ae8
fix: set maximum of goroutines
cx-rogerio-dalot-x Jun 4, 2025
d67cb31
fix: lock net version
cx-rogerio-dalot-x Jun 4, 2025
4b7ea21
fix:fix test
cx-rogerio-dalot-x Jun 4, 2025
7cd187d
fix: ignore testing secrets
cx-rogerio-dalot-x Jun 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 111 additions & 111 deletions .2ms.yml
View file
Open in desktop

Large diffs are not rendered by default.

41 changes: 23 additions & 18 deletions cmd/workers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,17 @@ package cmd

import (
"context"
"sync"

"github.com/checkmarx/2ms/v3/engine"
"github.com/checkmarx/2ms/v3/engine/extra"
"github.com/checkmarx/2ms/v3/lib/secrets"
"golang.org/x/sync/errgroup"
)

func ProcessItems(engineInstance engine.IEngine, pluginName string) {
defer Channels.WaitGroup.Done()

g, ctx := errgroup.WithContext(context.Background())
g.SetLimit(1000)
for item := range Channels.Items {
Report.TotalItemsScanned++
item := item
Expand Down Expand Up @@ -71,37 +70,43 @@ func ProcessSecretsWithValidation() {
func ProcessSecretsExtras() {
defer Channels.WaitGroup.Done()

wgExtras := &sync.WaitGroup{}
g := errgroup.Group{}
g.SetLimit(10)
for secret := range SecretsExtrasChan {
wgExtras.Add(1)
go extra.AddExtraToSecret(secret, wgExtras)
g.Go(func() error {
extra.AddExtraToSecret(secret)
return nil
})
}
wgExtras.Wait()
_ = g.Wait()
}

func ProcessValidationAndScoreWithValidation(engine engine.IEngine) {
defer Channels.WaitGroup.Done()

wgValidation := &sync.WaitGroup{}
g := errgroup.Group{}
g.SetLimit(10)
for secret := range ValidationChan {
wgValidation.Add(2)
go func(secret *secrets.Secret, wg *sync.WaitGroup) {
engine.RegisterForValidation(secret, wg)
engine.Score(secret, true, wg)
}(secret, wgValidation)
g.Go(func() error {
engine.RegisterForValidation(secret)
engine.Score(secret, true)
return nil
})
}
wgValidation.Wait()

_ = g.Wait()
engine.Validate()
}

func ProcessScoreWithoutValidation(engine engine.IEngine) {
defer Channels.WaitGroup.Done()

wgScore := &sync.WaitGroup{}
g := errgroup.Group{}
g.SetLimit(10)
for secret := range CvssScoreWithoutValidationChan {
wgScore.Add(1)
go engine.Score(secret, false, wgScore)
g.Go(func() error {
engine.Score(secret, false)
return nil
})
}
wgScore.Wait()
_ = g.Wait()
}
11 changes: 4 additions & 7 deletions engine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"os"
"regexp"
"strings"
"sync"
"text/tabwriter"

"github.com/checkmarx/2ms/v3/engine/chunk"
Expand Down Expand Up @@ -45,8 +44,8 @@ type IEngine interface {
DetectFragment(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, pluginName string) error
DetectFile(ctx context.Context, item plugins.ISourceItem, secretsChannel chan *secrets.Secret) error
AddRegexRules(patterns []string) error
RegisterForValidation(secret *secrets.Secret, wg *sync.WaitGroup)
Score(secret *secrets.Secret, validateFlag bool, wg *sync.WaitGroup)
RegisterForValidation(secret *secrets.Secret)
Score(secret *secrets.Secret, validateFlag bool)
Validate()
GetRuleBaseRiskScore(ruleId string) float64
}
Expand Down Expand Up @@ -252,13 +251,11 @@ func (e *Engine) AddRegexRules(patterns []string) error {
return nil
}

func (e *Engine) RegisterForValidation(secret *secrets.Secret, wg *sync.WaitGroup) {
defer wg.Done()
func (e *Engine) RegisterForValidation(secret *secrets.Secret) {
e.validator.RegisterForValidation(secret)
}

func (e *Engine) Score(secret *secrets.Secret, validateFlag bool, wg *sync.WaitGroup) {
defer wg.Done()
func (e *Engine) Score(secret *secrets.Secret, validateFlag bool) {
validationStatus := secrets.UnknownResult // default validity
if validateFlag {
validationStatus = secret.ValidationStatus
Expand Down
17 changes: 8 additions & 9 deletions engine/engine_mock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion engine/engine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ import (
"bytes"
"context"
"fmt"
"go.uber.org/mock/gomock"
"io"
"os"
"path/filepath"
"testing"

"go.uber.org/mock/gomock"

"github.com/checkmarx/2ms/v3/engine/chunk"
"github.com/checkmarx/2ms/v3/engine/rules"
"github.com/checkmarx/2ms/v3/engine/semaphore"
Expand Down
4 changes: 1 addition & 3 deletions engine/extra/extra.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"strings"
"sync"

"github.com/checkmarx/2ms/v3/lib/secrets"
)
Expand All @@ -16,8 +15,7 @@ var ruleIDToFunction = map[string]addExtraFunc{
"jwt": addExtraJWT,
}

func AddExtraToSecret(secret *secrets.Secret, wg *sync.WaitGroup) {
defer wg.Done()
func AddExtraToSecret(secret *secrets.Secret) {
if addExtra, ok := ruleIDToFunction[secret.RuleID]; ok {
extraData := addExtra(secret)
if extraData != nil && extraData != "" {
Expand Down
9 changes: 3 additions & 6 deletions engine/extra/extra_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ package extra
import (
"encoding/base64"
"fmt"
"testing"

"github.com/checkmarx/2ms/v3/lib/secrets"
"github.com/stretchr/testify/assert"
"sync"
"testing"
)

func TestAddExtraToSecret(t *testing.T) {
Expand Down Expand Up @@ -50,10 +50,7 @@ func TestAddExtraToSecret(t *testing.T) {
ExtraDetails: make(map[string]interface{}),
}

var wg sync.WaitGroup
wg.Add(1)
AddExtraToSecret(secret, &wg)
wg.Wait()
AddExtraToSecret(secret)

assert.Equal(t, tt.expectedOutput, secret.ExtraDetails["secretDetails"])
})
Expand Down
9 changes: 5 additions & 4 deletions engine/score/score_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package score_test

import (
"sync"
"testing"

. "github.com/checkmarx/2ms/v3/engine"
"github.com/checkmarx/2ms/v3/engine/rules"
"github.com/checkmarx/2ms/v3/engine/score"
"github.com/checkmarx/2ms/v3/lib/secrets"
"github.com/stretchr/testify/assert"
ruleConfig "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules"
"sync"
"testing"
)

func TestScore(t *testing.T) {
Expand Down Expand Up @@ -216,9 +217,9 @@ func TestScore(t *testing.T) {
expectedRuleScores := expectedCvssScores[secret.RuleID]
validityIndex := getValidityIndex(secret.ValidationStatus)
unknownIndex := getValidityIndex(secrets.UnknownResult)
engine.Score(secret, true, &wg)
engine.Score(secret, true)
assert.Equal(t, expectedRuleScores[validityIndex], secret.CvssScore, "rule: %s", secret.RuleID)
engine.Score(secret, false, &wg)
engine.Score(secret, false)
assert.Equal(t, expectedRuleScores[unknownIndex], secret.CvssScore, "rule: %s", secret.RuleID)
}
}
Expand Down
5 changes: 1 addition & 4 deletions engine/validation/pairs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package validation

import (
"sync"

"github.com/checkmarx/2ms/v3/lib/secrets"
)

Expand Down Expand Up @@ -38,8 +36,7 @@ func (p *pairsCollector) addIfNeeded(secret *secrets.Secret) bool {
return true
}

func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) {
defer wg.Done()
func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId) {
generalKeyToValidation[generalKey](rulesById)
}

Expand Down
7 changes: 1 addition & 6 deletions engine/validation/validator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package validation

import (
"sync"

"github.com/checkmarx/2ms/v3/engine/extra"
"github.com/checkmarx/2ms/v3/lib/secrets"
)
Expand Down Expand Up @@ -35,14 +33,11 @@ func (v *Validator) RegisterForValidation(secret *secrets.Secret) {
}

func (v *Validator) Validate() {
wg := &sync.WaitGroup{}
for generalKey, bySource := range v.pairsCollector.pairs {
for _, byRule := range bySource {
wg.Add(1)
v.pairsCollector.validate(generalKey, byRule, wg)
v.pairsCollector.validate(generalKey, byRule)
}
}
wg.Wait()
}

func IsCanValidateRule(ruleID string) bool {
Expand Down
13 changes: 5 additions & 8 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ require (
github.com/stretchr/testify v1.10.0
github.com/zricethezav/gitleaks/v8 v8.18.2
go.uber.org/mock v0.5.2
golang.org/x/sync v0.12.0
golang.org/x/net v0.40.0
golang.org/x/sync v0.14.0
golang.org/x/time v0.5.0
gopkg.in/yaml.v3 v3.0.1
)
Expand Down Expand Up @@ -48,11 +49,7 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.25.0 // indirect
)

//transitive dependency not applied but also reported as not used by go mod why

replace golang.org/x/net => golang.org/x/net v0.34.0
47 changes: 11 additions & 36 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zricethezav/gitleaks/v8 v8.18.2 h1:slo/sMmgs3qA+6Vv6iqVhsCv+gsl3RekQXqDN0M4g5M=
Expand All @@ -107,51 +106,27 @@ go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
Expand Down
Loading
Loading
0