From 5ad97ea0fb0b9f33f8159be033da19e83b45c67a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:50:21 -0300 Subject: [PATCH 01/13] Bump golang.org/x/sys from 0.28.0 to 0.29.0 (#1248) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 8b2636254..d728a5066 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/mock v0.5.0 golang.org/x/net v0.32.0 - golang.org/x/sys v0.28.0 + golang.org/x/sys v0.29.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.31.3 k8s.io/apimachinery v0.31.3 @@ -62,7 +62,6 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/onsi/ginkgo/v2 v2.21.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -84,7 +83,6 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.27.0 // indirect google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index c62604930..32d538249 100644 --- a/go.sum +++ b/go.sum @@ -310,8 +310,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 6b087c0aff338469955d46dcf9b112f18d4c5177 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:11:32 -0300 Subject: [PATCH 02/13] Bump golang.org/x/crypto from 0.30.0 to 0.31.0 (#1249) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d728a5066..76b8f77f6 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.30.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect diff --git a/go.sum b/go.sum index 32d538249..30c6b9f00 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= From 794e4c402c7045091e111a3bfc97722a2af214ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:08:23 -0300 Subject: [PATCH 03/13] Bump golang.org/x/net from 0.32.0 to 0.34.0 (#1250) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 76b8f77f6..7912881ba 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/vishvananda/netlink v1.3.0 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.5.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.34.0 golang.org/x/sys v0.29.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.31.3 @@ -73,12 +73,12 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/term v0.27.0 // indirect + golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.27.0 // indirect diff --git a/go.sum b/go.sum index 30c6b9f00..f146b8f6d 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -312,8 +312,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From af3e4777f6540b37195be10ccaa5361196b0c599 Mon Sep 17 00:00:00 2001 From: Matthew Sandoval Date: Wed, 15 Jan 2025 09:07:06 -0800 Subject: [PATCH 04/13] Add Pyroscope to config (#1252) --- .golangci.yml | 2 + cmd/config.go | 1 + .../user_guide/configuration_options.rst | 68 ++++++++++ go.mod | 8 +- go.sum | 18 ++- pkg/types/main.go | 117 +++++++++++++++++- 6 files changed, 209 insertions(+), 5 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 22fabb2d6..3f798b887 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -75,6 +75,8 @@ linters-settings: - "k8s.io/api/core" - "k8s.io/apimachinery/pkg" - "k8s.io/client-go" + - "github.com/grafana/pyroscope-go" + - "github.com/sirupsen/logrus" tests: files: - "$test" diff --git a/cmd/config.go b/cmd/config.go index 5f331d32b..8c67c497a 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -218,6 +218,7 @@ func RunConfigV1() { cl := cmdline.NewCmdline() cl.AddConfigType("node", "Specifies the node configuration of this instance", types.NodeCfg{}, cmdline.Required, cmdline.Singleton) cl.AddConfigType("local-only", "Runs a self-contained node with no backend", backends.NullBackendCfg{}, cmdline.Singleton) + cl.AddConfigType("pyroscope-client", "Profile Receptor using Pyroscope, client ", types.ReceptorPyroscopeCfg{}, cmdline.Singleton) // Add registered config types from imported modules for _, appName := range []string{ diff --git a/docs/source/user_guide/configuration_options.rst b/docs/source/user_guide/configuration_options.rst index 4e32b6867..0c2e3b047 100644 --- a/docs/source/user_guide/configuration_options.rst +++ b/docs/source/user_guide/configuration_options.rst @@ -117,6 +117,74 @@ Run qvis locally: - Qvis is now served on port 8080 - Upload qlogs to qvis and navigate to Sequence +^^^^^^^^^^^^^^^^ +Pyroscope Client +^^^^^^^^^^^^^^^^ + +Use pyroscope-client to implement pyroscope push-mode to profile receptor and push to a Pyroscope server. + +.. list-table:: Pyroscope client + :header-rows: 1 + :widths: auto + + * - Parameter + - Description + - Default value + - Type + * - ``applicationName`` + - Name of application used in Pyroscope UI + - No default value. + - string + * - ``tags`` + - Map of static tags. + - No default value. + - map of string + * - ``serverAddress`` + - Address of pyroscope server + - No default value. + - string + * - ``basicAuthUser`` + - Http basic auth user + - No default value. + - string + * - ``basicAuthPassword`` + - Http basic auth password + - No default value. + - string + * - ``tenantID`` + - Specify TenantId when using phlare multi-tenancy + - No default value. + - string + * - ``uploadRate`` + - Upload rate e.g. ``10s`` + - 15s + - string + * - ``profileTypes`` + - Optional profile types- ProfileGoroutines, ProfileMutexCount, ProfileMutexDuration, ProfileBlockCount, ProfileBlockDuration. + - Profile types set by default- ProfileCPU, ProfileAllocObjects, ProfileAlloSpace, ProfileInuseObjects, ProfileInuseSpace. + - list of sting + * - ``disableGCRuns`` + - This will disable automatic runtime.GC runs between getting the heap profiles + - true + - bool + * - ``HTTPHeaders`` + - Set the Authorization header manually + - No default value. + - map of string + + +.. code-block:: yaml + + pyroscope-client: + - applicationName: "receptor" + serverAddress: "http://localhost:4040" + profileTypes: + - ProfileGoroutines + - ProfileMutexCount + - ProfileMutexDuration + - ProfileBlockCount + - ProfileBlockDuration + ^^^^ Node ^^^^ diff --git a/go.mod b/go.mod index 7912881ba..b68fb235e 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/prep/socketpair v0.0.0-20171228153254-c2c6a7f821c2 github.com/quic-go/quic-go v0.48.2 github.com/rogpeppe/go-internal v1.13.1 + github.com/sirupsen/logrus v1.9.3 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 @@ -31,6 +32,8 @@ require ( k8s.io/client-go v0.31.3 ) +require github.com/stretchr/testify v1.10.0 // indirect + require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -47,11 +50,14 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241101162523-b92577c0c142 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grafana/pyroscope-go v1.2.0 + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -82,7 +88,7 @@ require ( golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.27.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index f146b8f6d..9bf2ab2a1 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,10 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -112,6 +116,8 @@ github.com/jupp0r/go-priority-queue v0.0.0-20160601094913-ab1073853bde h1:+5PMaa github.com/jupp0r/go-priority-queue v0.0.0-20160601094913-ab1073853bde/go.mod h1:RDgD/dfPmIwFH0qdUOjw71HjtWg56CtyLIoHL+R1wJw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -201,6 +207,8 @@ github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1l github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= @@ -220,15 +228,18 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -307,6 +318,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -353,8 +365,8 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/types/main.go b/pkg/types/main.go index a3956d1d6..77eccfd52 100644 --- a/pkg/types/main.go +++ b/pkg/types/main.go @@ -5,11 +5,16 @@ import ( "fmt" "os" "regexp" + "runtime" "strings" + "time" "github.com/ansible/receptor/pkg/controlsvc" "github.com/ansible/receptor/pkg/netceptor" "github.com/ansible/receptor/pkg/workceptor" + "github.com/grafana/pyroscope-go" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) type NodeCfg struct { @@ -23,6 +28,8 @@ type NodeCfg struct { ReceptorKubeClientsetRateLimiter string } +var receptorDataDir string + func (cfg NodeCfg) Init() error { var err error if cfg.ID == "" { @@ -46,6 +53,8 @@ func (cfg NodeCfg) Init() error { return fmt.Errorf("node ID \"localhost\" is reserved") } + receptorDataDir = cfg.DataDir + netceptor.MainInstance = netceptor.New(context.Background(), cfg.ID) if len(cfg.FirewallRules) > 0 { @@ -69,7 +78,7 @@ func (cfg NodeCfg) Init() error { } } - workceptor.MainInstance, err = workceptor.New(context.Background(), netceptor.MainInstance, cfg.DataDir) + workceptor.MainInstance, err = workceptor.New(context.Background(), netceptor.MainInstance, receptorDataDir) if err != nil { return err } @@ -87,3 +96,109 @@ func (cfg NodeCfg) Run() error { return nil } + +type ReceptorPyroscopeCfg struct { + ApplicationName string + Tags map[string]string + ServerAddress string // e.g http://pyroscope.services.internal:4040 + BasicAuthUser string // http basic auth user + BasicAuthPassword string // http basic auth password + TenantID string // specify TenantId when using phlare multi-tenancy + UploadRate string + ProfileTypes []string + DisableGCRuns bool // this will disable automatic runtime.GC runs between getting the heap profiles + HTTPHeaders map[string]string +} + +type UploadRate struct { + UploadRate time.Duration `yaml:"uploadRate"` +} + +func (pyroscopeCfg ReceptorPyroscopeCfg) Init() error { + if pyroscopeCfg.ApplicationName == "" { + return nil + } + + runtime.SetMutexProfileFraction(5) + runtime.SetBlockProfileRate(5) + + pyroscopeLogger := logrus.New() + pyroscopeLogger.SetLevel(logrus.DebugLevel) + + if _, err := os.Stat(receptorDataDir); os.IsNotExist(err) { + err := os.MkdirAll(receptorDataDir, 0o700) + if err != nil { + fmt.Printf("error creating directory: %v", err) + } + } + + logFile, err := os.OpenFile(fmt.Sprintf("%s/pyroscope.log", receptorDataDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o666) + if err != nil { + pyroscopeLogger.Fatalf("Error opening log file: %v", err) + } + pyroscopeLogger.SetOutput(logFile) + + pyroscopeLogger.SetFormatter(&logrus.JSONFormatter{}) + + _, err = pyroscope.Start(pyroscope.Config{ + ApplicationName: pyroscopeCfg.ApplicationName, + Tags: pyroscopeCfg.Tags, + ServerAddress: pyroscopeCfg.ServerAddress, + BasicAuthUser: pyroscopeCfg.BasicAuthUser, + BasicAuthPassword: pyroscopeCfg.BasicAuthPassword, + TenantID: pyroscopeCfg.TenantID, + UploadRate: getUploadRate(pyroscopeCfg), + Logger: pyroscopeLogger, + ProfileTypes: getProfileTypes(pyroscopeCfg), + DisableGCRuns: pyroscopeCfg.DisableGCRuns, + HTTPHeaders: pyroscopeCfg.HTTPHeaders, + }) + + if err != nil { + return err + } else { + return nil + } +} + +func getUploadRate(cfg ReceptorPyroscopeCfg) time.Duration { + if cfg.UploadRate == "" { + return 15 * time.Second + } + var uploadRate UploadRate + err := yaml.Unmarshal([]byte(cfg.UploadRate), &uploadRate) + if err != nil { + fmt.Println("failed to parse uploadRate from config file") + } + + return uploadRate.UploadRate +} + +func getProfileTypes(cfg ReceptorPyroscopeCfg) []pyroscope.ProfileType { + profileType := []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + } + if len(cfg.ProfileTypes) == 0 { + return profileType + } + for _, pt := range cfg.ProfileTypes { + switch pt { + case "ProfileGoroutines": + profileType = append(profileType, pyroscope.ProfileGoroutines) + case "ProfileMutexCount": + profileType = append(profileType, pyroscope.ProfileMutexCount) + case "ProfileMutexDuration": + profileType = append(profileType, pyroscope.ProfileMutexDuration) + case "ProfileBlockCount": + profileType = append(profileType, pyroscope.ProfileBlockCount) + case "ProfileBlockDuration": + profileType = append(profileType, pyroscope.ProfileBlockDuration) + } + } + + return profileType +} From b77a2659b7365c277e82bfe4dd49bcd41715516d Mon Sep 17 00:00:00 2001 From: Dan Leehr Date: Tue, 28 Jan 2025 05:42:06 -0500 Subject: [PATCH 05/13] Update example config syntax (#1257) --- .../creating_a_basic_network.rst | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/docs/source/getting_started_guide/creating_a_basic_network.rst b/docs/source/getting_started_guide/creating_a_basic_network.rst index c3daef241..5e85b9b95 100644 --- a/docs/source/getting_started_guide/creating_a_basic_network.rst +++ b/docs/source/getting_started_guide/creating_a_basic_network.rst @@ -24,20 +24,18 @@ Receptor configurations .. code-block:: yaml - --- - version: 2 - node: - id: foo + - node: + id: foo - control-services: - - service: control + - control-service: + service: control filename: /tmp/foo.sock - tcp-peers: - - address: localhost:2222 + - tcp-peer: + address: localhost:2222 - log-level: - level: debug + - log-level: + level: debug ... @@ -46,19 +44,18 @@ Receptor configurations .. code-block:: yaml --- - version: 2 - node: - id: bar + - node: + id: bar - control-services: - - service: control + - control-service: + service: control filename: /tmp/bar.sock - tcp-listeners: - - port: 2222 + - tcp-listener: + port: 2222 - log-level: - level: debug + - log-level: + level: debug ... @@ -67,19 +64,18 @@ Receptor configurations .. code-block:: yaml --- - version: 2 - node: - id: baz + - node: + id: baz - control-services: - - service: control + - control-service: + service: control filename: /tmp/baz.sock - tcp-peers: - - address: localhost:2222 + - tcp-peer: + address: localhost:2222 - log-level: - level: debug + - log-level: + level: debug - work-command: workType: echo From c79eadb9e33d7772d2423f54f67b32a839a6c65a Mon Sep 17 00:00:00 2001 From: a_nackov Date: Tue, 28 Jan 2025 18:16:37 +0200 Subject: [PATCH 06/13] Reference #1258: fix timestamp debug message (#1259) Signed-off-by: Adrian Nackov --- pkg/workceptor/kubernetes.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index c766685ff..b2d6872bf 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -416,12 +416,13 @@ func (kw *KubeUnit) KubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout msg := line timestamp := ParseTime(split[0]) if timestamp != nil { - kw.GetWorkceptor().nc.GetLogger().Debug("No timestamp received, log line: '%s'", line) if !timestamp.After(sinceTime) && !successfulWrite { continue } sinceTime = *timestamp msg = split[1] + } else { + kw.GetWorkceptor().nc.GetLogger().Debug("No timestamp received, log line: '%s'", line) } _, err = stdout.Write([]byte(msg)) From 8b251156bc33c5943810e85b8e8df57853af89ca Mon Sep 17 00:00:00 2001 From: AaronH88 Date: Wed, 29 Jan 2025 14:23:17 -0300 Subject: [PATCH 07/13] Add run and reload tests to null backends (#1256) --- pkg/backends/null_test.go | 61 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/pkg/backends/null_test.go b/pkg/backends/null_test.go index 881204e71..18586a0fe 100644 --- a/pkg/backends/null_test.go +++ b/pkg/backends/null_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/ansible/receptor/pkg/backends" + "github.com/ansible/receptor/pkg/netceptor" ) func TestNullBackendCfgGetAddr(t *testing.T) { @@ -123,3 +124,63 @@ func TestNullBackendCfgGetTLS(t *testing.T) { }) } } + +func TestNullBackendRun(t *testing.T) { + type fields struct { + Local bool + } + tests := []struct { + name string + fields fields + want error + }{ + { + name: "Positive", + fields: fields{ + Local: true, + }, + want: nil, + }, + } + netceptor.MainInstance = netceptor.New(context.Background(), "null_backends_test") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &backends.NullBackendCfg{ + Local: tt.fields.Local, + } + if got := cfg.Run(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NullBackendCfg.Run() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNullBackendReload(t *testing.T) { + type fields struct { + Local bool + } + tests := []struct { + name string + fields fields + want error + }{ + { + name: "Positive", + fields: fields{ + Local: true, + }, + want: nil, + }, + } + netceptor.MainInstance = netceptor.New(context.Background(), "null_backends_test_reload") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &backends.NullBackendCfg{ + Local: tt.fields.Local, + } + if got := cfg.Reload(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NullBackendCfg.Run() = %v, want %v", got, tt.want) + } + }) + } +} From 75eb3712743793914bd7e0bfe6e69b4f8d7bab73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 12:50:13 -0300 Subject: [PATCH 08/13] Bump golang.org/x/sys from 0.29.0 to 0.30.0 (#1262) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b68fb235e..ef42ec89d 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/mock v0.5.0 golang.org/x/net v0.34.0 - golang.org/x/sys v0.29.0 + golang.org/x/sys v0.30.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.31.3 k8s.io/apimachinery v0.31.3 diff --git a/go.sum b/go.sum index 9bf2ab2a1..24e4cf00d 100644 --- a/go.sum +++ b/go.sum @@ -322,8 +322,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 9eb82565882cc2060a72a76288a8186d22e3ffbd Mon Sep 17 00:00:00 2001 From: Matthew Sandoval Date: Thu, 6 Feb 2025 08:55:03 -0800 Subject: [PATCH 09/13] Remove ping_test (#1263) --- pkg/netceptor/ping_test.go | 381 ------------------------------------- 1 file changed, 381 deletions(-) delete mode 100644 pkg/netceptor/ping_test.go diff --git a/pkg/netceptor/ping_test.go b/pkg/netceptor/ping_test.go deleted file mode 100644 index ef9c7f080..000000000 --- a/pkg/netceptor/ping_test.go +++ /dev/null @@ -1,381 +0,0 @@ -package netceptor_test - -import ( - "context" - "errors" - "net" - "testing" - "time" - - "github.com/ansible/receptor/pkg/netceptor" - "github.com/ansible/receptor/pkg/netceptor/mock_netceptor" - "go.uber.org/mock/gomock" -) - -// setupTest sets up TestPing tests. -func setupTest(t *testing.T) (*gomock.Controller, *mock_netceptor.MockNetcForPing, *mock_netceptor.MockPacketConner) { - ctrl := gomock.NewController(t) - - // Prepare mocks - mockNetceptor := mock_netceptor.NewMockNetcForPing(ctrl) - mockPacketConn := mock_netceptor.NewMockPacketConner(ctrl) - - return ctrl, mockNetceptor, mockPacketConn -} - -// createChannel creates a channel that passes an error to errorChan inside of createPing. -func createChannel(mockPacketConn *mock_netceptor.MockPacketConner) { - mockUnreachableMessage := netceptor.UnreachableMessage{ - FromNode: "", - ToNode: "", - FromService: "", - ToService: "", - Problem: "test", - } - - mockUnreachableNotification := netceptor.UnreachableNotification{ - mockUnreachableMessage, - "test", - } - channel := make(chan netceptor.UnreachableNotification) - - mockPacketConn.EXPECT().SubscribeUnreachable(gomock.Any()).Return(channel) - go func() { - channel <- mockUnreachableNotification - }() -} - -// checkPing checks TestPing tests by comparing return values to expected values. -func checkPing(duration time.Duration, expectedDuration int, remote string, expectedRemote string, err error, expectedError error, t *testing.T) { - if expectedError == nil && err != nil { - t.Errorf("Expected no error, got: %v", err) - } else if expectedError != nil && (err == nil || err.Error() != expectedError.Error()) { - t.Errorf("Expected error: %s, got: %v", expectedError.Error(), err) - } - if expectedDuration != int(duration) && expectedDuration != 0 { - t.Errorf("Expected duration to be %v, got: %v", expectedDuration, duration) - } - if expectedRemote != remote && expectedRemote != "" { - t.Errorf("Expected remote to be %v, got: %v", expectedRemote, remote) - } -} - -func setupTestExpects(args ...interface{}) { - mockNetceptor := args[0].(*mock_netceptor.MockNetcForPing) - mockPacketConn := args[1].(*mock_netceptor.MockPacketConner) - testCase := args[2].(pingTestCaseStruct) - - testExpects := map[string]func(){ - "ListenPacketReturn": func() { - mockNetceptor.EXPECT().ListenPacket(gomock.Any()).Return(testCase.returnListenPacket.packetConn, testCase.returnListenPacket.err).Times(testCase.returnListenPacket.times) - }, - "SubscribeUnreachableReturn": func() { - mockPacketConn.EXPECT().SubscribeUnreachable(gomock.Any()).Return(make(chan netceptor.UnreachableNotification)) - }, - "WriteToReturn": func() { - mockPacketConn.EXPECT().WriteTo(gomock.Any(), gomock.Any()).Return(testCase.returnWriteTo.packetLen, testCase.returnWriteTo.err).Times(testCase.returnWriteTo.times) - }, - "ReadFromReturn": func() { - mockPacketConn.EXPECT().ReadFrom(gomock.Any()).Return(0, testCase.returnReadFrom.address, testCase.returnReadFrom.err).MaxTimes(testCase.returnReadFrom.times) - }, - "ReadFromDo": func() { - mockPacketConn.EXPECT().ReadFrom(gomock.Any()).Do(func([]byte) { - time.Sleep(time.Second * 11) - }).Times(testCase.returnReadFrom.times) - }, - "ReadFromDoAndReturn": func() { - mockPacketConn.EXPECT().ReadFrom(gomock.Any()).DoAndReturn(func([]byte) (int, net.Addr, error) { - time.Sleep(time.Second * 2) - - return 0, testCase.returnReadFrom.address, testCase.returnReadFrom.err - }).MaxTimes(testCase.returnReadFrom.times) - }, - "ContextReturn": func() { - mockNetceptor.EXPECT().Context().Return(testCase.returnContext.ctx).MaxTimes(testCase.returnContext.times) - }, - "ContextDoAndReturn": func() { - mockNetceptor.EXPECT().Context().DoAndReturn(func() context.Context { - newCtx, ctxCancel := context.WithCancel(context.Background()) - ctxCancel() - - return newCtx - }).MaxTimes(testCase.returnContext.times) - }, - "SetHopsToLiveReturn": func() { mockPacketConn.EXPECT().SetHopsToLive(gomock.Any()).Times(testCase.returnSetHopsToLiveTimes) }, - "CloseReturn": func() { mockPacketConn.EXPECT().Close().Return(nil).Times(testCase.returnCloseTimes) }, - "NewAddrReturn": func() { - mockNetceptor.EXPECT().NewAddr(gomock.Any(), gomock.Any()).Return(netceptor.Addr{}).Times(testCase.returnNewAddrTimes) - }, - "NodeID": func() { mockNetceptor.EXPECT().NodeID().Return("nodeID") }, - "CreateChannel": func() { createChannel(mockPacketConn) }, - "SleepOneSecond": func() { time.Sleep(time.Second * 1) }, - } - - for _, expect := range testCase.expects { - testExpects[expect]() - } -} - -type listenPacketReturn struct { - packetConn netceptor.PacketConner - err error - times int - returnType string -} - -type writeToReturn struct { - packetLen int - err error - times int - returnType string -} - -type contextReturn struct { - ctx context.Context - times int - returnType string -} - -type readFromReturn struct { - data int - address net.Addr - err error - times int - returnType string -} - -type pingTestCaseStruct struct { - name string - pingTarget string - pingHopsToLive byte - returnSetHopsToLiveTimes int - returnCloseTimes int - returnNewAddrTimes int - returnListenPacket listenPacketReturn - returnWriteTo writeToReturn - returnContext contextReturn - returnReadFrom readFromReturn - expects []string - setupTestExpects func(args ...interface{}) - expectedDuration int - expectedRemote string - expectedError error -} - -// TestCreatePing tests CreatePing inside ping.go. -func TestCreatePing(t *testing.T) { - ctrl, mockNetceptor, mockPacketConn := setupTest(t) - - pingTestCases := []pingTestCaseStruct{ - { - "NetceptorShutdown Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, nil, 1, "return"}, - contextReturn{context.Background(), 2, "doAndReturn"}, - readFromReturn{0, nil, nil, 1, "return"}, - []string{"ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "SubscribeUnreachableReturn", "WriteToReturn", "ReadFromReturn", "ContextDoAndReturn", "SleepOneSecond"}, - setupTestExpects, - 0, - "", - errors.New("netceptor shutdown"), - }, - { - "SubscribeUnreachable Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, nil, 1, "return"}, - contextReturn{context.Background(), 2, "return"}, - readFromReturn{0, nil, nil, 1, "return"}, - []string{"CreateChannel", "ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "WriteToReturn", "ReadFromDoAndReturn", "ContextReturn"}, - setupTestExpects, - 0, - "", - errors.New("test"), - }, - { - "CreatePing Success", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, nil, 1, "return"}, - contextReturn{context.Background(), 2, "return"}, - readFromReturn{0, &netceptor.Addr{}, nil, 1, "return"}, - []string{"ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "SubscribeUnreachableReturn", "WriteToReturn", "ReadFromReturn", "ContextReturn"}, - setupTestExpects, - 0, - ":", - nil, - }, - { - "ListenPacket Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{nil, errors.New("Catch ListenPacket error"), 1, "return"}, - writeToReturn{0, nil, 0, "return"}, - contextReturn{context.Background(), 0, "return"}, - readFromReturn{0, &netceptor.Addr{}, nil, 0, "return"}, - []string{"ListenPacketReturn"}, - setupTestExpects, - 0, - "", - errors.New("Catch ListenPacket error"), - }, - { - "ReadFrom Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, nil, 1, "return"}, - contextReturn{context.Background(), 2, "return"}, - readFromReturn{0, nil, errors.New("ReadFrom error"), 1, "return"}, - []string{"ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "SubscribeUnreachableReturn", "WriteToReturn", "ReadFromReturn", "ContextReturn"}, - setupTestExpects, - 0, - "", - errors.New("ReadFrom error"), - }, - { - "WriteTo Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, errors.New("WriteTo error"), 1, "return"}, - contextReturn{context.Background(), 2, "return"}, - readFromReturn{0, nil, nil, 1, "return"}, - []string{"ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "SubscribeUnreachableReturn", "WriteToReturn", "ReadFromReturn", "ContextReturn", "NodeID"}, - setupTestExpects, - 0, - "", - errors.New("WriteTo error"), - }, - { - "Timeout Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, nil, 1, "return"}, - contextReturn{context.Background(), 2, "return"}, - readFromReturn{0, nil, nil, 1, "do"}, - []string{"ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "SubscribeUnreachableReturn", "WriteToReturn", "ReadFromDo", "ContextReturn"}, - setupTestExpects, - 0, - "", - errors.New("timeout"), - }, - { - "User Cancel Error", - "target", - byte(1), - 1, - 1, - 1, - listenPacketReturn{mockPacketConn, nil, 1, "return"}, - writeToReturn{0, nil, 1, "return"}, - contextReturn{context.Background(), 2, "return"}, - readFromReturn{0, nil, nil, 1, "doAndReturn"}, - []string{"ListenPacketReturn", "SetHopsToLiveReturn", "CloseReturn", "NewAddrReturn", "SubscribeUnreachableReturn", "WriteToReturn", "ReadFromDoAndReturn", "ContextReturn"}, - setupTestExpects, - 0, - "", - errors.New("user cancelled"), - }, - } - - for _, testCase := range pingTestCases { - ctx := context.Background() - t.Run(testCase.name, func(t *testing.T) { - testCase.setupTestExpects(mockNetceptor, mockPacketConn, testCase) - if testCase.name == "NetceptorShutdown Error" { - time.Sleep(time.Second * 1) - } - if testCase.name == "User Cancel Error" { - newCtx, ctxCancel := context.WithCancel(ctx) - - time.AfterFunc(1*time.Second, ctxCancel) - - duration, remote, err := netceptor.SendPing(newCtx, mockNetceptor, testCase.pingTarget, testCase.pingHopsToLive) - checkPing(duration, testCase.expectedDuration, remote, testCase.expectedRemote, err, testCase.expectedError, t) - } else { - duration, remote, err := netceptor.SendPing(ctx, mockNetceptor, testCase.pingTarget, testCase.pingHopsToLive) - checkPing(duration, testCase.expectedDuration, remote, testCase.expectedRemote, err, testCase.expectedError, t) - } - - ctrl.Finish() - ctx.Done() - }) - } -} - -type pingReturn struct { - duration time.Duration - remote string - err error -} - -type expectedResult struct { - from string - time time.Duration - err error -} - -// TestCreateTraceroute tests CreateTraceroute inside ping.go. -func TestCreateTraceroute(t *testing.T) { - ctrl := gomock.NewController(t) - - mockNetceptor := mock_netceptor.NewMockNetcForTraceroute(ctrl) - ctx := context.Background() - defer ctx.Done() - - createTracerouteTestCases := []struct { - name string - createTracerouteTarget string - returnPing pingReturn - expectedResult expectedResult - }{ - {"CreateTraceroute Success", "target", pingReturn{time.Since(time.Now()), "target", nil}, expectedResult{":", time.Since(time.Now()), nil}}, - {"CreateTraceroute Error", "target", pingReturn{time.Since(time.Now()), "target", errors.New("traceroute error")}, expectedResult{":", time.Since(time.Now()), errors.New("traceroute error")}}, - } - - for _, testCase := range createTracerouteTestCases { - t.Run(testCase.name, func(t *testing.T) { - mockNetceptor.EXPECT().Context().Return(context.Background()) - mockNetceptor.EXPECT().MaxForwardingHops().Return(byte(1)) - mockNetceptor.EXPECT().Ping(ctx, testCase.createTracerouteTarget, byte(0)).Return(testCase.returnPing.duration, testCase.returnPing.remote, testCase.returnPing.err) - - result := netceptor.CreateTraceroute(ctx, mockNetceptor, testCase.createTracerouteTarget) - for res := range result { - if testCase.expectedResult.err == nil && res.Err != nil { - t.Errorf("Expected no error, got: %v", res.Err.Error()) - } else if testCase.expectedResult.err != nil && (res.Err == nil || res.Err.Error() != testCase.expectedResult.err.Error()) { - t.Errorf("Expected error: %s, got: %v", testCase.expectedResult.err.Error(), res.Err) - } - } - - ctrl.Finish() - }) - } -} From ce9c65caf357aa343c458ab908f84b672f8b9537 Mon Sep 17 00:00:00 2001 From: Matthew Sandoval Date: Fri, 7 Feb 2025 01:27:09 -0800 Subject: [PATCH 10/13] Add lock to fix flaky test (#1261) --- pkg/workceptor/command.go | 2 +- pkg/workceptor/kubernetes.go | 36 +++++++++++++++++++++++++++++-- pkg/workceptor/kubernetes_test.go | 1 + pkg/workceptor/remote_work.go | 2 +- pkg/workceptor/workunitbase.go | 2 ++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pkg/workceptor/command.go b/pkg/workceptor/command.go index 0db29f6c1..fd174f994 100644 --- a/pkg/workceptor/command.go +++ b/pkg/workceptor/command.go @@ -250,13 +250,13 @@ func (cw *commandUnit) Status() *StatusFileData { // UnredactedStatus returns a copy of the status currently loaded in memory, including secrets. func (cw *commandUnit) UnredactedStatus() *StatusFileData { cw.GetStatusLock().RLock() - defer cw.GetStatusLock().RUnlock() status := cw.GetStatusWithoutExtraData() ed, ok := cw.GetStatusCopy().ExtraData.(*CommandExtraData) if ok { edCopy := *ed status.ExtraData = &edCopy } + cw.GetStatusLock().RUnlock() return status } diff --git a/pkg/workceptor/kubernetes.go b/pkg/workceptor/kubernetes.go index b2d6872bf..a1f9d2d82 100644 --- a/pkg/workceptor/kubernetes.go +++ b/pkg/workceptor/kubernetes.go @@ -262,7 +262,9 @@ func (kw *KubeUnit) kubeLoggingConnectionHandler(timestamps bool, sinceTime time podOptions.SinceTime = &metav1.Time{Time: sinceTime} } + KubeAPIWrapperLock.Lock() logReq := KubeAPIWrapperInstance.GetLogs(kw.clientset, podNamespace, podName, podOptions) + KubeAPIWrapperLock.Unlock() // get logstream, with retry for retries := 5; retries > 0; retries-- { logStream, err = logReq.Stream(kw.GetContext()) @@ -333,7 +335,9 @@ func (kw *KubeUnit) KubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout // get pod, with retry for retries := 5; retries > 0; retries-- { + KubeAPIWrapperLock.Lock() kw.pod, err = KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{}) + KubeAPIWrapperLock.Unlock() if err == nil { break } @@ -522,8 +526,10 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error { pod.Spec.Containers[0].Env = evs } + KubeAPIWrapperLock.Lock() // get pod and store to kw.pod kw.pod, err = KubeAPIWrapperInstance.Create(kw.GetContext(), kw.clientset, ked.KubeNamespace, pod, metav1.CreateOptions{}) + KubeAPIWrapperLock.Unlock() if err != nil { return err } @@ -541,8 +547,8 @@ func (kw *KubeUnit) CreatePod(env map[string]string) error { status.ExtraData.(*KubeExtraData).PodName = kw.pod.Name }) - // Wait for the pod to be running KubeAPIWrapperLock.Lock() + // Wait for the pod to be running fieldSelector := KubeAPIWrapperInstance.OneTermEqualSelector("metadata.name", kw.pod.Name).String() KubeAPIWrapperLock.Unlock() lw := &cache.ListWatch{ @@ -684,7 +690,9 @@ func (kw *KubeUnit) runWorkUsingLogger() { default: } + KubeAPIWrapperLock.Lock() kw.pod, err = KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{}) + KubeAPIWrapperLock.Unlock() if err == nil { break } @@ -709,7 +717,9 @@ func (kw *KubeUnit) runWorkUsingLogger() { // Attach stdin stream to the pod var exec remotecommand.Executor if !skipStdin { + KubeAPIWrapperLock.Lock() req := KubeAPIWrapperInstance.SubResource(kw.clientset, podName, podNamespace) + KubeAPIWrapperLock.Unlock() req.VersionedParams( &corev1.PodExecOptions{ @@ -722,7 +732,9 @@ func (kw *KubeUnit) runWorkUsingLogger() { scheme.ParameterCodec, ) var err error + KubeAPIWrapperLock.Lock() exec, err = KubeAPIWrapperInstance.NewSPDYExecutor(kw.config, "POST", req.URL()) + KubeAPIWrapperLock.Unlock() if err != nil { errMsg := fmt.Sprintf("Error creating SPDY executor: %s", err) kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0) @@ -816,10 +828,12 @@ func (kw *KubeUnit) runWorkUsingLogger() { var err error for retries := 5; retries > 0; retries-- { + KubeAPIWrapperLock.Lock() err = KubeAPIWrapperInstance.StreamWithContext(kw.GetContext(), exec, remotecommand.StreamOptions{ Stdin: stdin, Tty: false, }) + KubeAPIWrapperLock.Unlock() if err != nil { // NOTE: io.EOF for stdin is handled by remotecommand and will not trigger this kw.GetWorkceptor().nc.GetLogger().Warning( @@ -1194,8 +1208,12 @@ func (kw *KubeUnit) connectUsingKubeconfig() error { var err error ked := kw.UnredactedStatus().ExtraData.(*KubeExtraData) if ked.KubeConfig == "" { + KubeAPIWrapperLock.Lock() clr := KubeAPIWrapperInstance.NewDefaultClientConfigLoadingRules() + KubeAPIWrapperLock.Unlock() + KubeAPIWrapperLock.Lock() kw.config, err = KubeAPIWrapperInstance.BuildConfigFromFlags("", clr.GetDefaultFilename()) + KubeAPIWrapperLock.Unlock() if ked.KubeNamespace == "" { c, err := clr.Load() if err != nil { @@ -1211,7 +1229,9 @@ func (kw *KubeUnit) connectUsingKubeconfig() error { } } } else { + KubeAPIWrapperLock.Lock() cfg, err := KubeAPIWrapperInstance.NewClientConfigFromBytes([]byte(ked.KubeConfig)) + KubeAPIWrapperLock.Unlock() if err != nil { return err } @@ -1238,7 +1258,9 @@ func (kw *KubeUnit) connectUsingKubeconfig() error { func (kw *KubeUnit) connectUsingIncluster() error { var err error + KubeAPIWrapperLock.Lock() kw.config, err = KubeAPIWrapperInstance.InClusterConfig() + KubeAPIWrapperLock.Unlock() if err != nil { return err } @@ -1330,16 +1352,22 @@ func (kw *KubeUnit) connectToKube() error { if ok { switch envRateLimiter { case "never": + KubeAPIWrapperLock.Lock() kw.config.RateLimiter = KubeAPIWrapperInstance.NewFakeNeverRateLimiter() + KubeAPIWrapperLock.Unlock() case "always": + KubeAPIWrapperLock.Lock() kw.config.RateLimiter = KubeAPIWrapperInstance.NewFakeAlwaysRateLimiter() + KubeAPIWrapperLock.Unlock() default: } kw.GetWorkceptor().nc.GetLogger().Debug("RateLimiter: %s", envRateLimiter) } kw.GetWorkceptor().nc.GetLogger().Debug("QPS: %f, Burst: %d", kw.config.QPS, kw.config.Burst) + KubeAPIWrapperLock.Lock() kw.clientset, err = KubeAPIWrapperInstance.NewForConfig(kw.config) + KubeAPIWrapperLock.Unlock() if err != nil { return err } @@ -1463,13 +1491,13 @@ func (kw *KubeUnit) Status() *StatusFileData { // Status returns a copy of the status currently loaded in memory. func (kw *KubeUnit) UnredactedStatus() *StatusFileData { kw.GetStatusLock().RLock() - defer kw.GetStatusLock().RUnlock() status := kw.GetStatusWithoutExtraData() ked, ok := kw.GetStatusCopy().ExtraData.(*KubeExtraData) if ok { kedCopy := *ked status.ExtraData = &kedCopy } + kw.GetStatusLock().RUnlock() return status } @@ -1508,7 +1536,9 @@ func (kw *KubeUnit) Restart() error { if err != nil { kw.GetWorkceptor().nc.GetLogger().Warning("Pod %s could not be deleted: %s", ked.PodName, err.Error()) } else { + KubeAPIWrapperLock.Lock() err := KubeAPIWrapperInstance.Delete(context.Background(), kw.clientset, ked.KubeNamespace, ked.PodName, metav1.DeleteOptions{}) + KubeAPIWrapperLock.Unlock() if err != nil { kw.GetWorkceptor().nc.GetLogger().Warning("Pod %s could not be deleted: %s", ked.PodName, err.Error()) } @@ -1533,7 +1563,9 @@ func (kw *KubeUnit) Cancel() error { kw.CancelContext() kw.UpdateBasicStatus(WorkStateCanceled, "Canceled", -1) if kw.pod != nil { + KubeAPIWrapperLock.Lock() err := KubeAPIWrapperInstance.Delete(context.Background(), kw.clientset, kw.pod.Namespace, kw.pod.Name, metav1.DeleteOptions{}) + KubeAPIWrapperLock.Unlock() if err != nil { kw.GetWorkceptor().nc.GetLogger().Error("Error deleting pod %s: %s", kw.pod.Name, err) } diff --git a/pkg/workceptor/kubernetes_test.go b/pkg/workceptor/kubernetes_test.go index 48c7f34e8..1c66371a3 100644 --- a/pkg/workceptor/kubernetes_test.go +++ b/pkg/workceptor/kubernetes_test.go @@ -546,6 +546,7 @@ func TestKubeLoggingWithReconnect(t *testing.T) { t.Run(tt.name, func(t *testing.T) { tt.expectedCalls() ku.Start() + time.Sleep(10 * time.Millisecond) kw.CreatePod(nil) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/pkg/workceptor/remote_work.go b/pkg/workceptor/remote_work.go index 7aec7773b..105d7df8a 100644 --- a/pkg/workceptor/remote_work.go +++ b/pkg/workceptor/remote_work.go @@ -547,7 +547,6 @@ func (rw *remoteUnit) Status() *StatusFileData { // UnredactedStatus returns a copy of the status currently loaded in memory, including secrets. func (rw *remoteUnit) UnredactedStatus() *StatusFileData { rw.GetStatusLock().RLock() - defer rw.GetStatusLock().RUnlock() status := rw.GetStatusWithoutExtraData() ed, ok := rw.GetStatusCopy().ExtraData.(*RemoteExtraData) if ok { @@ -558,6 +557,7 @@ func (rw *remoteUnit) UnredactedStatus() *StatusFileData { } status.ExtraData = &edCopy } + rw.GetStatusLock().RUnlock() return status } diff --git a/pkg/workceptor/workunitbase.go b/pkg/workceptor/workunitbase.go index 476b7ba0d..4989486eb 100644 --- a/pkg/workceptor/workunitbase.go +++ b/pkg/workceptor/workunitbase.go @@ -402,7 +402,9 @@ func (bwu *BaseWorkUnit) MonitorLocalStatus() { watcherErrors = make(chan error) if bwu.watcher != nil { + bwu.statusLock.Lock() err := bwu.watcher.Add(statusFile) + bwu.statusLock.Unlock() if err == nil { defer func() { werr := bwu.watcher.Close() From 239498094cef60a1a178f8844c3e78f55fd6b481 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:22:59 +0000 Subject: [PATCH 11/13] Bump wntrblm/nox from 2024.10.09 to 2025.02.09 (#1265) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pull_request.yml | 2 +- .github/workflows/reusable-nox.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f65777e95..72463410a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -189,7 +189,7 @@ jobs: run: sudo chmod a+x /usr/local/bin/receptor - name: Set up nox - uses: wntrblm/nox@2024.10.09 + uses: wntrblm/nox@2025.02.09 with: python-versions: ${{ env.DESIRED_PYTHON_VERSION }} diff --git a/.github/workflows/reusable-nox.yml b/.github/workflows/reusable-nox.yml index c270022a8..8dd6f5e6c 100644 --- a/.github/workflows/reusable-nox.yml +++ b/.github/workflows/reusable-nox.yml @@ -45,7 +45,7 @@ jobs: run: sudo chmod a+x /usr/local/bin/receptor - name: Set up nox - uses: wntrblm/nox@2024.10.09 + uses: wntrblm/nox@2025.02.09 with: python-versions: ${{ inputs.python-version }} From 2510281eb2804ae1020f00b49175fb694781491a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:43:50 +0000 Subject: [PATCH 12/13] Bump golang.org/x/net from 0.34.0 to 0.35.0 (#1266) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index ef42ec89d..f8cc90f45 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/vishvananda/netlink v1.3.0 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.5.0 - golang.org/x/net v0.34.0 + golang.org/x/net v0.35.0 golang.org/x/sys v0.30.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.31.3 @@ -79,13 +79,13 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.32.0 // indirect + golang.org/x/crypto v0.33.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/term v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.27.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/go.sum b/go.sum index 24e4cf00d..2604061e7 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= @@ -293,8 +293,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -309,8 +309,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -324,13 +324,13 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= From 122aa3e74aae8929741a7f6c7629237d8b2bf66b Mon Sep 17 00:00:00 2001 From: Matthew Sandoval Date: Mon, 17 Feb 2025 08:08:39 -0800 Subject: [PATCH 13/13] Close all open files, clean up fsnotify watcher (#1268) --- pkg/netceptor/conn.go | 6 ++++++ pkg/workceptor/command.go | 8 ++++++++ pkg/workceptor/mock_workceptor/workunitbase.go | 14 ++++++++++++++ pkg/workceptor/remote_work.go | 12 ++++++++++++ pkg/workceptor/workunitbase.go | 12 +++++++++--- pkg/workceptor/workunitbase_test.go | 1 + 6 files changed, 50 insertions(+), 3 deletions(-) diff --git a/pkg/netceptor/conn.go b/pkg/netceptor/conn.go index bf566c86c..07438a740 100644 --- a/pkg/netceptor/conn.go +++ b/pkg/netceptor/conn.go @@ -151,6 +151,12 @@ func (s *Netceptor) tracer(ctx context.Context, p logging.Perspective, connID qu return nil } + defer func() { + err := f.Close() + if err != nil { + s.GetLogger().Error("Error closing %s: %s", qlogPath+filename, err) + } + }() return qlog.NewConnectionTracer(f, p, connID) } else { diff --git a/pkg/workceptor/command.go b/pkg/workceptor/command.go index fd174f994..ff9df35be 100644 --- a/pkg/workceptor/command.go +++ b/pkg/workceptor/command.go @@ -209,6 +209,14 @@ loop: MainInstance.nc.GetLogger().Error("Error updating status file %s: %s", statusFilename, err) } } + err = stdin.Close() + if err != nil { + MainInstance.nc.GetLogger().Error("Error closing %s: %s", path.Join(unitdir, "stdin"), err) + } + err = stdout.Close() + if err != nil { + MainInstance.nc.GetLogger().Error("Error closing %s: %s", path.Join(unitdir, "stdout"), err) + } os.Exit(cmd.ProcessState.ExitCode()) return nil diff --git a/pkg/workceptor/mock_workceptor/workunitbase.go b/pkg/workceptor/mock_workceptor/workunitbase.go index 815a6d88d..4e72c5466 100644 --- a/pkg/workceptor/mock_workceptor/workunitbase.go +++ b/pkg/workceptor/mock_workceptor/workunitbase.go @@ -95,3 +95,17 @@ func (mr *MockWatcherWrapperMockRecorder) EventChannel() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EventChannel", reflect.TypeOf((*MockWatcherWrapper)(nil).EventChannel)) } + +// Remove mocks base method. +func (m *MockWatcherWrapper) Remove(path string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remove", path) + ret0, _ := ret[0].(error) + return ret0 +} + +// Remove indicates an expected call of Remove. +func (mr *MockWatcherWrapperMockRecorder) Remove(path any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockWatcherWrapper)(nil).Remove), path) +} diff --git a/pkg/workceptor/remote_work.go b/pkg/workceptor/remote_work.go index 105d7df8a..bf69680d5 100644 --- a/pkg/workceptor/remote_work.go +++ b/pkg/workceptor/remote_work.go @@ -198,6 +198,12 @@ func (rw *remoteUnit) startRemoteUnit(ctx context.Context, conn net.Conn, reader if err != nil { return fmt.Errorf("error opening stdin file: %s", err) } + defer func() { + err := stdin.Close() + if err != nil { + MainInstance.nc.GetLogger().Error("Error closing %s: %s", path.Join(rw.UnitDir(), "stdin"), err) + } + }() _, err = io.Copy(conn, stdin) if err != nil { return fmt.Errorf("error sending stdin file: %s", err) @@ -467,6 +473,12 @@ func (rw *remoteUnit) monitorRemoteStdout(mw *utils.JobContext) { return } + defer func() { + err := stdout.Close() + if err != nil { + MainInstance.nc.GetLogger().Error("Error closing %s: %s", rw.StdoutFileName(), err) + } + }() doneChan := make(chan struct{}) go func() { select { diff --git a/pkg/workceptor/workunitbase.go b/pkg/workceptor/workunitbase.go index 4989486eb..0e9a17da2 100644 --- a/pkg/workceptor/workunitbase.go +++ b/pkg/workceptor/workunitbase.go @@ -36,6 +36,7 @@ const ( // WatcherWrapper is wrapping the fsnofity Watcher struct and exposing the Event chan within. type WatcherWrapper interface { Add(name string) error + Remove(path string) error Close() error ErrorChannel() chan error EventChannel() chan fsnotify.Event @@ -49,6 +50,10 @@ func (rw *RealWatcher) Add(name string) error { return rw.watcher.Add(name) } +func (rw *RealWatcher) Remove(path string) error { + return rw.watcher.Remove(path) +} + func (rw *RealWatcher) Close() error { return rw.watcher.Close() } @@ -407,9 +412,10 @@ func (bwu *BaseWorkUnit) MonitorLocalStatus() { bwu.statusLock.Unlock() if err == nil { defer func() { - werr := bwu.watcher.Close() - if werr != nil { - bwu.w.nc.GetLogger().Error("Error in defer closing %s: %s", statusFile, err) + bwu.watcher.Remove(statusFile) + err = bwu.watcher.Close() + if err != nil { + bwu.w.nc.GetLogger().Error("Error closing watcher: %v", err) } }() watcherEvents = bwu.watcher.EventChannel() diff --git a/pkg/workceptor/workunitbase_test.go b/pkg/workceptor/workunitbase_test.go index 7a62fc3cc..8f593eb02 100644 --- a/pkg/workceptor/workunitbase_test.go +++ b/pkg/workceptor/workunitbase_test.go @@ -389,6 +389,7 @@ func TestMonitorLocalStatus(t *testing.T) { mockFileSystem.EXPECT().Stat(gomock.Any()).Return(tc.statObjLater, nil).AnyTimes() } mockWatcher.EXPECT().Add(gomock.Any()).Return(tc.addWatcherErr) + mockWatcher.EXPECT().Remove(gomock.Any()).AnyTimes() mockWatcher.EXPECT().Close().AnyTimes() if tc.fsNotifyEvent != nil {