8000 feat(api): dump heap profile (#4630) · ovh/cds@ed18a99 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit ed18a99

Browse files
fsaminyesnault
fsamin
authored and
yesnault
committed
feat(api): dump heap profile (#4630)
* feat(api): dump heap profile set CDS_MAX_HEAP_SIZE in bytes
1 parent ef6b8a3 commit ed18a99

File tree

6 files changed

+81
-13
lines changed

6 files changed

+81
-13
lines changed

‎engine/api/api.go

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
package api
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
78
"io"
9+
"io/ioutil"
810
"net/http"
911
"os"
12+
"os/signal"
13+
"runtime/pprof"
1014
"strings"
1115
"time"
1216

@@ -662,7 +666,7 @@ func (a *API) Serve(ctx context.Context) error {
662666
log.Error("unable to init api metrics: %v", err)
663667
}
664668

665-
//Intialize notification package
669+
// Intialize notification package
666670
notification.Init(a.Config.URL.UI)
667671

668672
log.Info("Initializing Authentication driver...")
@@ -857,6 +861,23 @@ func (a *API) Serve(ctx context.Context) error {
857861
return sdk.WrapError(err, "Cannot upsert cds version")
858862
}
859863

864+
// Dump heap to objecstore on SIGINFO
865+
siginfoChan := make(chan os.Signal, 1)
866+
signal.Notify(siginfoChan, sdk.SIGINFO)
867+
go func() {
868+
<-siginfoChan
869+
signal.Stop(siginfoChan)
870+
var buffer = new(bytes.Buffer)
871+
pprof.Lookup("heap").WriteTo(buffer, 1)
872+
var heapProfile = heapProfile{uuid: sdk.UUID()}
873+
s, err := a.SharedStorage.Store(heapProfile, ioutil.NopCloser(buffer))
874+
if err != nil {
875+
log.Error("unable to upload heap profile: %v", err)
876+
return
877+
}
878+
log.Error("api> heap dump uploaded to %s", s)
879+
}()
880+
860881
log.Info("Starting CDS API HTTP Server on %s:%d", a.Config.HTTP.Addr, a.Config.HTTP.Port)
861882
if err := s.ListenAndServe(); err != nil {
862883
return fmt.Errorf("Cannot start HTTP server: %v", err)
@@ -872,3 +893,17 @@ func (a *API) PanicDump() func(s string) (io.WriteCloser, error) {
872893
return cache.NewWriteCloser(a.Cache, cache.Key("api", "panic_dump", s), panicDumpTTL), nil
873894
}
874895
}
896+
897+
type heapProfile struct {
898+
uuid string
899+
}
900+
901+
var _ objectstore.Object = new(heapProfile)
902+
903+
func (p heapProfile) GetName() string {
904+
return p.uuid
905+
}
906+
func (p heapProfile) GetPath() string {
907+
hostname, _ := os.Hostname()
908+
return fmt.Sprintf("api-heap-profile-%d-%s", time.Now().Unix(), hostname)
909+
}

‎engine/service/metrics.go

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,16 @@ import (
55
"fmt"
66
"os"
77
"runtime"
8+
"strconv"
9+
"sync"
810
"time"
911

10-
"github.com/ovh/cds/sdk"
1112
"go.opencensus.io/stats"
1213
"go.opencensus.io/stats/view"
1314
"go.opencensus.io/tag"
15+
16+
"github.com/ovh/cds/sdk"
17+
"github.com/ovh/cds/sdk/log"
1418
)
1519

1620
func CommonMetricsView(ctx context.Context) []*view.View {
@@ -76,6 +80,13 @@ func CommonMetricsView(ctx context.Context) []*view.View {
7680
hostname, _ := os.Hostname()
7781
ctx, _ = tag.New(ctx, tag.Upsert(tagHostname, hostname))
7882

83+
var maxMemoryS = os.Getenv("CDS_MAX_HEAP_SIZE") // in bytes
84+
var maxMemory uint64
85+
var onceMaxMemorySignal = new(sync.Once)
86+
if maxMemoryS != "" {
87+
maxMemory, _ = strconv.ParseUint(maxMemoryS, 10, 64)
88+
}
89+
7990
var tick = time.NewTicker(10 * time.Second)
8091
defer tick.Stop()
8192
for {
@@ -89,6 +100,21 @@ func CommonMetricsView(ctx context.Context) []*view.View {
89100
stats.Record(ctx, totalAllocStats.M(int64(m.TotalAlloc)))
90101
stats.Record(ctx, sysStats.M(int64(m.Sys)))
91102
stats.Record(ctx, gcStats.M(int64(m.NumGC)))
103+
104+
if maxMemory > 0 && m.Alloc >= maxMemory {
105+
onceMaxMemorySignal.Do(func() {
106+
p, err := os.FindProcess(os.Getpid())
107+
if err != nil {
108+
log.Error("unable to find current process: %v", err)
109+
return
110+
}
111+
if err := p.Signal(sdk.SIGINFO); err != nil {
112+
log.Error("unable to send signal: %v", err)
113+
return
114+
}
115+
log.Info("metrics> SIGINFO signal send to %v", os.Getpid())
116+
})
117+
}
92118
}
93119
}
94120
})

‎go.mod

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ require (
1313
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
1414
github.com/SermoDigital/jose v0.9.1 // indirect
1515
github.com/Shopify/sarama v1.19.0
16-
github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect
1716
github.com/alecthomas/jsonschema v0.0.0-20190429041900-eff3f6c90428
1817
github.com/andygrunwald/go-gerrit v0.0.0-20181207071854-19ef3e9332a4
1918
github.com/araddon/gou v0.0.0-20180315155215-820e9f87cd05 // indirect
@@ -59,8 +58,6 @@ require (
5958
github.com/dsnet/compress v0.0.0-20171208185109-cc9eb1d7ad76 // indirect
6059
github.com/duosecurity/duo_api_golang v0.0.0-20180315112207-d0530c80e49a // indirect
6160
github.com/eapache/go-resiliency v1.1.0
62-
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
63-
github.com/eapache/queue v1.1.0 // indirect
6461
github.com/fatih/color v0.0.0-20161018201348-a360acfe359f
6562
github.com/fatih/structs v1.0.0
6663
github.com/fernet/fernet-go v0.0.0-20151007213151-1b2437bc582b // indirect
@@ -79,10 +76,8 @@ require (
7976
github.com/go-sql-driver/mysql v0.0.0-20180113200744-2cc627ac8def // indirect
8077
github.com/go-stomp/stomp v1.0.1 // indirect
8178
github.com/gocql/gocql v0.0.0-20181018123354-22229812a83e // indirect
82-
github.com/gogo/protobuf v1.2.0 // indirect
8379
github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 // indirect
8480
github.com/golang/protobuf v1.3.1
85-
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
8681
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a // indirect
8782
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
8883
github.com/google/gops v0.0.0-20170728214508-806455e841dc
@@ -167,8 +162,6 @@ require (
167162
github.com/nwaples/rardecode v1.0.0 // indirect
168163
github.com/olekukonko/tablewriter v0.0.0-20160621093029-daf2955e742c
169164
github.com/olivere/elastic v6.2.17+incompatible // indirect
170-
github.com/onsi/ginkgo v1.7.0 // indirect
171-
github.com/onsi/gomega v1.4.3 // indirect
172165
github.com/opencontainers/go-digest v1.0.0-rc1
173166
github.com/opencontainers/image-spec v1.0.1 // indirect
174167
github.com/opencontainers/runc v0.0.0-20180718062236-bc1467269fce // indirect
@@ -185,12 +178,9 @@ require (
185178
github.com/pelletier/go-toml v0.0.0-20171222114548-0131db6d737c
186179
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
187180
github.com/phayes/permbits v0.0.0-20160816172359-32459f104210
188-
github.com/pierrec/lz4 v2.0.5+incompatible // indirect
189181
github.com/pkg/browser v0.0.0-20170505125900-c90ca0c84f15
190182
github.com/pkg/errors v0.8.0
191183
github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6 // indirect
192-
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
193-
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
194184
github.com/rubenv/sql-migrate v0.0.0-20160620083229-6f4757563362
195185
github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect
196186
github.com/satori/go.uuid v1.2.0 // indirect
@@ -240,7 +230,7 @@ require (
240230
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734
241231
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c
242232
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
243-
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 // indirect
233+
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82
244234
golang.org/x/text v0.3.2
245235
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
246236
google.golang.org/grpc v1.20.1
@@ -277,3 +267,5 @@ replace github.com/docker/docker => github.com/docker/engine v0.0.0-201808160814
277267
replace github.com/ovh/cds/sdk/interpolate => ./sdk/interpolate
278268

279269
replace github.com/ovh/cds/sdk/izanami => ./sdk/izanami
270+
271+
go 1.13

‎sdk/signal_darwin.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package sdk
2+
3+
import "golang.org/x/sys/unix"
4+
5+
var SIGINFO = unix.SIGINFO

‎sdk/signal_linux.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package sdk
2+
3+
import "golang.org/x/sys/unix"
4+
5+
var SIGINFO = unix.SIGPWR

‎sdk/signal_windows.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package sdk
2+
3+
import "golang.org/x/sys/unix"
4+
5+
var SIGINFO = unix.SIGPWR

0 commit comments

Comments
 (0)
0