8000 feat(hatchery): remove stacktraces older than 15 minutes (#3177) · ovh/cds@8cdbd0a · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content
8000

Commit 8cdbd0a

Browse files
fsaminyesnault
fsamin
authored and
yesnault
committed
feat(hatchery): remove stacktraces older than 15 minutes (#3177)
1 parent 1e9bab1 commit 8cdbd0a

File tree

3 files changed

+37
-2
lines changed

3 files changed

+37
-2
lines changed

‎engine/hatchery/serve.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,41 @@ func (c *Common) servePanicDumpList() ([]string, error) {
4545
return res, nil
4646
}
4747

48+
func init() {
49+
// This go routine deletes panic dumps older than 15 minutes
50+
go func() {
51+
for {
52+
time.Sleep(1 * time.Minute)
53+
dir, err := os.Getwd()
54+
if err != nil {
55+
log.Warning("unable to get working directory: %v", err)
56+
continue
57+
}
58+
59+
path := filepath.Join(dir, panicDumpDir)
60+
files, err := ioutil.ReadDir(path)
61+
if err != nil {
62+
log.Warning("unable to list files in %s: %v", path, err)
63+
continue
64+
}
65+
66+
for _, f := range files {
67+
filename := filepath.Join(path, f.Name())
68+
file, err := os.Stat(filename)
69+
if err != nil {
70+
log.Warning("unable to get file %s info: %v", f.Name(), err)
71+
continue
72+
}
73+
if file.ModTime().Before(time.Now().Add(-15 * time.Minute)) {
74+
if err := os.Remove(filename); err != nil {
75+
log.Warning("unable to remove file %s: %v", filename, err)
76+
}
77+
}
78+
}
79+
}
80+
}()
81+
}
82+
4883
func (c *Common) servePanicDump(f string) (io.ReadCloser, error) {
4984
dir, _ := os.Getwd()
5085
path := filepath.Join(dir, panicDumpDir, f)

‎sdk/common.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,14 @@ func FileSHA512sum(filePath string) (string, error) {
108108

109109
// GoRoutine runs the function within a goroutine with a panic recovery
110110
func GoRoutine(name string, fn func(), writerFactories ...func(s string) (io.WriteCloser, error)) {
111+
hostname, _ := os.Hostname()
111112
go func() {
112113
defer func() {
113114
if r := recover(); r != nil {
114115
buf := make([]byte, 1<<16)
115116
runtime.Stack(buf, false)
116117
uuid := UUID()
117-
log.Error("[PANIC] %s Failed (%s)", name, uuid)
118+
log.Error("[PANIC][%s] %s Failed (%s)", hostname, name, uuid)
118119

119120
for _, f := range writerFactories {
120121
w, err := f(uuid)

‎sdk/hatchery/starter.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ func workerStarter(h Interface, jobs <-chan workerStarterRequest, results chan<-
7575
// Start a worker for a job
7676
if m := j.registerWorkerModel; m == nil {
7777
_, end := observability.Span(j.ctx, "hatchery.workerStarter")
78-
7978
//Try to start the worker
8079
isRun, err := spawnWorkerForJob(h, j)
8180
//Check the result

0 commit comments

Comments
 (0)
0