8000 fix(api): catch locked element and add retry in sdk if it's locked (c… · ovh/cds@4fe015e · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 4fe015e

Browse files
bnjjjbnjjj
authored
fix(api): catch locked element and add retry in sdk if it's locked (close #3581) (#3589)
Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com>
1 parent 4bf5d50 commit 4fe015e

File tree

5 files changed

+55
-9
lines changed

5 files changed

+55
-9
lines changed

‎engine/api/workflow/dao_node_run.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"time"
99

1010
"github.com/go-gorp/gorp"
11+
"github.com/lib/pq"
1112
"github.com/ovh/venom"
1213

1314
"github.com/ovh/cds/engine/api/cache"
@@ -164,6 +165,9 @@ func LoadAndLockNodeRunByID(ctx context.Context, db gorp.SqlExecutor, id int64,
164165
query += " nowait"
165166
}
166167
if err := db.SelectOne(&rr, query, id); err != nil {
168+
if errPG, ok := err.(*pq.Error); ok && errPG.Code == "55P03" {
169+
return nil, sdk.ErrWorkflowNodeRunLocked
170+
}
167171
return nil, sdk.WrapError(err, "Unable to load workflow_node_run node=%d", id)
168172
}
169173
return fromDBNodeRun(rr, LoadRunOptions{})
@@ -678,8 +682,8 @@ func PreviousNodeRun(db gorp.SqlExecutor, nr sdk.WorkflowNodeRun, n sdk.Workflow
678682
SELECT %s FROM workflow_node_run
679683
JOIN workflow_run ON workflow_run.id = workflow_node_run.workflow_run_id AND workflow_run.workflow_id = $1
680684
WHERE workflow_node_run.workflow_node_name = $2
681-
AND workflow_node_run.vcs_branch = $3 AND workflow_node_run.vcs_tag = $4
682-
AND workflow_node_run.num <= $5
685+
AND workflow_node_run.vcs_branch = $3 AND workflow_node_run.vcs_tag = $4
686+
AND workflow_node_run.num <= $5
683687
AND workflow_node_run.id != $6
684688
ORDER BY workflow_node_run.num DESC, workflow_node_run.sub_num DESC
685689
LIMIT 1

‎engine/api/workflow/dao_node_run_job.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"time"
1010

1111
"github.com/go-gorp/gorp"
12+
"github.com/lib/pq"
1213

1314
"github.com/ovh/cds/engine/api/cache"
1415
"github.com/ovh/cds/engine/api/group"
@@ -221,6 +222,9 @@ func LoadNodeJobRun(db gorp.SqlExecutor, store cache.Store, id int64) (*sdk.Work
221222
if err == sql.ErrNoRows {
222223
return nil, sdk.ErrWorkflowNodeRunJobNotFound
223224
}
225+
if errPG, ok := err.(*pq.Error); ok && errPG.Code == "55P03" {
226+
return nil, sdk.ErrJobLocked
227+
}
224228
return nil, err
225229
}
226230
if store != nil {
@@ -284,6 +288,9 @@ func LoadAndLockNodeJobRunNoWait(ctx context.Context, db gorp.SqlExecutor, store
284288
j := JobRun{}
285289
query := `select workflow_node_run_job.* from workflow_node_run_job where id = $1 for update nowait`
286290
if err := db.SelectOne(&j, query, id); err != nil {
291+
if errPG, ok := err.(*pq.Error); ok && errPG.Code == "55P03" {
292+
return nil, sdk.ErrJobLocked
293+
}
287294
return nil, err
288295
}
289296
getHatcheryInfo(store, &j)

‎engine/worker/builtin_junit.go

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ import (
66
"encoding/xml"
77
"fmt"
88
"io/ioutil"
9+
"net/http"
910
"net/url"
1011
"path/filepath"
1112
"strings"
13+
"time"
1214

1315
"github.com/ovh/cds/sdk"
1416
"github.com/ovh/venom"
@@ -93,13 +95,21 @@ func runParseJunitTestResultAction(w *currentWorker) BuiltInAction {
9395
uri = fmt.Sprintf("/project/%s/application/%s/pipeline/%s/build/%s/test?envName=%s", proj, app, pip, bnS, url.QueryEscape(envName))
9496
}
9597

96-
_, code, err := sdk.Request("POST", uri, []byte(dataS))
97-
if err == nil && code > 300 {
98-
err = fmt.Errorf("HTTP %d", code)
98+
var statusCode int
99+
var errPost error
100+
for retry := 0; retry < 10; retry++ {
101+
_, statusCode, errPost = sdk.Request("POST", uri, []byte(dataS))
102+
if statusCode != http.StatusConflict {
103+
break
104+
}
105+
time.Sleep(500 * time.Millisecond)
106+
}
107+
if errPost == nil && statusCode > 300 {
108+
errPost = fmt.Errorf("HTTP %d", statusCode)
99109
}
100110

101-
if err != nil {
102-
res.Reason = fmt.Sprintf("JUnit parse: failed to send tests details: %s", err)
111+
if errPost != nil {
112+
res.Reason = fmt.Sprintf("JUnit parse: failed to send tests details: %s", errPost)
103113
res.Status = sdk.StatusFail.String()
104114
sendLog(res.Reason)
105115
return res

‎sdk/cdsclient/client_queue.go

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,16 @@ func (c *client) QueueJobSendSpawnInfo(ctx context.Context, isWorkflowJob bool,
268268
path = fmt.Sprintf("/queue/%d/spawn/infos", id)
269269
}
270270

271-
_, err := c.PostJSON(ctx, path, &in, nil)
271+
var statusCode int
272+
var err error
273+
for retry := 0; retry < 10; retry++ {
274+
statusCode, err = c.PostJSON(ctx, path, &in, nil)
275+
if statusCode != http.StatusConflict {
276+
break
277+
}
278+
time.Sleep(500 * time.Millisecond)
279+
}
280+
272281
return err
273282
}
274283

@@ -304,8 +313,18 @@ func (c *client) QueueJobRelease(isWorkflowJob bool, id int64) error {
304313
}
305314

306315
func (c *client) QueueSendResult(ctx context.Context, id int64, res sdk.Result) error {
316+
var statusCode int
317+
var err error
318+
307319
path := fmt.Sprintf("/queue/workflows/%d/result", id)
308-
_, err := c.PostJSON(ctx, path, res, nil)
320+
for retry := 0; retry < 10; retry++ {
321+
statusCode, err = c.PostJSON(ctx, path, res, nil)
322+
if statusCode != http.StatusConflict {
323+
break
324+
}
325+
time.Sleep(500 * time.Millisecond)
326+
}
327+
309328
return err
310329
}
311330

‎sdk/error.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,8 @@ var (
161161
ErrColorBadFormat = Error{ID: 144, Status: http.StatusBadRequest}
162162
ErrInvalidHookConfiguration = Error{ID: 145, Status: http.StatusBadRequest}
163163
ErrWorkerModelDeploymentFailed = Error{ID: 146, Status: http.StatusBadRequest}
164+
ErrJobLocked = Error{ID: 147, Status: http.StatusConflict}
165+
ErrWorkflowNodeRunLocked = Error{ID: 148, Status: http.StatusConflict}
164166
)
165167

166168
var errorsAmericanEnglish = map[int]string{
@@ -308,6 +310,8 @@ var errorsAmericanEnglish = map[int]string{
308310
ErrColorBadFormat.ID: "The format of color isn't correct. You must use hexadecimal format (example: #FFFF)",
309311
ErrInvalidHookConfiguration.ID: "Invalid hook configuration",
310312
ErrWorkerModelDeploymentFailed.ID: "Worker deployment failed",
313+
ErrJobLocked.ID: "Job already locked",
314+
ErrWorkflowNodeRunLocked.ID: "Workflow node run already locked",
311315
}
312316

313317
var errorsFrench = map[int]string{
@@ -455,6 +459,8 @@ var errorsFrench = map[int]string{
455459
ErrColorBadFormat.ID: "Format de la couleur incorrect. Vous devez utiliser le format hexadécimal (exemple: #FFFF)",
456460
ErrInvalidHookConfiguration.ID: "Configuration de hook invalide",
457461
ErrWorkerModelDeploymentFailed.ID: "Échec de déploiement du modèle de worker",
462+
ErrJobLocked.ID: "Job déjà verrouillé",
463+
ErrWorkflowNodeRunLocked.ID: "Noeud de workflow run déjà verrouillé",
458464
}
459465

460466
var errorsLanguages = []map[int]string{

0 commit comments

Comments
 (0)
0