8000 feat(api): delete workflow run on branch delete (#4177) · ovh/cds@a1e8b44 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit a1e8b44

Browse files
bnjjjbnjjj
authored
feat(api): delete workflow run on branch delete (#4177)
Signed-off-by: Benjamin Coenen <benjamin.coenen@corp.ovh.com>
1 parent 0c93bdb commit a1e8b44

File tree

15 files changed

+325
-8
lines changed

15 files changed

+325
-8
lines changed

‎docs/content/docs/concepts/workflow/hooks/git-repo-webhook.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@ You have to:
1212
* add a Repository Webhook on the root pipeline, this pipeline have the application linked in the [context]({{< relref "/docs/concepts/workflow/pipeline-context.md" >}})
1313

1414
GitHub / Bitbucket / GitLab are supported by CDS.
15+
16+
> When you add a repository webhook, it will also automatically delete your runs which are linked to a deleted branch (24h after branch deletion).

‎docs/content/docs/integrations/bitbucket.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,3 +181,6 @@ $ engine start vcs
181181
$ engine start api vcs
182182
```
183183

184+
## Vcs events
185+
186+
For now, CDS supports push events. CDS uses this push event to remove existing runs for deleted branches (24h after branch deletion).

‎docs/content/docs/integrations/github.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,4 +266,8 @@ If you hesitate between the two: the `RepositoryWebhook` is more *reactive* than
266266
Before adding a hook on your Workflow, you have to add the application in the Pipeline Context.
267267
Select the first pipeline, then click on **Edit the pipeline context** from the [sidebar]({{<relref "/docs/concepts/workflow/sidebar.md">}}).
268268

269-
[Pipeline Context Documentation]({{<relref "/docs/concepts/workflow/pipeline-context.md">}})
269+
[Pipeline Context Documentation]({{<relref "/docs/concepts/workflow/pipeline-context.md">}})
270+
271+
## Vcs events
272+
273+
For now, CDS supports push events. CDS uses this push event to remove existing runs for deleted branches (24h after branch deletion).

‎docs/content/docs/integrations/gitlab.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,6 @@ $ engine start vcs
8383
$ engine start api vcs
8484
```
8585

86+
## Vcs events
87+
88+
For now, CDS supports push events. CDS uses this push event to remove existing runs for deleted branches (24h after branch deletion).

‎engine/api/api_routes.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ func (api *API) InitRouter() {
230230
r.Handle("/project/{permProjectKey}/runs", r.GET(api.getWorkflowAllRunsHandler, EnableTracing()))
231231
r.Handle("/project/{key}/workflows/{permWorkflowName}/artifact/{artifactId}", r.GET(api.getDownloadArtifactHandler))
232232
r.Handle("/project/{key}/workflows/{permWorkflowName}/runs", r.GET(api.getWorkflowRunsHandler, EnableTracing()), r.POSTEXECUTE(api.postWorkflowRunHandler, AllowServices(true), EnableTracing()))
233+
r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/branch/{branch}", r.DELETE(api.deleteWorkflowRunsBranchHandler, NeedService()))
233234
r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/latest", r.GET(api.getLatestWorkflowRunHandler))
234235
r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/tags", r.GET(api.getWorkflowRunTagsHandler))
235236
r.Handle("/project/{key}/workflows/{permWorkflowName}/runs/num", r.GET(api.getWorkflowRunNumHandler), r.POST(api.postWorkflowRunNumHandler))

‎engine/api/router_auth.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ func (api *API) authDeprecatedMiddleware(ctx context.Context, w http.ResponseWri
165165
if getService(ctx) != nil {
166166
return ctx, false, nil
167167
}
168-
return ctx, false, sdk.WrapError(sdk.ErrForbidden, "Router> Need worker")
168+
return ctx, false, sdk.WrapError(sdk.ErrForbidden, "Router> Need service")
169169
}
170170

171171
if rc.Options["needWorker"] == "true" {

‎engine/api/workflow/dao_run.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,44 @@ func LoadRuns(db gorp.SqlExecutor, projectkey, workflowname string, offset, limi
406406
return wruns, offset, limit, int(count), nil
407407
}
408408

409+
// LoadRunsIDByTag load workflow run ids for given tag and his value
410+
func LoadRunsIDByTag(db gorp.SqlExecutor, projectKey, workflowName, tag, tagValue string) ([]int64, error) {
411+
query := `SELECT workflow_run.id
412+
FROM workflow_run
413+
JOIN project on workflow_run.project_id = project.id
414+
JOIN workflow on workflow_run.workflow_id = workflow.id
415+
JOIN (
416+
SELECT workflow_run_id, string_agg(all_tags, ',') AS tags
417+
FROM (
418+
SELECT workflow_run_id, tag || '=' || value "all_tags"
419+
FROM workflow_run_tag
420+
WHERE workflow_run_tag.tag = $3 AND workflow_run_tag.value = $4
421+
ORDER BY tag
422+
) AS all_wr_tags
423+
GROUP BY workflow_run_id
424+
) AS tags on workflow_run.id = tags.workflow_run_id
425+
WHERE project.projectkey = $1
426+
AND workflow.name = $2
427+
ORDER BY workflow_run.start DESC`
428+
429+
idsDB := []struct {
430+
ID int64 `db:"id"`
431+
}{}
432+
if _, err := db.Select(&idsDB, query, projectKey, workflowName, tag, tagValue); err != nil {
433+
if err == sql.ErrNoRows {
434+
return nil, nil
435+
}
436+
return nil, sdk.WrapError(err, "Cannot load runs id by tag")
437+
}
438+
439+
ids := make([]int64, len(idsDB))
440+
for i := range idsDB {
441+
ids[i] = idsDB[i].ID
442+
}
443+
444+
return ids, nil
445+
}
446+
409447
func loadRunTags(db gorp.SqlExecutor, run *sdk.WorkflowRun) error {
410448
dbRunTags := []RunTag{}
411449
if _, err := db.Select(&dbRunTags, "SELECT * from workflow_run_tag WHERE workflow_run_id=$1", run.ID); err != nil {
@@ -692,6 +730,15 @@ func PurgeAllWorkflowRunsByWorkflowID(db gorp.SqlExecutor, id int64) (int, error
692730
return int(n), nil
693731
}
694732

733+
// MarkWorkflowRunsAsDelete marks workflow runs to be deleted
734+
func MarkWorkflowRunsAsDelete(db gorp.SqlExecutor, ids []int64) error {
735+
idsStr := gorpmapping.IDsToQueryString(ids)
736+
if _, err := db.Exec("update workflow_run set to_delete = true where id = ANY(string_to_array($1, ',')::int[])", idsStr); err != nil {
737+
return sdk.WrapError(err, "Unable to mark as delete workflow id %s", idsStr)
738+
}
739+
return nil
740+
}
741+
695742
type byInt64Desc []int64
696743

697744
func (a byInt64Desc) Len() int { return len(a) }

‎engine/api/workflow_run.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,26 @@ func (api *API) getWorkflowRunsHandler() service.Handler {
161161
}
162162
}
163163

164+
func (api *API) deleteWorkflowRunsBranchHandler() service.Handler {
165+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
166+
vars := mux.Vars(r)
167+
key := vars["key"]
168+
name := vars["permWorkflowName"]
169+
branch := vars["branch"]
170+
171+
wfIDs, err := workflow.LoadRunsIDByTag(api.mustDB(), key, name, "git.branch", branch)
172+
if err != nil {
173+
return err
174+
}
175+
176+
if err := workflow.MarkWorkflowRunsAsDelete(api.mustDB(), wfIDs); err != nil {
177+
return err
178+
}
179+
180+
return service.WriteJSON(w, nil, http.StatusOK)
181+
}
182+
}
183+
164184
// getWorkflowRunNumHandler returns the last run number for the given workflow
165185
func (api *API) getWorkflowRunNumHandler() service.Handler {
166186
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {

‎engine/api/workflow_run_test.go

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ import (
2121
"github.com/ovh/cds/engine/api/project"
2222
"github.com/ovh/cds/engine/api/repositoriesmanager"
2323
"github.com/ovh/cds/engine/api/services"
24+
"github.com/ovh/cds/engine/api/sessionstore"
2425
"github.com/ovh/cds/engine/api/test"
2526
"github.com/ovh/cds/engine/api/test/assets"
27+
"github.com/ovh/cds/engine/api/token"
2628
"github.com/ovh/cds/engine/api/user"
2729
"github.com/ovh/cds/engine/api/workflow"
2830
"github.com/ovh/cds/sdk"
@@ -1632,3 +1634,161 @@ func Test_getWorkflowNodeRunJobServiceLogsHandler(t *testing.T) {
16321634
assert.Equal(t, 200, rec.Code)
16331635
assert.Equal(t, "098765432109876... truncated\n", logs[0].Val)
16341636
}
1637+
1638+
func Test_deleteWorkflowRunsBranchHandler(t *testing.T) {
1639+
api, db, router, end := newTestAPI(t, bootstrap.InitiliazeDB)
1640+
defer end()
1641+
u, pass := assets.InsertAdminUser(api.mustDB())
1642+
key := sdk.RandomString(10)
1643+
proj := assets.InsertTestProject(t, db, api.Cache, key, key, u)
1644+
1645+
//First pipeline
1646+
pip := sdk.Pipeline{
1647+
ProjectID: proj.ID,
1648+
ProjectKey: proj.Key,
1649+
Name: "pip1",
1650+
}
1651+
test.NoError(t, pipeline.InsertPipeline(api.mustDB(), api.Cache, proj, &pip, u))
1652+
1653+
s := sdk.NewStage("stage 1")
1654+
s.Enabled = true
1655+
s.PipelineID = pip.ID
1656+
test.NoError(t, pipeline.InsertStage(api.mustDB(), s))
1657+
j := &sdk.Job{
1658+
Enabled: true,
1659+
Action: sdk.Action{
1660+
Enabled: true,
1661+
},
1662+
}
1663+
test.NoError(t, pipeline.InsertJob(api.mustDB(), j, s.ID, &pip))
1664+
s.Jobs = append(s.Jobs, *j)
1665+
1666+
pip.Stages = append(pip.Stages, *s)
1667+
1668+
//Second pipeline
1669+
pip2 := sdk.Pipeline{
1670+
ProjectID: proj.ID,
1671+
ProjectKey: proj.Key,
1672+
Name: "pip2",
1673+
}
1674+
test.NoError(t, pipeline.InsertPipeline(api.mustDB(), api.Cache, proj, &pip2, u))
1675+
s = sdk.NewStage("stage 1")
1676+
s.Enabled = true
1677+
s.PipelineID = pip2.ID
1678+
test.NoError(t, pipeline.InsertStage(api.mustDB(), s))
1679+
j = &sdk.Job{
1680+
Enabled: true,
1681+
Action: sdk.Action{
1682+
Enabled: true,
1683+
},
1684+
}
1685+
test.NoError(t, pipeline.InsertJob(api.mustDB(), j, s.ID, &pip2))
1686+
s.Jobs = append(s.Jobs, *j)
1687+
1688+
w := sdk.Workflow{
1689+
Name: "test_1",
1690+
ProjectID: proj.ID,
1691+
ProjectKey: proj.Key,
1692+
WorkflowData: &sdk.WorkflowData{
1693+
Node: sdk.Node{
1694+
Name: "root",
1695+
Type: sdk.NodeTypePipeline,
1696+
Context: &sdk.NodeContext{
1697+
PipelineID: pip.ID,
1698+
},
1699+
Triggers: []sdk.NodeTrigger{
1700+
{
1701+
ChildNode: sdk.Node{
1702+
Name: "child",
1703+
Type: sdk.NodeTypePipeline,
1704+
Context: &sdk.NodeContext{
1705+
PipelineID: pip.ID,
1706+
},
1707+
},
1708+
},
1709+
},
1710+
},
1711+
},
1712+
}
1713+
1714+
(&w).RetroMigrate()
1715+
1716+
proj2, errP := project.Load(api.mustDB(), api.Cache, proj.Key, u, project.LoadOptions.WithPipelines, project.LoadOptions.WithGroups, project.LoadOptions.WithIntegrations)
1717+
test.NoError(t, errP)
1718+
1719+
test.NoError(t, workflow.Insert(api.mustDB(), api.Cache, &w, proj2, u))
1720+
w1, err := workflow.Load(context.TODO(), api.mustDB(), api.Cache, proj, "test_1", u, workflow.LoadOptions{})
1721+
test.NoError(t, err)
1722+
1723+
wr, err := workflow.CreateRun(db, w1, nil, u)
1724+
assert.NoError(t, err)
1725+
wr.Workflow = *w1
1726+
wr.Tag("git.branch", "master")
1727+
assert.NoError(t, workflow.UpdateWorkflowRun(context.TODO(), api.mustDB(), wr))
1728+
_, err = workflow.StartWorkflowRun(context.TODO(), db, api.Cache, proj, wr, &sdk.WorkflowRunPostHandlerOption{
1729+
Manual: &sdk.WorkflowNodeRunManual{
1730+
User: *u,
1731+
Payload: `{"git.branch": "master"}`,
1732+
},
1733+
}, u, nil)
1734+
test.NoError(t, err)
1735+
1736+
// Generate a fake service
1737+
gr := assets.InsertTestGroup(t, api.mustDB(), sdk.RandomString(10))
1738+
test.NotNil(t, gr)
1739+
//Generate token
1740+
tk, err := token.GenerateToken()
1741+
test.NoError(t, err)
1742+
//Insert token
1743+
test.NoError(t, token.InsertToken(api.mustDB(), gr.ID, tk, sdk.Persistent, "", ""))
1744+
1745+
//Generate a hash
1746+
hash, errsession := sessionstore.NewSessionKey()
1747+
if errsession != nil {
1748+
t.Fatal(errsession)
1749+
}
1750+
1751+
service := &sdk.Service{
1752+
Name: sdk.RandomString(10),
1753+
GroupID: &gr.ID,
1754+
Type: services.TypeVCS,
1755+
Token: tk,
1756+
Hash: string(hash),
1757+
}
1758+
1759+
err = services.Insert(api.mustDB(), service)
1760+
test.NoError(t, err)
1761+
//Prepare request
1762+
vars := map[string]string{
1763+
"key": proj.Key,
1764+
"permWorkflowName": w1.Name,
1765+
"branch": "master",
1766+
}
1767+
uri := router.GetRoute("DELETE", api.deleteWorkflowRunsBranchHandler, vars)
1768+
test.NotEmpty(t, uri)
1769+
req := assets.NewAuthentifiedRequestFromHatchery(t, service, "DELETE", uri, vars)
1770+
1771+
//Do the request
1772+
rec := httptest.NewRecorder()
1773+
router.Mux.ServeHTTP(rec, req)
1774+
assert.Equal(t, 200, rec.Code)
1775+
1776+
//Prepare request
1777+
vars = map[string]string{
1778+
"key": proj.Key,
1779+
"permWorkflowName": w1.Name,
1780+
}
1781+
uri = router.GetRoute("GET", api.getWorkflowRunsHandler, vars)
1782+
test.NotEmpty(t, uri)
1783+
req = assets.NewAuthentifiedRequest(t, u, pass, "GET", uri, vars)
1784+
1785+
//Do the request
1786+
rec = httptest.NewRecorder()
1787+
router.Mux.ServeHTTP(rec, req)
1788+
assert.Equal(t, 200, rec.Code)
1789+
1790+
var wfRuns []sdk.WorkflowRun
1791+
test.NoError(t, json.Unmarshal(rec.Body.Bytes(), &wfRuns))
1792+
assert.Equal(t, 1, len(wfRuns))
1793+
assert.True(t, wfRuns[0].ToDelete)
1794+
}

‎engine/hooks/branch_deletion.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package hooks
2+
3+
import (
4+
"github.com/ovh/cds/sdk"
5+
"github.com/ovh/cds/sdk/log"
6+
)
7+
8+
func (s *Service) doBranchDeletionTaskExecution(t *sdk.TaskExecution) (*sdk.WorkflowNodeRunHookEvent, error) {
9+
log.Debug("Hooks> Processing branch deletion task %s", t.UUID)
10+
11+
projectKey := t.Config["project"].Value
12+
workflowName := t.Config["workflow"].Value
13+
branch := t.Config["branch"].Value
14+
err := s.Client.WorkflowRunsDeleteByBranch(projectKey, workflowName, branch)
15+
16+
return nil, sdk.WrapError(err, "cannot mark to delete workflow runs")
17+
}

0 commit comments

Comments
 (0)
0