8000 feat(ui, api): Improve hooks infos (#3314) · ovh/cds@2583088 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 2583088

Browse files
richardltbnjjj
richardlt
authored and
bnjjj
committed
feat(ui, api): Improve hooks infos (#3314)
1 parent 66a88c1 commit 2583088

28 files changed

+742
-48
lines changed

‎cli/cdsctl/admin_hooks.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package main
33
import (
44
"encoding/json"
55
"fmt"
6+
"net/url"
7+
"reflect"
68
"time"
79

810
"github.com/spf13/cobra"
@@ -32,10 +34,25 @@ var (
3234
var adminHooksTaskListCmd = cli.Command{
3335
Name: "list",
3436
Short: "List CDS Hooks Tasks",
37+
Flags: []cli.Flag{
38+
{
39+
Kind: reflect.String,
40+
Name: "sort",
41+
Usage: "Sort task by nb_executions_total,nb_executions_todo",
42+
Default: "",
43+
},
44+
},
3545
}
3646

3747
func adminHooksTaskListRun(v cli.Values) (cli.ListResult, error) {
38-
btes, err := client.ServiceCallGET("hooks", "/task")
48+
url, _ := url.Parse("/task")
49+
if s := v.GetString("sort"); s != "" {
50+
q := url.Query()
51+
q.Add("sort", s)
52+
url.RawQuery = q.Encode()
53+
}
54+
55+
btes, err := client.ServiceCallGET("hooks", url.String())
3956
if err != nil {
4057
return nil, err
4158
}

‎engine/api/cache/redis.go

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -361,22 +361,38 @@ func (s *RedisStore) SetScan(key string, members ...interface{}) error {
361361
return sdk.WrapError(err, "redis zrange error")
362362
}
363363

364-
for i := range members {
365-
if i >= len(values) {
366-
break
364+
keys := make([]string, len(values))
365+
for i, v := range values {
366+
keys[i] = Key(key, v)
367+
}
368+
369+
if len(keys) > 0 {
370+
res, err := s.Client.MGet(keys...).Result()
371+
if err != nil {
372+
return sdk.WrapError(err, "redis mget error")
367373
}
368-
val := values[i]
369-
memKey := Key(key, val)
370-
if !s.Get(memKey, members[i]) {
371-
//If the member is not found, return an error because the members are inconsistents
372-
// but try to delete the member from the Redis ZSET
373-
log.Error("redis>SetScan member %s not found", memKey)
374-
if err := s.Client.ZRem(key, val).Err(); err != nil {
375-
log.Error("redis>SetScan unable to delete member %s", memKey)
374+
375+
for i := range members {
376+
if i >= len(values) {
377+
break
378+
}
379+
380+
if res[i] == nil {
381+
//If the member is not found, return an error because the members are inconsistents
382+
// but try to delete the member from the Redis ZSET
383+
log.Error("redis>SetScan member %s not found", keys[i])
384+
if err := s.Client.ZRem(key, values[i]).Err(); err != nil {
385+
log.Error("redis>SetScan unable to delete member %s", keys[i])
386+
return err
387+
}
388+
log.Info("redis> member %s deleted", keys[i])
389+
return fmt.Errorf("SetScan member %s not found", keys[i])
390+
}
391+
392+
if err := json.Unmarshal([]byte(res[i].(string)), members[i]); err != nil {
393+
log.Warning("redis> cannot unmarshal %s :%s", keys[i], err)
376394
return err
377395
}
378-
log.Info("redis> member %s deleted", memKey)
379-
return fmt.Errorf("SetScan member %s not found", memKey)
380396
}
381397
}
382398
return nil

‎engine/api/router_util.go

Lines changed: 55 additions & 0 deletions
E377
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package api
33
import (
44
"context"
55
"encoding/json"
6+
"fmt"
67
"io/ioutil"
78
"net/http"
89
"reflect"
@@ -117,6 +118,60 @@ func QueryStrings(r *http.Request, key string) ([]string, error) {
117118
return nil, nil
118119
}
119120

121+
// SortOrder constant.
122+
type SortOrder string
123+
124+
// SortOrders.
125+
const (
126+
ASC SortOrder = "asc"
127+
DESC SortOrder = "desc"
128+
)
129+
130+
func validateSortOrder(s string) bool {
131+
switch SortOrder(s) {
132+
case ASC, DESC:
133+
return true
134+
}
135+
return false
136+
}
137+
138+
// SortCompareInt returns the result of the right compare equation depending of given sort order.
139+
func SortCompareInt(i, j int, o SortOrder) bool {
140+
if o == ASC {
141+
return i < j
142+
}
143+
return i > j
144+
}
145+
146+
// QuerySort returns the a of key found in sort query param or nil if sort param not found.
147+
func QuerySort(r *http.Request) (map[string]SortOrder, error) {
148+
if err := r.ParseForm(); err != nil {
149+
return nil, err
150+
}
151+
v, ok := r.Form["sort"]
152+
if !ok {
153+
return nil, nil
154+
}
155+
156+
res := map[string]SortOrder{}
157+
for _, item := range strings.Split(v[0], ",") {
158+
if item == "" {
159+
return nil, sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("invalid given sort key"))
160+
}
161+
s := strings.Split(item, ":")
162+
if len(s) > 1 {
163+
if !validateSortOrder(s[1]) {
164+
return nil, sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("invalid given sort param"))
165+
}
166+
res[s[0]] = SortOrder(s[1])
167+
} else {
168+
res[s[0]] = ASC
169+
}
170+
}
171+
172+
return res, nil
173+
}
174+
120175
// FormInt return a int from query params
121176
func FormInt(r *http.Request, s string) (int, error) {
122177
stringValue := FormString(r, s)

‎engine/api/router_util_test.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package api_test
2+
3+
import (
4+
"net/http"
5+
"net/url"
6+
"testing"
7+
8+
"github.com/ovh/cds/engine/api"
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
func TestQuerySort(t *testing.T) {
13+
url, _ := url.Parse("http://localhost?sort=column1,column2:asc,column3:desc")
14+
m, _ := api.QuerySort(&http.Request{URL: url})
15+
assert.Len(t, m, 3)
16+
assert.Equal(t, m, map[string]api.SortOrder{
17+
"column1": api.ASC,
18+
"column2": api.ASC,
19+
"column3": api.DESC,
20+
})
21+
}
22+
func TestQuerySortError(t *testing.T) {
23+
url, _ := url.Parse("http://localhost?sort=column1,,column3:desc")
24+
_, err := api.QuerySort(&http.Request{URL: url})
25+
assert.Error(t, err)
26+
27+
url, _ = url.Parse("http://localhost?sort=column1,column3:unknown")
28+
_, err = api.QuerySort(&http.Request{URL: url})
29+
assert.Error(t, err)
30+
}

‎engine/hooks/dao.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,16 @@ func (d *dao) FindAllTaskExecutions(t *sdk.Task) ([]sdk.TaskExecution, error) {
9090

9191
return allexecs, nil
9292
}
93+
94+
func (d *dao) FindAllTaskExecutionsForTasks(ts ...sdk.Task) ([]sdk.TaskExecution, error) {
95+
var tes []sdk.TaskExecution
96+
for _, t := range ts {
97+
res, err := d.FindAllTaskExecutions(&t)
98+
if err != nil {
99+
return nil, err
100+
}
101+
tes = append(tes, res...)
102+
}
103+
104+
return tes, nil
105+
}

‎engine/hooks/hooks_handlers.go

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -146,28 +146,62 @@ func (s *Service) postTaskHandler() service.Handler {
146146
}
147147
}
148148

149+
const (
150+
sortKeyNbExecutionsTotal = "nb_executions_total"
151+
sortKeyNbExecutionsTodo = "nb_executions_todo"
152+
)
153+
149154
func (s *Service) getTasksHandler() service.Handler {
150155
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
156+
sortParams, err := api.QuerySort(r)
157+
if err != nil {
158+
return sdk.NewError(sdk.ErrWrongRequest, err)
159+
}
160+
for k := range sortParams {
161+
if k != sortKeyNbExecutionsTotal && k != sortKeyNbExecutionsTodo {
162+
return sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("invalid given sort key"))
163+
}
164+
}
165+
151166
tasks, err := s.Dao.FindAllTasks()
152167
if err != nil {
153168
return sdk.WrapError(err, "Hooks> getTasksHandler")
154169
}
155-
for i := range tasks {
156-
execs, err := s.Dao.FindAllTaskExecutions(&tasks[i])
157-
if err != nil {
158-
log.Error("getTasksHandler> Unable to find all task executions (%s): %v", tasks[i].UUID, err)
159-
continue
160-
}
161170

171+
execs, err := s.Dao.FindAllTaskExecutionsForTasks(tasks...)
172+
if err != nil {
173+
return sdk.WrapError(err, "Hooks> getTasksHandler")
174+
}
175+
176+
m := make(map[string][]sdk.TaskExecution, len(tasks))
177+
for _, e := range execs {
178+
m[e.UUID] = append(m[e.UUID], e)
179+
}
180+
181+
for i, t := range tasks {
162182
var nbTodo int
163-
for _, e := range execs {
183+
for _, e := range m[t.UUID] {
164184
if e.ProcessingTimestamp == 0 {
165185
nbTodo++
166186
}
167187
}
168-
tasks[i].NbExecutionsTotal = len(execs)
188+
tasks[i].NbExecutionsTotal = len(m[t.UUID])
169189
tasks[i].NbExecutionsTodo = nbTodo
170190
}
191+
192+
for k, p := range sortParams {
193+
switch k {
194+
case sortKeyNbExecutionsTotal:
195+
sort.Slice(tasks, func(i, j int) bool {
196+
return api.SortCompareInt(tasks[i].NbExecutionsTotal, tasks[j].NbExecutionsTotal, p)
197+
})
198+
case sortKeyNbExecutionsTodo:
199+
sort.Slice(tasks, func(i, j int) bool {
200+
return api.SortCompareInt(tasks[i].NbExecutionsTodo, tasks[j].NbExecutionsTodo, p)
201+
})
202+
}
203+
}
204+
171205
return service.WriteJSON(w, tasks, http.StatusOK)
172206
}
173207
}

‎engine/vcs/gitlab/client_repos.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@ func (c *gitlabClient) Repos(ctx context.Context) ([]sdk.VCSRepo, error) {
1414
var repos []sdk.VCSRepo
1515

1616
pp := 1000
17-
opts := &gitlab.ListProjectsOptions{}
17+
opts := &gitlab.ListProjectsOptions{
18+
Membership: gitlab.Bool(true),
19+
}
1820
opts.PerPage = pp
1921

2022
projects, resp, err := c.client.Projects.ListProjects(opts)

‎sdk/hooks.go

Lines changed: 2 additions & 2 deletions
< 57AE td data-grid-cell-id="diff-ffda731c8fd448e3f5a429ce2d1151aff46b8ff81a1cea1e05e755cf630eca01-40-40-0" data-selected="false" role="gridcell" style="background-color:var(--diffBlob-additionNum-bgColor, var(--diffBlob-addition-bgColor-num));text-align:center" tabindex="-1" valign="top" class="focusable-grid-cell diff-line-number position-relative left-side">
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@ type WebHookExecution struct {
3737

3838
// KafkaTaskExecution contains specific data for a kafka hook
3939
type KafkaTaskExecution struct {
40-
Message []byte
40+
Message []byte `json:"message"`
4141
}
4242

4343
// RabbitMQTaskExecution contains specific data for a kafka hook
4444
type RabbitMQTaskExecution struct {
45-
Message []byte
45+
Message []byte `json:"message"`
4646
}
4747

4848
// ScheduledTaskExecution contains specific data for a scheduled task execution

‎ui/src/app/model/workflow.hook.model.ts

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import {WorkflowNodeHookConfigValue} from './workflow.model';
1+
import { WorkflowNodeHookConfigValue } from './workflow.model';
22

33
export class WorkflowHookModel {
44
id: number;
@@ -15,9 +15,10 @@ export class WorkflowHookModel {
1515
}
1616

1717
export enum HookStatus {
18-
DONE = 'DONE',
19-
DOING = 'DOING',
20-
FAIL = 'FAIL'
18+
DONE = 'DONE',
19+
DOING = 'DOING',
20+
FAIL = 'FAIL',
21+
SCHEDULED = 'SCHEDULED'
2122
}
2223

2324
export class WorkflowHookTask {
@@ -26,6 +27,8 @@ export class WorkflowHookTask {
2627
config: Map<string, WorkflowNodeHookConfigValue>;
2728
type: string;
2829
executions: TaskExecution[];
30+
nb_executions_total: number;
31+
nb_executions_todo: number;
2932
}
3033

3134
export class TaskExecution {
@@ -38,6 +41,8 @@ export class TaskExecution {
3841
workflow_run: number;
3942
config: Map<string, WorkflowNodeHookConfigValue>;
4043
webhook: Webhook;
44+
rabbitmq: RabbitMQ;
45+
kafka: Kafka;
4146
scheduled_task?: any;
4247
status: HookStatus;
4348
}
@@ -47,3 +52,12 @@ export class Webhook {
4752
request_body: string;
4853
request_header: Map<string, string[]>;
4954
}
55+
56+
export class RabbitMQ {
57+
message: string;
58+
}
59+
60+
export class Kafka {
61+
message: string;
62+
}
63+

‎ui/src/app/service/hook/hook.service.ts

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
1-
2-
import {HttpClient} from '@angular/common/http';
3-
import {Injectable} from '@angular/core';
4-
import {Observable} from 'rxjs';
5-
import {map} from 'rxjs/operators';
6-
import {Project} from '../../model/project.model';
7-
import {WorkflowHookModel, WorkflowHookTask} from '../../model/workflow.hook.model';
8-
import {Workflow, WorkflowNode} from '../../model/workflow.model';
9-
1+
import { HttpClient } from '@angular/common/http';
2+
import { Injectable } from '@angular/core';
3+
import { Observable } from 'rxjs';
4+
import { map } from 'rxjs/operators';
5+
import { Project } from '../../model/project.model';
6+
import { WorkflowHookModel, WorkflowHookTask } from '../../model/workflow.hook.model';
7+
import { Workflow, WorkflowNode } from '../../model/workflow.model';
108

119
@Injectable()
1210
export class HookService {
@@ -17,11 +15,23 @@ export class HookService {
1715
getHookModel(p: Project, w: Workflow, n: WorkflowNode): Observable<Array<WorkflowHookModel>> {
1816
return this._http.get<Array<WorkflowHookModel>>('/project/' + p.key + '/workflow/' + w.name +
1917
'/node/' + n.id + '/hook/model').pipe(map(ms => {
20-
return ms;
21-
}));
18+
return ms;
19+
}));
2220
}
2321

2422
getHookLogs(projectKey: string, workflowName: string, uuid: string): Observable<WorkflowHookTask> {
25-
return this._http.get<WorkflowHookTask>(`/project/${projectKey}/workflows/${workflowName}/hooks/${uuid}`);
23+
return this._http.get<WorkflowHookTask>(`/project/${projectKey}/workflows/${workflowName}/hooks/${uuid}`);
24+
}
25+
26+
getAdminTasks(sort: string): Observable<Array<WorkflowHookTask>> {
27+
return this.callServiceHooks<Array<WorkflowHookTask>>('/task' + (sort ? '?sort=' + sort : ''));
28+
}
29+
30+
getAdminTaskExecution(uuid: string): Observable<WorkflowHookTask> {
31+
return this.callServiceHooks<WorkflowHookTask>('/task/' + uuid + '/execution');
32+
}
33+
34+
callServiceHooks<T>(query: string): Observable<T> {
35+
return this._http.get<T>('/admin/services/call?type=hooks&query=' + encodeURIComponent(query));
2636
}
2737
}

0 commit comments

Comments
 (0)
0