8000 feat(ui): add overview page + push metrics (#3292) · ovh/cds@cdf70e0 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit cdf70e0

Browse files
sguiheuxbnjjj
sguiheux
authored and
bnjjj
committed
feat(ui): add overview page + push metrics (#3292)
1 parent c860687 commit cdf70e0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+948
-74
lines changed

‎engine/api/api.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -701,6 +701,7 @@ func (a *API) Serve(ctx context.Context) error {
701701
workflow.Initialize(ctx, a.DBConnectionFactory.GetDBMap, a.Config.URL.UI, a.Config.DefaultOS, a.Config.DefaultArch)
702702
})
703703
sdk.GoRoutine("PushInElasticSearch", func() { event.PushInElasticSearch(ctx, a.mustDB(), a.Cache) })
704+
metrics.Init(ctx, a.DBConnectionFactory.GetDBMap)
704705
sdk.GoRoutine("Purge", func() { purge.Initialize(ctx, a.Cache, a.DBConnectionFactory.GetDBMap) })
705706

706707
s := &http.Server{

‎engine/api/api_routes.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,8 @@ func (api *API) InitRouter() {
105105
r.Handle("/mon/metrics", r.GET(api.getMetricsHandler, Auth(false)))
106106
r.Handle("/mon/stats", r.GET(observability.StatsHandler, Auth(false)))
107107

108-
r.Handle("/navbar", r.GET(api.getNavbarHandler))
108+
r.Handle("/ui/navbar", r.GET(api.getNavbarHandler))
109+
r.Handle("/ui/project/{key}/application/{permApplicationName}/overview", r.GET(api.getApplicationOverviewHandler))
109110

110111
// Import As Code
111112
r.Handle("/import/{permProjectKey}", r.POST(api.postImportAsCodeHandler))
@@ -141,6 +142,7 @@ func (api *API) InitRouter() {
141142

142143
// Application
143144
r.Handle("/project/{key}/application/{permApplicationName}", r.GET(api.getApplicationHandler), r.PUT(api.updateApplicationHandler), r.DELETE(api.deleteApplicationHandler))
145+
r.Handle("/project/{key}/application/{permApplicationName}/metrics/{metricName}", r.GET(api.getApplicationMetricHandler))
144146
r.Handle("/project/{key}/application/{permApplicationName}/keys", r.GET(api.getKeysInApplicationHandler), r.POST(api.addKeyInApplicationHandler))
145147
r.Handle("/project/{key}/application/{permApplicationName}/keys/{name}", r.DELETE(api.deleteKeyInApplicationHandler))
146148
r.Handle("/project/{key}/application/{permApplicationName}/branches", r.GET(api.getApplicationBranchHandler))

‎engine/api/application/application_vunerability.go

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,36 @@ package application
22

33
import (
44
"database/sql"
5-
65
"github.com/go-gorp/gorp"
76

7+
"github.com/ovh/cds/engine/api/database/gorpmapping"
88
"github.com/ovh/cds/sdk"
99
)
1010

11+
// LoadVulnerabilitiesSummary compute vulnerabilities summary
12+
func LoadVulnerabilitiesSummary(db gorp.SqlExecutor, appID int64) (map[string]int64, error) {
13+
query := `
14+
SELECT json_object_agg(severity, nb)::TEXT
15+
FROM (
16+
SELECT count(id) AS nb, severity
17+
FROM application_vulnerability
18+
WHERE application_id = $1
19+
GROUP BY severity
20+
) tmp;
21+
`
22+
23+
var summary map[string]int64
24+
var result sql.NullString
25+
if err := db.QueryRow(query, appID).Scan(&result); err != nil {
26+
return nil, sdk.WrapError(err, "LoadVulnerabilitiesSummary")
27+
}
28+
29+
if err := gorpmapping.JSONNullString(result, &summary); err != nil {
30+
return nil, sdk.WrapError(err, "LoadVulnerabilitiesSummary> Unable to unmarshal summary")
31+
}
32+
return summary, nil
33+
}
34+
1135
// InsertVulnerabilities Insert vulnerabilities
1236
func InsertVulnerabilities(db gorp.SqlExecutor, vs []sdk.Vulnerability, appID int64, t string) error {
1337
if _, err := db.Exec("DELETE FROM application_vulnerability WHERE application_id = $1 AND type = $2", appID, t); err != nil {

‎engine/api/application_metric.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package api
2+
3+
import (
4+
"context"
5+
"net/http"
6+
7+
"github.com/gorilla/mux"
8+
9+
"github.com/ovh/cds/engine/api/application"
10+
"github.com/ovh/cds/engine/api/metrics"
11+
"github.com/ovh/cds/engine/service"
12+
"github.com/ovh/cds/sdk"
13+
)
14+
15+
func (api *API) getApplicationMetricHandler() service.Handler {
16+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
17+
vars := mux.Vars(r)
18+
key := vars["key"]
19+
appName := vars["permApplicationName"]
20+
metricName := vars["metricName"]
21+
22+
app, errA := application.LoadByName(api.mustDB(), api.Cache, key, appName, getUser(ctx))
23+
if errA != nil {
24+
return sdk.WrapError(errA, "getApplicationMetricHandler> unable to load application")
25+
}
26+
27+
result, err := metrics.GetMetrics(api.mustDB(), key, app.ID, metricName)
28+
if err != nil {
29+
return sdk.WrapError(err, "getApplicationMetricHandler> Cannot get metrics")
30+
31+
}
32+
return service.WriteJSON(w, result, http.StatusOK)
33+
}
34+
}

‎engine/api/metrics/elasticsearch.go

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package metrics
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"time"
7+
8+
"github.com/go-gorp/gorp"
9+
"gopkg.in/olivere/elastic.v5"
10+
11+
"github.com/ovh/cds/engine/api/services"
12+
"github.com/ovh/cds/sdk"
13+
"github.com/ovh/cds/sdk/log"
14+
"github.com/ovh/venom"
15+
)
16+
17+
var metricsChan chan sdk.Metric
18+
19+
func Init(c context.Context, DBFunc func() *gorp.DbMap) {
20+
metricsChan = make(chan sdk.Metric, 50)
21+
sdk.GoRoutine("metrics.PushInElasticSearch", func() { pushInElasticSearch(c, DBFunc) })
22+
}
23+
24+
func pushInElasticSearch(c context.Context, DBFunc func() *gorp.DbMap) {
25+
for {
26+
select {
27+
case <-c.Done():
28+
if c.Err() != nil {
29+
log.Error("metrics.pushInElasticSearch> Exiting: %v", c.Err())
30+
return
31+
}
32+
case e := <-metricsChan:
33+
db := DBFunc()
34+
esServices, errS := services.FindByType(db, services.TypeElasticsearch)
35+
if errS != nil {
36+
log.Error("metrics.pushInElasticSearch> Unable to get elasticsearch service: %v", errS)
37+
continue
38+
}
39+
40+
if len(esServices) == 0 {
41+
continue
42+
}
43+
44+
code, errD := services.DoJSONRequest(context.Background(), esServices, "POST", "/metrics", e, nil)
45+
if code >= 400 || errD != nil {
46+
log.Error("metrics.pushInElasticSearch> Unable to send metrics to elasticsearch [%d]: %v", code, errD)
47+
continue
48+
}
49+
}
50+
}
51+
}
52+
53+
// GetMetrics retrieves metrics from elasticsearch
54+
func GetMetrics(db gorp.SqlExecutor, key string, appID int64, metricName string) ([]json.RawMessage, error) {
55+
metricsRequest := sdk.MetricRequest{
56+
ProjectKey: key,
57+
ApplicationID: appID,
58+
Key: metricName,
59+
}
60+
61+
srvs, err := services.FindByType(db, services.TypeElasticsearch)
62+
if err != nil {
63+
return nil, sdk.WrapError(err, "GetMetrics> Unable to get elasticsearch service")
64+
}
65+
66+
var esMetrics []elastic.SearchHit
67+
if _, err := services.DoJSONRequest(context.Background(), srvs, "GET", "/metrics", metricsRequest, &esMetrics); err != nil {
68+
return nil, sdk.WrapError(err, "GetMetrics> Unable to get metrics")
69+
}
70+
71+
events := make([]json.RawMessage, 0, len(esMetrics))
72+
for _, h := range esMetrics {
73+
events = append(events, *h.Source)
74+
}
75+
return events, nil
76+
}
77+
78+
// PushVulnerabilities Create metrics from vulnerabilities and send them
79+
func PushVulnerabilities(projKey string, appID int64, workflowID int64, num int64, summary map[string]int64) {
80+
m := sdk.Metric{
81+
Date: time.Now(),
82+
ProjectKey: projKey,
83+
WorkflowID: workflowID,
84+
Num: num,
85+
ApplicationID: appID,
86+
Key: sdk.MetricKeyVulnerability,
87+
Value: summary,
88+
}
89+
metricsChan <- m
90+
}
91+
92+
// PushUnitTests Create metrics from unit tests and send them
93+
func PushUnitTests(projKey string, appID int64, workflowID int64, num int64, tests venom.Tests) {
94+
m := sdk.Metric{
95+
Date: time.Now(),
96+
ProjectKey: projKey,
97+
ApplicationID: appID,
98+
WorkflowID: workflowID,
99+
Key: sdk.MetricKeyUnitTest,
100+
Num: num,
101+
}
102+
103+
summary := make(map[string]int, 3)
104+
summary["total"] = tests.Total
105+
summary["ko"] = tests.TotalKO
106+
summary["ok"] = tests.TotalOK
107+
summary["skip"] = tests.TotalSkipped
108+
109+
m.Value = summary
110+
111+
metricsChan <- m
112+
}

‎engine/api/navbar.go

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎engine/api/repositoriesmanager/repositories_manager.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,20 @@ func (c *vcsClient) Branch(ctx context.Context, fullname string, branchName stri
303303
return &branch, nil
304304
}
305305

306+
// DefaultBranch get default branch from given repository
307+
func DefaultBranch(ctx context.Context, c sdk.VCSAuthorizedClient, fullname string) (string, error) {
308+
branches, err := c.Branches(ctx, fullname)
309+
if err != nil {
310+
return "", sdk.WrapError(err, "DefaultBranch> Unable to list branches")
311+
}
312+
for _, b := range branches {
313+
if b.Default {
314+
return b.DisplayID, nil
315+
}
316+
}
317+
return "", sdk.ErrNotFound
318+
}
319+
306320
func (c *vcsClient) Commits(ctx context.Context, fullname, branch, since, until string) ([]sdk.VCSCommit, error) {
307321
commits := []sdk.VCSCommit{}
308322
path := fmt.Sprintf("/vcs/%s/repos/%s/branches/commits?branch=%s&since=%s&until=%s", c.name, fullname, url.QueryEscape(branch), url.QueryEscape(since), url.QueryEscape(until))

‎engine/api/ui.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package api
2+
3+
import (
4+
"context"
5+
"net/http"
6+
7+
"github.com/gorilla/mux"
8+
9+
"github.com/ovh/cds/engine/api/application"
10+
"github.com/ovh/cds/engine/api/metrics"
11+
"github.com/ovh/cds/engine/api/navbar"
12+
"github.com/ovh/cds/engine/api/project"
13+
"github.com/ovh/cds/engine/api/repositoriesmanager"
14+
"github.com/ovh/cds/engine/api/workflow"
15+
"github.com/ovh/cds/engine/service"
16+
"github.com/ovh/cds/sdk"
17+
)
18+
19+
func (api *API) getNavbarHandler() service.Handler {
20+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
21+
data, err := navbar.LoadNavbarData(api.mustDB(), api.Cache, getUser(ctx))
22+
if err != nil {
23+
return sdk.WrapError(err, "getNavbarHandler")
24+
}
25+
return service.WriteJSON(w, data, http.StatusOK)
26+
}
27+
}
28+
29+
func (api *API) getApplicationOverviewHandler() service.Handler {
30+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
31+
vars := mux.Vars(r)
32+
key := vars["key"]
33+
appName := vars["permApplicationName"]
34+
35+
p, errP := project.Load(api.mustDB(), api.Cache, key, getUser(ctx))
36+
if errP != nil {
37+
return sdk.WrapError(errP, "getApplicationOverviewHandler> unable to load project")
38+
}
39+
40+
app, errA := application.LoadByName(api.mustDB(), api.Cache, key, appName, getUser(ctx))
41+
if errA != nil {
42+
return sdk.WrapError(errA, "getApplicationOverviewHandler> unable to load application")
43+
}
44+
45+
usage, errU := loadApplicationUsage(api.mustDB(), key, appName)
46+
if errU != nil {
47+
return sdk.WrapError(errU, "getApplicationOverviewHandler> Cannot load application usage")
48+
}
49+
app.Usage = &usage
50+
51+
appOverview := sdk.ApplicationOverview{
52+
Graphs: make([]sdk.ApplicationOverviewGraph, 0, 2),
53+
History: make(map[string][]sdk.WorkflowRun, len(app.Usage.Workflows)),
54+
}
55+
56+
// GET METRICS
57+
m1, errMV := metrics.GetMetrics(api.mustDB(), key, app.ID, sdk.MetricKeyVulnerability)
58+
if errMV != nil {
59+
return sdk.WrapError(errMV, "getApplicationOverviewHandler> Cannot list vulnerability metrics")
60+
}
61+
appOverview.Graphs = append(appOverview.Graphs, sdk.ApplicationOverviewGraph{
62+
Type: sdk.MetricKeyVulnerability,
63+
Datas: m1,
64+
})
65+
66+
m2, errUT := metrics.GetMetrics(api.mustDB(), key, app.ID, sdk.MetricKeyUnitTest)
67+
if errUT != nil {
68+
return sdk.WrapError(errUT, "getApplicationOverviewHandler> Cannot list Unit test metrics")
69+
}
70+
appOverview.Graphs = append(appOverview.Graphs, sdk.ApplicationOverviewGraph{
71+
Type: sdk.MetricKeyUnitTest,
72+
Datas: m2,
73+
})
74+
75+
// GET VCS URL
76+
// Get vcs info to known if we are on the default branch or not
77+
projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, app.VCSServer)
78+
client, erra := repositoriesmanager.AuthorizedClient(ctx, api.mustDB(), api.Cache, projectVCSServer)
79+
if erra != nil {
80+
return sdk.WrapError(erra, "getApplicationOverviewHandler> Cannot get repo client %s : %v", app.VCSServer)
81+
}
82+
vcsRepo, errRepo := client.RepoByFullname(ctx, app.RepositoryFullname)
83+
if errRepo != nil {
84+
return sdk.WrapError(errRepo, "getApplicationOverviewHandler> unable to get repo")
85+
}
86+
appOverview.GitURL = vcsRepo.URL
87+
defaultBranch, errB := repositoriesmanager.DefaultBranch(ctx, client, app.RepositoryFullname)
88+
if errB != nil {
89+
return sdk.WrapError(errB, "getApplicationOverviewHandler> Unable to get default branch")
90+
}
91+
92+
// GET LAST BUILD
93+
94+
tagFilter := make(map[string]string, 1)
95+
tagFilter["git.branch"] = defaultBranch
96+
for _, w := range app.Usage.Workflows {
97+
runs, _, _, _, errR := workflow.LoadRuns(api.mustDB(), key, w.Name, 0, 5, tagFilter)
98+
if errR != nil {
99+
return sdk.WrapError(errR, "getApplicationOverviewHandler> Unable to load runs")
100+
}
101+
appOverview.History[w.Name] = runs
102+
}
103+
104+
return service.WriteJSON(w, appOverview, http.StatusOK)
105+
}
106+
}

‎engine/api/navbar_test.go renamed to ‎engine/api/ui_test.go

File renamed without changes.

0 commit comments

Comments
 (0)
0