8000 feat(api): queue metrics & refactor previous metrics · ovh/cds@34373ee · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 34373ee

Browse files
yesnaultyesnault
authored
feat(api): queue metrics & refactor previous metrics
Signed-off-by: Yvonnick Esnault <yvonnick.esnault@corp.ovh.com>
1 parent ab478b3 commit 34373ee

File tree

1 file changed

+60
-34
lines changed

1 file changed

+60
-34
lines changed

‎engine/api/metrics/metrics.go

Lines changed: 60 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,18 @@ var (
1919
func Initialize(c context.Context, DBFunc func() *gorp.DbMap, instance string) {
2020
labels := prometheus.Labels{"instance": instance}
2121

22-
nbUsers := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_users", Help: "metrics nb_users", ConstLabels: labels})
23-
nbApplications := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_applications", Help: "metrics nb_applications", ConstLabels: labels})
24-
nbProjects := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_projects", Help: "metrics nb_projects", ConstLabels: labels})
25-
nbGroups := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_groups", Help: "metrics nb_groups", ConstLabels: labels})
26-
nbPipelines := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_pipelines", Help: "metrics nb_pipelines", ConstLabels: labels})
27-
nbWorkflows := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_workflows", Help: "metrics nb_workflows", ConstLabels: labels})
28-
nbArtifacts := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_artifacts", Help: "metrics nb_artifacts", ConstLabels: labels})
29-
nbWorkerModels := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_worker_models", Help: "metrics nb_worker_models", ConstLabels: labels})
30-
nbWorkflowRuns := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_workflow_runs", Help: "metrics nb_workflow_runs", ConstLabels: labels})
31-
nbWorkflowNodeRuns := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_workflow_node_runs", Help: "metrics nb_workflow_node_runs", ConstLabels: labels})
32-
nbWorkflowNodeRunJobs := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_workflow_node_run_jobs", Help: "metrics nb_workflow_node_run_jobs", ConstLabels: labels})
33-
nbMaxWorkersBuilding := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_max_workers_building", Help: "metrics nb_max_workers_building", ConstLabels: labels})
34-
35-
nbOldPipelineBuilds := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_old_pipeline_builds", Help: "metrics nb_old_pipeline_builds", ConstLabels: labels})
36-
nbOldPipelineBuildJobs := prometheus.NewSummary(prometheus.SummaryOpts{Name: "nb_old_pipeline_build_jobs", Help: "metrics nb_old_pipeline_build_jobs", ConstLabels: labels})
22+
nbUsers := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_users", Help: "metrics nb_users", ConstLabels: labels})
23+
nbApplications := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_applications", Help: "metrics nb_applications", ConstLabels: labels})
24+
nbProjects := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_projects", Help: "metrics nb_projects", ConstLabels: labels})
25+
nbGroups := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_groups", Help: "metrics nb_groups", ConstLabels: labels})
26+
nbPipelines := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_pipelines", Help: "metrics nb_pipelines", ConstLabels: labels})
27+
nbWorkflows := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_workflows", Help: "metrics nb_workflows", ConstLabels: labels})
28+
nbArtifacts := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_artifacts", Help: "metrics nb_artifacts", ConstLabels: labels})
29+
nbWorkerModels := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_worker_models", Help: "metrics nb_worker_models", ConstLabels: labels})
30+
nbWorkflowRuns := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_workflow_runs", Help: "metrics nb_workflow_runs", ConstLabels: labels})
31+
nbWorkflowNodeRuns := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_workflow_node_runs", Help: "metrics nb_workflow_node_runs", ConstLabels: labels})
32+
nbMaxWorkersBuilding := prometheus.NewCounter(prometheus.CounterOpts{Name: "nb_max_workers_building", Help: "metrics nb_max_workers_building", ConstLabels: labels})
33+
queue := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "queue", Help: "metrics queue", ConstLabels: prometheus.Labels{"instance": instance}}, []string{"status", "range"})
3734

3835
registry.MustRegister(nbUsers)
3936
registry.MustRegister(nbApplications)
@@ -45,10 +42,8 @@ func Initialize(c context.Context, DBFunc func() *gorp.DbMap, instance string) {
4542
registry.MustRegister(nbWorkerModels)
4643
registry.MustRegister(nbWorkflowRuns)
4744
registry.MustRegister(nbWorkflowNodeRuns)
48-
registry.MustRegister(nbWorkflowNodeRunJobs)
49-
registry.MustRegister(nbOldPipelineBuilds)
50-
registry.MustRegister(nbOldPipelineBuildJobs)
5145
registry.MustRegister(nbMaxWorkersBuilding)
46+
registry.MustRegister(queue)
5247

5348
tick := time.NewTicker(30 * time.Second).C
5449

@@ -61,26 +56,44 @@ func Initialize(c context.Context, DBFunc func() *gorp.DbMap, instance string) {
6156
return
6257
}
6358
case <-tick:
64-
count(DBFunc(), "SELECT COUNT(1) FROM \"user\"", nbUsers)
65-
count(DBFunc(), "SELECT COUNT(1) FROM application", nbApplications)
66-
count(DBFunc(), "SELECT COUNT(1) FROM project", nbProjects)
67-
count(DBFunc(), "SELECT COUNT(1) FROM \"group\"", nbGroups)
68-
count(DBFunc(), "SELECT COUNT(1) FROM pipeline", nbPipelines)
69-
count(DBFunc(), "SELECT COUNT(1) FROM workflow", nbWorkflows)
70-
count(DBFunc(), "SELECT COUNT(1) FROM artifact", nbArtifacts)
71-
count(DBFunc(), "SELECT COUNT(1) FROM worker_model", nbWorkerModels)
72-
count(DBFunc(), "SELECT MAX(id) FROM workflow_run", nbWorkflowRuns)
73-
count(DBFunc(), "SELECT MAX(id) FROM workflow_node_run", nbWorkflowNodeRuns)
74-
count(DBFunc(), "SELECT MAX(id) FROM workflow_node_run_job", nbWorkflowNodeRunJobs)
75-
count(DBFunc(), "SELECT MAX(id) FROM pipeline_build", nbOldPipelineBuilds)
76-
count(DBFunc(), "SELECT MAX(id) FROM pipeline_build_job", nbOldPipelineBuildJobs)
77-
count(DBFunc(), "SELECT COUNT(1) FROM worker where status like 'Building' ", nbMaxWorkersBuilding)
59+
count(DBFunc(), nbUsers, "SELECT COUNT(1) FROM \"user\"")
60+
count(DBFunc(), nbApplications, "SELECT COUNT(1) FROM application")
61+
count(DBFunc(), nbProjects, "SELECT COUNT(1) FROM project")
62+
count(DBFunc(), nbGroups, "SELECT COUNT(1) FROM \"group\"")
63< 8000 /code>+
count(DBFunc(), nbPipelines, "SELECT COUNT(1) FROM pipeline")
64+
count(DBFunc(), nbWorkflows, "SELECT COUNT(1) FROM workflow")
65+
count(DBFunc(), nbArtifacts, "SELECT COUNT(1) FROM artifact")
66+
count(DBFunc(), nbWorkerModels, "SELECT COUNT(1) FROM worker_model")
67+
count(DBFunc(), nbWorkflowRuns, "SELECT MAX(id) FROM workflow_run")
68+
count(DBFunc(), nbWorkflowNodeRuns, "SELECT MAX(id) FROM workflow_node_run")
69+
count(DBFunc(), nbMaxWorkersBuilding, "SELECT COUNT(1) FROM worker where status = 'Building'")
70+
71+
now := time.Now()
72+
now10s := now.Add(-10 * time.Second)
73+
now30s := now.Add(-30 * time.Second)
74+
now1min := now.Add(-1 * time.Minute)
75+
now2min := now.Add(-2 * time.Minute)
76+
now5min := now.Add(-5 * time.Minute)
77+
now10min := now.Add(-10 * time.Minute)
78+
79+
queryBuilding := "SELECT COUNT(1) FROM workflow_node_run_job where status = 'Building'"
80+
query := "select COUNT(1) from workflow_node_run_job where queued > $1 and queued <= $2 and status = 'Waiting'"
81+
queryOld := "select COUNT(1) from workflow_node_run_job where queued < $1 and status = 'Waiting'"
82+
83+
countGauge(DBFunc(), *queue, "building", "all", queryBuilding)
84+
countGauge(DBFunc(), *queue, "waiting", "less_10s", query, now10s, now)
85+
countGauge(DBFunc(), *queue, "waiting", "more_10s_less_30s", query, now30s, now10s)
86+
countGauge(DBFunc(), *queue, "waiting", "more_30s_less_1min", query, now1min, now30s)
87+
countGauge(DBFunc(), *queue, "waiting", "more_1min_less_2min", query, now2min, now1min)
88+
countGauge(DBFunc(), *queue, "waiting", "more_2min_less_5min", query, now5min, now2min)
89+
countGauge(DBFunc(), *queue, "waiting", "more_5min_less_10min", query, now10min, now5min)
90+
countGauge(DBFunc(), *queue, "waiting", "more_10min", queryOld, now10min)
7891
}
7992
}
8093
}(c, DBFunc)
8194
}
8295

83-
func count(db *gorp.DbMap, query string, v prometheus.Summary) {
96+
func count(db *gorp.DbMap, v prometheus.Counter, query string) {
8497
if db == nil {
8598
return
8699
}
@@ -90,9 +103,22 @@ func count(db *gorp.DbMap, query string, v prometheus.Summary) {
90103
return
91104
}
92105
if n.Valid {
93-
v.Observe(float64(n.Int64))
106+
v.Set(float64(n.Int64))
94107
}
108+
}
95109

110+
func countGauge(db *gorp.DbMap, v prometheus.GaugeVec, status, timerange string, query string, args ...interface{}) {
111+
if db == nil {
112+
return
113+
}
114+
var n sql.NullInt64
115+
if err := db.QueryRow(query, args...).Scan(&n); err != nil {
116+
log.Warning("metrics>Errors while fetching count %s: %v", query, err)
117+
return
118+
}
119+
if n.Valid {
120+
v.WithLabelValues(status, timerange).Set(float64(n.Int64))
121+
}
96122
}
97123

98124
// GetGatherer returns CDS API gatherer

0 commit comments

Comments
 (0)
0