From 2beff6ca95e3b431de1ba476277ebf6cfb6d67ea Mon Sep 17 00:00:00 2001 From: Neven Miculinic Date: Wed, 17 Apr 2024 09:48:41 +0100 Subject: [PATCH] Added prometheus zeros init Signed-off-by: Neven Miculinic --- pkg/controller/chi/metrics.go | 15 +++++++++++++++ pkg/controller/chi/worker-chi-reconciler.go | 1 + 2 files changed, 16 insertions(+) diff --git a/pkg/controller/chi/metrics.go b/pkg/controller/chi/metrics.go index d8465cb46..ffa05a4c6 100644 --- a/pkg/controller/chi/metrics.go +++ b/pkg/controller/chi/metrics.go @@ -157,6 +157,21 @@ func prepareLabels(chi *api.ClickHouseInstallation) (attributes []attribute.KeyV return attributes } +// metricsCHIInitZeroValues initializes all metrics for CHI to zero values if not already present with appropriate labels +// +// This is due to `rate` prometheus function limitation where it expects the metric to be 0-initialized with all possible labels +// and doesn't default to 0 if the metric is not present. +func metricsCHIInitZeroValues(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().CHIReconcilesStarted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + ensureMetrics().CHIReconcilesCompleted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + ensureMetrics().CHIReconcilesAborted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + + ensureMetrics().HostReconcilesStarted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + ensureMetrics().HostReconcilesCompleted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + ensureMetrics().HostReconcilesRestarts.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + ensureMetrics().HostReconcilesErrors.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) +} + func metricsCHIReconcilesStarted(ctx context.Context, chi *api.ClickHouseInstallation) { ensureMetrics().CHIReconcilesStarted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } diff --git a/pkg/controller/chi/worker-chi-reconciler.go b/pkg/controller/chi/worker-chi-reconciler.go index bde05f053..dc8280572 100644 --- a/pkg/controller/chi/worker-chi-reconciler.go +++ b/pkg/controller/chi/worker-chi-reconciler.go @@ -59,6 +59,7 @@ func (w *worker) reconcileCHI(ctx context.Context, old, new *api.ClickHouseInsta w.a.M(new).S().P() defer w.a.M(new).E().P() + metricsCHIInitZeroValues(ctx, new) metricsCHIReconcilesStarted(ctx, new) startTime := time.Now()