Skip to content

ROX-33626: Add Prometheus metrics to alert manager #19459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dashrews78 merged 2 commits into from
Mar 18, 2026
Merged

ROX-33626: Add Prometheus metrics to alert manager #19459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9664b10
ROX-33626: Add Prometheus metrics to alert manager
dashrews78 Mar 17, 2026
a634bf6
ROX-33626: Improve readability of alert manager metrics
dashrews78 Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions central/detection/alertmanager/alert_manager_impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ func getDeploymentIDsFromAlerts(alertSlices ...[]*storage.Alert) set.StringSet {

// AlertAndNotify is the main function that implements the AlertManager interface
func (d *alertManagerImpl) AlertAndNotify(ctx context.Context, currentAlerts []*storage.Alert, oldAlertFilters ...AlertFilterOption) (set.StringSet, error) {
defer observeDurationMs(alertAndNotifyDuration)()
alertAndNotifyIncomingCount.Observe(float64(len(currentAlerts)))

// Merge the old and the new alerts.
newAlerts, updatedAlerts, toBeResolvedAlerts, err := d.mergeManyAlerts(ctx, currentAlerts, oldAlertFilters...)
if err != nil {
Expand Down Expand Up @@ -377,6 +380,8 @@ func (d *alertManagerImpl) mergeManyAlerts(
incomingAlerts []*storage.Alert,
oldAlertFilters ...AlertFilterOption,
) (newAlerts, updatedAlerts, toBeResolvedAlerts []*storage.Alert, err error) {
defer observeDurationMs(mergeManyAlertsDuration)()

qb := search.NewQueryBuilder().AddExactMatches(
search.ViolationState,
storage.ViolationState_ACTIVE.String(),
Expand All @@ -389,6 +394,7 @@ func (d *alertManagerImpl) mergeManyAlerts(
err = errors.Wrapf(err, "couldn't load previous alerts (query was %s)", qb.Query())
return
}
mergeManyAlertsPreviousCount.Observe(float64(len(previousAlerts)))

// Merge any alerts that have new and old alerts.
for _, alert := range incomingAlerts {
Expand Down Expand Up @@ -445,6 +451,7 @@ func (d *alertManagerImpl) mergeManyAlerts(
}
}
}
recordAlertOutcomes(len(newAlerts), len(updatedAlerts), len(toBeResolvedAlerts))
return
}

Expand Down
77 changes: 77 additions & 0 deletions central/detection/alertmanager/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package alertmanager

import (
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/stackrox/rox/pkg/metrics"
)

// observeDurationMs returns a function that, when called, observes the elapsed
// time in milliseconds since observeDurationMs was invoked. Intended for use
// with defer:
//
// defer observeDurationMs(myHistogram)()
func observeDurationMs(h prometheus.Histogram) func() {
start := time.Now()
return func() {
h.Observe(float64(time.Since(start).Milliseconds()))
}
}

var (
alertAndNotifyDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metrics.PrometheusNamespace,
Subsystem: metrics.CentralSubsystem.String(),
Name: "alert_and_notify_duration_ms",
Help: "End-to-end duration of AlertAndNotify in milliseconds",
Buckets: prometheus.ExponentialBuckets(4, 2, 12),
})

alertAndNotifyIncomingCount = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metrics.PrometheusNamespace,
Subsystem: metrics.CentralSubsystem.String(),
Name: "alert_and_notify_incoming_count",
Help: "Number of incoming alerts per AlertAndNotify call",
Buckets: prometheus.ExponentialBuckets(1, 2, 14),
})

mergeManyAlertsDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metrics.PrometheusNamespace,
Subsystem: metrics.CentralSubsystem.String(),
Name: "merge_many_alerts_duration_ms",
Help: "Duration of mergeManyAlerts in milliseconds",
Buckets: prometheus.ExponentialBuckets(4, 2, 12),
})

mergeManyAlertsPreviousCount = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metrics.PrometheusNamespace,
Subsystem: metrics.CentralSubsystem.String(),
Name: "merge_many_alerts_previous_count",
Help: "Number of previous alerts fetched from DB per mergeManyAlerts call",
Buckets: prometheus.ExponentialBuckets(1, 2, 14),
})

alertOutcomeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: metrics.PrometheusNamespace,
Subsystem: metrics.CentralSubsystem.String(),
Name: "alert_outcome_total",
Help: "Cumulative count of alert outcomes from mergeManyAlerts",
}, []string{"outcome"})
)

func recordAlertOutcomes(newCount, updatedCount, resolvedCount int) {
alertOutcomeTotal.WithLabelValues("new").Add(float64(newCount))
alertOutcomeTotal.WithLabelValues("updated").Add(float64(updatedCount))
alertOutcomeTotal.WithLabelValues("resolved").Add(float64(resolvedCount))
}

func init() {
metrics.EmplaceCollector(
alertAndNotifyDuration,
alertAndNotifyIncomingCount,
mergeManyAlertsDuration,
mergeManyAlertsPreviousCount,
alertOutcomeTotal,
)
}
Loading