From 2a4c7c2e78043dad265ea1583231f9f4b89dfe33 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 11 Mar 2026 08:18:06 -0700 Subject: [PATCH 01/11] X-Smart-Branch-Parent: master From 70007d6df8d83aeffa3826127a15ff1625ce9579 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 1 Feb 2026 09:53:43 -0800 Subject: [PATCH 02/11] Added metrics for histogram of argument length, total bytes of arguments stored, and number of process indicators inserted --- .../datastore/datastore_impl.go | 1 + central/processindicator/datastore/metrics.go | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/central/processindicator/datastore/datastore_impl.go b/central/processindicator/datastore/datastore_impl.go index 887a4294d34d2..6f0623f301cb3 100644 --- a/central/processindicator/datastore/datastore_impl.go +++ b/central/processindicator/datastore/datastore_impl.go @@ -98,6 +98,7 @@ func (ds *datastoreImpl) AddProcessIndicators(ctx context.Context, indicators .. return err } } else { + recordProcessIndicatorsBatchAdded(identifierBatch) log.Debugf("successfully added a batch of %d process indicators", len(identifierBatch)) } } diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 601c2ed978273..36e1e99760916 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -2,6 +2,7 @@ package datastore import ( "github.com/prometheus/client_golang/prometheus" + "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/metrics" ) @@ -26,6 +27,28 @@ var ( Name: "process_pruning_cache_misses", Help: "Number of times we miss the cache, and have to evaluate, when trying to prune processes", }) + + processArgsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_args_size_bytes", + Help: "Distribution of process argument sizes in bytes for indicators written to database", + Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + + processIndicatorsAddedCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_indicators_added_total", + Help: "Total number of process indicators written to the database", + }) + + processArgsWrittenCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_args_written_bytes_total", + Help: "Total bytes of process arguments written to the database", + }) ) func incrementPrunedProcessesMetric(num int) { @@ -40,10 +63,37 @@ func incrementProcessPruningCacheMissesMetric() { processPruningCacheMisses.Inc() } +// getProcessArgsSizeBytes safely calculates the size of process args. +// Returns 0 if signal or args are nil/empty. +func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { + if indicator == nil || indicator.GetSignal() == nil { + return 0 + } + return len(indicator.GetSignal().GetArgs()) +} + +// recordProcessIndicatorAdded records metrics for a single process indicator added to DB. +func recordProcessIndicatorAdded(argsSize int) { + processArgsHistogram.Observe(float64(argsSize)) + processIndicatorsAddedCounter.Inc() + processArgsWrittenCounter.Add(float64(argsSize)) +} + +// recordProcessIndicatorsBatchAdded records metrics for a batch of process indicators successfully written to DB. +func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { + for _, indicator := range indicators { + argsSize := getProcessArgsSizeBytes(indicator) + recordProcessIndicatorAdded(argsSize) + } +} + func init() { prometheus.MustRegister( prunedProcesses, processPruningCacheHits, processPruningCacheMisses, + processArgsHistogram, + processIndicatorsAddedCounter, + processArgsWrittenCounter, ) } From 5f98b79195cff6dcf5c90e2461b5eda8dad5ab5b Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 4 Feb 2026 11:34:11 -0800 Subject: [PATCH 03/11] Removed a redundant metric --- central/processindicator/datastore/metrics.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 36e1e99760916..a4b180e87edf3 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -42,13 +42,6 @@ var ( Name: "process_indicators_added_total", Help: "Total number of process indicators written to the database", }) - - processArgsWrittenCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.CentralSubsystem.String(), - Name: "process_args_written_bytes_total", - Help: "Total bytes of process arguments written to the database", - }) ) func incrementPrunedProcessesMetric(num int) { @@ -76,7 +69,6 @@ func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { func recordProcessIndicatorAdded(argsSize int) { processArgsHistogram.Observe(float64(argsSize)) processIndicatorsAddedCounter.Inc() - processArgsWrittenCounter.Add(float64(argsSize)) } // recordProcessIndicatorsBatchAdded records metrics for a batch of process indicators successfully written to DB. @@ -94,6 +86,5 @@ func init() { processPruningCacheMisses, processArgsHistogram, processIndicatorsAddedCounter, - processArgsWrittenCounter, ) } From 84026e7b6169d96a4237be4b5a1e407ecb6d3cd5 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 10 Feb 2026 15:34:49 -0800 Subject: [PATCH 04/11] Removed recordProcessIndicatorAdded --- central/processindicator/datastore/metrics.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index a4b180e87edf3..b30884900e32d 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -65,17 +65,12 @@ func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { return len(indicator.GetSignal().GetArgs()) } -// recordProcessIndicatorAdded records metrics for a single process indicator added to DB. -func recordProcessIndicatorAdded(argsSize int) { - processArgsHistogram.Observe(float64(argsSize)) - processIndicatorsAddedCounter.Inc() -} - // recordProcessIndicatorsBatchAdded records metrics for a batch of process indicators successfully written to DB. func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { for _, indicator := range indicators { argsSize := getProcessArgsSizeBytes(indicator) - recordProcessIndicatorAdded(argsSize) + processArgsHistogram.Observe(float64(argsSize)) + processIndicatorsAddedCounter.Inc() } } From 9a04b18b39e849c6d801ae12309e7756fd59f9cb Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 2 Mar 2026 19:58:47 -0800 Subject: [PATCH 05/11] Added a metric for the number of characters in arguments --- central/processindicator/datastore/metrics.go | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index b30884900e32d..167d9c811a59f 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -1,6 +1,8 @@ package datastore import ( + "unicode/utf8" + "github.com/prometheus/client_golang/prometheus" "github.com/stackrox/rox/generated/storage" "github.com/stackrox/rox/pkg/metrics" @@ -36,6 +38,14 @@ var ( Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, }) + processArgsCharsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_args_size_chars", + Help: "Distribution of process argument sizes in characters for indicators written to database", + Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + processIndicatorsAddedCounter = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), @@ -56,7 +66,7 @@ func incrementProcessPruningCacheMissesMetric() { processPruningCacheMisses.Inc() } -// getProcessArgsSizeBytes safely calculates the size of process args. +// getProcessArgsSizeBytes safely calculates the size of process args in bytes. // Returns 0 if signal or args are nil/empty. func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { if indicator == nil || indicator.GetSignal() == nil { @@ -65,11 +75,22 @@ func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { return len(indicator.GetSignal().GetArgs()) } +// getProcessArgsSizeChars safely calculates the size of process args in characters (runes). +// Returns 0 if signal or args are nil/empty. +func getProcessArgsSizeChars(indicator *storage.ProcessIndicator) int { + if indicator == nil || indicator.GetSignal() == nil { + return 0 + } + return utf8.RuneCountInString(indicator.GetSignal().GetArgs()) +} + // recordProcessIndicatorsBatchAdded records metrics for a batch of process indicators successfully written to DB. func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { for _, indicator := range indicators { - argsSize := getProcessArgsSizeBytes(indicator) - processArgsHistogram.Observe(float64(argsSize)) + argsSizeBytes := getProcessArgsSizeBytes(indicator) + argsSizeChars := getProcessArgsSizeChars(indicator) + processArgsHistogram.Observe(float64(argsSizeBytes)) + processArgsCharsHistogram.Observe(float64(argsSizeChars)) processIndicatorsAddedCounter.Inc() } } @@ -80,6 +101,7 @@ func init() { processPruningCacheHits, processPruningCacheMisses, processArgsHistogram, + processArgsCharsHistogram, processIndicatorsAddedCounter, ) } From ea58b3e4b8be6243a9bd5a4fc6c2b53ea1f11700 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 3 Mar 2026 21:12:02 -0800 Subject: [PATCH 06/11] Removed processIndicatorsAddedCounter --- central/processindicator/datastore/metrics.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 167d9c811a59f..912cac6251d26 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -45,13 +45,6 @@ var ( Help: "Distribution of process argument sizes in characters for indicators written to database", Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, }) - - processIndicatorsAddedCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.CentralSubsystem.String(), - Name: "process_indicators_added_total", - Help: "Total number of process indicators written to the database", - }) ) func incrementPrunedProcessesMetric(num int) { @@ -91,7 +84,6 @@ func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { argsSizeChars := getProcessArgsSizeChars(indicator) processArgsHistogram.Observe(float64(argsSizeBytes)) processArgsCharsHistogram.Observe(float64(argsSizeChars)) - processIndicatorsAddedCounter.Inc() } } @@ -102,6 +94,5 @@ func init() { processPruningCacheMisses, processArgsHistogram, processArgsCharsHistogram, - processIndicatorsAddedCounter, ) } From b33b185e36b439db28441d2d685a1d525db97608 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sat, 7 Mar 2026 20:02:17 -0800 Subject: [PATCH 07/11] Removed process_args_size_bytes metric --- central/processindicator/datastore/metrics.go | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 912cac6251d26..a279ea987e648 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -30,14 +30,6 @@ var ( Help: "Number of times we miss the cache, and have to evaluate, when trying to prune processes", }) - processArgsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: metrics.PrometheusNamespace, - Subsystem: metrics.CentralSubsystem.String(), - Name: "process_args_size_bytes", - Help: "Distribution of process argument sizes in bytes for indicators written to database", - Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, - }) - processArgsCharsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), @@ -59,15 +51,6 @@ func incrementProcessPruningCacheMissesMetric() { processPruningCacheMisses.Inc() } -// getProcessArgsSizeBytes safely calculates the size of process args in bytes. -// Returns 0 if signal or args are nil/empty. -func getProcessArgsSizeBytes(indicator *storage.ProcessIndicator) int { - if indicator == nil || indicator.GetSignal() == nil { - return 0 - } - return len(indicator.GetSignal().GetArgs()) -} - // getProcessArgsSizeChars safely calculates the size of process args in characters (runes). // Returns 0 if signal or args are nil/empty. func getProcessArgsSizeChars(indicator *storage.ProcessIndicator) int { @@ -80,9 +63,7 @@ func getProcessArgsSizeChars(indicator *storage.ProcessIndicator) int { // recordProcessIndicatorsBatchAdded records metrics for a batch of process indicators successfully written to DB. func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { for _, indicator := range indicators { - argsSizeBytes := getProcessArgsSizeBytes(indicator) argsSizeChars := getProcessArgsSizeChars(indicator) - processArgsHistogram.Observe(float64(argsSizeBytes)) processArgsCharsHistogram.Observe(float64(argsSizeChars)) } } @@ -92,7 +73,6 @@ func init() { prunedProcesses, processPruningCacheHits, processPruningCacheMisses, - processArgsHistogram, processArgsCharsHistogram, ) } From 1d3e13b79450ffd5248b51d6e34f592201e1ae3b Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sat, 7 Mar 2026 20:07:36 -0800 Subject: [PATCH 08/11] Renamed process_args_size_chars to process_upserted_args_size --- central/processindicator/datastore/metrics.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index a279ea987e648..924b7d7657684 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -30,11 +30,11 @@ var ( Help: "Number of times we miss the cache, and have to evaluate, when trying to prune processes", }) - processArgsCharsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + processUpsertedArgsSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), - Name: "process_args_size_chars", - Help: "Distribution of process argument sizes in characters for indicators written to database", + Name: "process_upserted_args_size", + Help: "Distribution of process argument sizes in characters for upserted indicators", Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, }) ) @@ -64,7 +64,7 @@ func getProcessArgsSizeChars(indicator *storage.ProcessIndicator) int { func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { for _, indicator := range indicators { argsSizeChars := getProcessArgsSizeChars(indicator) - processArgsCharsHistogram.Observe(float64(argsSizeChars)) + processUpsertedArgsSizeHistogram.Observe(float64(argsSizeChars)) } } @@ -73,6 +73,6 @@ func init() { prunedProcesses, processPruningCacheHits, processPruningCacheMisses, - processArgsCharsHistogram, + processUpsertedArgsSizeHistogram, ) } From 3ce4c0a9a5e93c63c2dea2f72faabf0fc6cbfd36 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sat, 7 Mar 2026 20:16:44 -0800 Subject: [PATCH 09/11] The metric takes in the cluster id as a parameter --- central/processindicator/datastore/metrics.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 924b7d7657684..c8fb30fe832ee 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -30,13 +30,13 @@ var ( Help: "Number of times we miss the cache, and have to evaluate, when trying to prune processes", }) - processUpsertedArgsSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + processUpsertedArgsSizeHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "process_upserted_args_size", Help: "Distribution of process argument sizes in characters for upserted indicators", Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, - }) + }, []string{"cluster"}) ) func incrementPrunedProcessesMetric(num int) { @@ -64,7 +64,8 @@ func getProcessArgsSizeChars(indicator *storage.ProcessIndicator) int { func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { for _, indicator := range indicators { argsSizeChars := getProcessArgsSizeChars(indicator) - processUpsertedArgsSizeHistogram.Observe(float64(argsSizeChars)) + clusterID := indicator.GetClusterId() + processUpsertedArgsSizeHistogram.WithLabelValues(clusterID).Observe(float64(argsSizeChars)) } } From 0758d89b60df6e1cf568899d3cdfebcb5452fa51 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 12 Mar 2026 15:26:08 -0700 Subject: [PATCH 10/11] Process arguments lengths are now specified by cluster and namespace --- central/processindicator/datastore/metrics.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index c8fb30fe832ee..322b2679d6d93 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -36,7 +36,7 @@ var ( Name: "process_upserted_args_size", Help: "Distribution of process argument sizes in characters for upserted indicators", Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, - }, []string{"cluster"}) + }, []string{"cluster", "namespace"}) ) func incrementPrunedProcessesMetric(num int) { @@ -65,7 +65,8 @@ func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { for _, indicator := range indicators { argsSizeChars := getProcessArgsSizeChars(indicator) clusterID := indicator.GetClusterId() - processUpsertedArgsSizeHistogram.WithLabelValues(clusterID).Observe(float64(argsSizeChars)) + namespace := indicator.GetNamespace() + processUpsertedArgsSizeHistogram.WithLabelValues(clusterID, namespace).Observe(float64(argsSizeChars)) } } From ccc26a81f9d37660024871ee62efa0f74e5fe05a Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Fri, 13 Mar 2026 08:46:16 -0700 Subject: [PATCH 11/11] There is just one histogram for central. Argument lengths are still broken down by cluster and namespace --- central/processindicator/datastore/metrics.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/central/processindicator/datastore/metrics.go b/central/processindicator/datastore/metrics.go index 322b2679d6d93..958b1f178db1a 100644 --- a/central/processindicator/datastore/metrics.go +++ b/central/processindicator/datastore/metrics.go @@ -30,12 +30,19 @@ var ( Help: "Number of times we miss the cache, and have to evaluate, when trying to prune processes", }) - processUpsertedArgsSizeHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + processUpsertedArgsSizeHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: metrics.PrometheusNamespace, Subsystem: metrics.CentralSubsystem.String(), Name: "process_upserted_args_size", Help: "Distribution of process argument sizes in characters for upserted indicators", Buckets: []float64{0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}, + }) + + processUpsertedArgsSizeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.PrometheusNamespace, + Subsystem: metrics.CentralSubsystem.String(), + Name: "process_upserted_args_size_total", + Help: "Total process argument sizes in characters by cluster and namespace", }, []string{"cluster", "namespace"}) ) @@ -66,7 +73,10 @@ func recordProcessIndicatorsBatchAdded(indicators []*storage.ProcessIndicator) { argsSizeChars := getProcessArgsSizeChars(indicator) clusterID := indicator.GetClusterId() namespace := indicator.GetNamespace() - processUpsertedArgsSizeHistogram.WithLabelValues(clusterID, namespace).Observe(float64(argsSizeChars)) + + processUpsertedArgsSizeHistogram.Observe(float64(argsSizeChars)) + + processUpsertedArgsSizeTotal.WithLabelValues(clusterID, namespace).Add(float64(argsSizeChars)) } } @@ -76,5 +86,6 @@ func init() { processPruningCacheHits, processPruningCacheMisses, processUpsertedArgsSizeHistogram, + processUpsertedArgsSizeTotal, ) }