From 41df802a09070014e75378d307259535832530fe Mon Sep 17 00:00:00 2001 From: Guzman Date: Tue, 31 Mar 2026 01:05:16 +0200 Subject: [PATCH 1/4] ci: split Go unit test job into parallel component shards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single `go` job running all ~940 Go unit test packages is split into 4 parallel shards (central, pkg, sensor, other) × 2 gotags variants = 8 parallel jobs. This reduces wall-clock feedback time significantly. Changes: - Makefile: add UNIT_TEST_PKG_INCLUDE/EXCLUDE filter variables (grep patterns) to go-unit-tests and go-postgres-unit-tests targets. Defaults are no-op so local `make test` behavior is unchanged. - Makefile: extract go-log-level-tests into its own target (run only in the pkg shard in CI, still included in `make test`). - Workflow: replace single `go` job with `go-unit-tests` (4-component matrix) and `go-extra-tests` (integration + operator tests). - Workflow: update slack notification needs and mention_author references. Package distribution across shards: central: 469, pkg: 244, sensor+compliance: 90, other: 137 Partially generated by AI. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/unit-tests.yaml | 69 +++++++++++++++++++++++++++++-- Makefile | 19 ++++++--- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index d0fcecc988a2b..bb50f34be834f 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -17,11 +17,25 @@ concurrency: cancel-in-progress: true jobs: - go: + go-unit-tests: + name: "Go Unit Tests (${{ matrix.component.name }}, ${{ matrix.gotags }})" strategy: fail-fast: false matrix: gotags: [ 'GOTAGS=""', 'GOTAGS=release' ] + component: + - name: central + pkg_include: 'stackrox/rox/central/' + pkg_exclude: '^$' + - name: pkg + pkg_include: 'stackrox/rox/pkg/' + pkg_exclude: '^$' + - name: sensor + pkg_include: 'stackrox/rox/(sensor|compliance)/' + pkg_exclude: '^$' + - name: other + pkg_include: '.' + pkg_exclude: 'stackrox/rox/(central|pkg|sensor|compliance)/' runs-on: ubuntu-latest outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} @@ -30,6 +44,9 @@ jobs: volumes: - /usr:/mnt/usr - /opt:/mnt/opt + env: + UNIT_TEST_PKG_INCLUDE: ${{ matrix.component.pkg_include }} + UNIT_TEST_PKG_EXCLUDE: ${{ matrix.component.pkg_exclude }} steps: - name: Checkout uses: actions/checkout@v6 @@ -44,7 +61,7 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies - - name: Go Unit Tests + - name: Go Unit Tests (${{ matrix.component.name }}) run: ${{ matrix.gotags }} make go-unit-tests - uses: codecov/codecov-action@v3 @@ -52,6 +69,10 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} flags: go-unit-tests + - name: Go Log Level Tests + if: matrix.component.name == 'pkg' + run: ${{ matrix.gotags }} make go-log-level-tests + - name: Generate junit report if: always() run: make generate-junit-reports @@ -62,6 +83,45 @@ jobs: with: paths: 'junit-reports/report.xml' + - name: Report test failures to Jira + if: (!cancelled()) + id: junit2jira + uses: ./.github/actions/junit2jira + with: + create-jiras: ${{ github.event_name == 'push' }} + jira-user: ${{ secrets.JIRA_USER }} + jira-token: ${{ secrets.JIRA_TOKEN }} + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + directory: 'junit-reports' + + go-extra-tests: + name: "Go Extra Tests (${{ matrix.gotags }})" + strategy: + fail-fast: false + matrix: + gotags: [ 'GOTAGS=""', 'GOTAGS=release' ] + runs-on: ubuntu-latest + outputs: + new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} + container: + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.3 + volumes: + - /usr:/mnt/usr + - /opt:/mnt/opt + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: ./.github/actions/job-preamble + with: + gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + free-disk-space: 30 + + - name: Cache Go dependencies + uses: ./.github/actions/cache-go-dependencies + - name: Go Integration Unit Tests run: ${{ matrix.gotags }} make integration-unit-tests @@ -481,7 +541,8 @@ jobs: name: Post failure message to Slack runs-on: ubuntu-latest needs: - - go + - go-unit-tests + - go-extra-tests - go-bench - go-postgres - local-roxctl-tests @@ -499,7 +560,7 @@ jobs: - name: Slack message env: GITHUB_CONTEXT: ${{ toJSON(github) }} - mention_author: ${{ needs.go.outputs.new-jiras || needs.go-postgres.outputs.new-jiras || needs.local-roxctl-tests.outputs.new-jiras || needs.ui.outputs.new-jiras || needs.go.outputs.new-jiras || needs.shell-unit-tests.outputs.new-jiras || needs.sensor-integration-tests.outputs.new-jiras }} + mention_author: ${{ needs.go-unit-tests.outputs.new-jiras || needs.go-extra-tests.outputs.new-jiras || needs.go-postgres.outputs.new-jiras || needs.local-roxctl-tests.outputs.new-jiras || needs.ui.outputs.new-jiras || needs.shell-unit-tests.outputs.new-jiras || needs.sensor-integration-tests.outputs.new-jiras }} run: | source scripts/ci/lib.sh slack_workflow_failure diff --git a/Makefile b/Makefile index 63c4ad12659c8..70afb292262b2 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,12 @@ SILENT ?= @ # TODO: [ROX-19070] Update postgres store test generation to work for foreign keys UNIT_TEST_IGNORE := "stackrox/rox/sensor/tests|stackrox/rox/operator/tests|stackrox/rox/central/reports/config/store/postgres|stackrox/rox/central/complianceoperator/v2/scanconfigurations/store/postgres|stackrox/rox/central/auth/store/postgres|stackrox/rox/scanner/e2etests" +# UNIT_TEST_PKG_INCLUDE/EXCLUDE allow filtering the package list for parallel CI sharding. +# UNIT_TEST_PKG_INCLUDE: grep -E pattern to include only matching packages (default: match all) +# UNIT_TEST_PKG_EXCLUDE: grep -Ev pattern to exclude matching packages (default: match none) +UNIT_TEST_PKG_INCLUDE ?= . +UNIT_TEST_PKG_EXCLUDE ?= ^$$ + GOBUILD := $(CURDIR)/scripts/go-build.sh DOCKERBUILD := $(CURDIR)/scripts/docker-build.sh GO_TEST_OUTPUT_PATH=$(CURDIR)/test-output/test.log @@ -533,9 +539,12 @@ test-prep: go-unit-tests: build-prep test-prep set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test scripts/go-test.sh -timeout 25m -race -cover -coverprofile test-output/coverage.out -v \ - $(shell git ls-files -- '*_test.go' | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list| grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git ls-files -- '*_test.go' | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list| grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE) | grep -E "$(UNIT_TEST_PKG_INCLUDE)" | grep -Ev "$(UNIT_TEST_PKG_EXCLUDE)") \ | tee $(GO_TEST_OUTPUT_PATH) - # Exercise the logging package for all supported logging levels to make sure that initialization works properly + +# Exercise the logging package for all supported logging levels to make sure that initialization works properly. +.PHONY: go-log-level-tests +go-log-level-tests: @echo "Run log tests" for encoding in console json; do \ for level in debug info warn error fatal panic; do \ @@ -555,12 +564,12 @@ sensor-pipeline-benchmark: build-prep test-prep go-postgres-unit-tests: build-prep test-prep set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test,sql_integration scripts/go-test.sh -timeout 15m -race -cover -coverprofile test-output/coverage.out -v \ - $(shell git grep -rl "//go:build sql_integration" central pkg tools | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git grep -rl "//go:build sql_integration" central pkg tools | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE) | grep -E "$(UNIT_TEST_PKG_INCLUDE)" | grep -Ev "$(UNIT_TEST_PKG_EXCLUDE)") \ | tee $(GO_TEST_OUTPUT_PATH) @# The -p 1 passed to go test is required to ensure that tests of different packages are not run in parallel, so as to avoid conflicts when interacting with the DB. set -o pipefail ; \ CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 MUTEX_WATCHDOG_TIMEOUT_SECS=30 GOTAGS=$(GOTAGS),test,sql_integration scripts/go-test.sh -p 1 -race -cover -coverprofile test-output/migrator-coverage.out -v \ - $(shell git grep -rl "//go:build sql_integration" migrator | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE)) \ + $(shell git grep -rl "//go:build sql_integration" migrator | sed -e 's@^@./@g' | xargs -n 1 dirname | sort | uniq | xargs go list -tags sql_integration | grep -v '^github.com/stackrox/rox/tests$$' | grep -Ev $(UNIT_TEST_IGNORE) | grep -E "$(UNIT_TEST_PKG_INCLUDE)" | grep -Ev "$(UNIT_TEST_PKG_EXCLUDE)") \ | tee -a $(GO_TEST_OUTPUT_PATH) .PHONY: go-postgres-bench-tests @@ -595,7 +604,7 @@ ui-component-tests: make -C ui test-component .PHONY: test -test: go-unit-tests ui-test shell-unit-tests +test: go-unit-tests go-log-level-tests ui-test shell-unit-tests .PHONY: integration-unit-tests integration-unit-tests: build-prep test-prep From 5d63808ec2db1d71fbcb8f87d1073ec78d2eeb53 Mon Sep 17 00:00:00 2001 From: Guzman Date: Tue, 31 Mar 2026 01:43:52 +0200 Subject: [PATCH 2/4] test: add deliberate sensor test failure for shard isolation validation Temporary commit to prove that the split CI jobs correctly isolate failures to the failing component's shard. Will be reverted after data collection. Co-Authored-By: Claude Opus 4.6 --- sensor/common/sensor/ci_shard_probe_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 sensor/common/sensor/ci_shard_probe_test.go diff --git a/sensor/common/sensor/ci_shard_probe_test.go b/sensor/common/sensor/ci_shard_probe_test.go new file mode 100644 index 0000000000000..50711a2785295 --- /dev/null +++ b/sensor/common/sensor/ci_shard_probe_test.go @@ -0,0 +1,8 @@ +package sensor + +import "testing" + +// Temporary test to validate CI shard isolation - will be reverted. +func TestCIShardIsolationProbe(t *testing.T) { + t.Fatal("deliberate failure to prove shard isolation") +} From 38dbc4136770ff2ba28bcec1bcf63943310b434f Mon Sep 17 00:00:00 2001 From: Guzman Date: Tue, 31 Mar 2026 02:10:01 +0200 Subject: [PATCH 3/4] test: revert deliberate sensor test failure Shard isolation validated: sensor shard failed in ~11min while all other shards (central, pkg, other, extra) passed independently. Co-Authored-By: Claude Opus 4.6 --- sensor/common/sensor/ci_shard_probe_test.go | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 sensor/common/sensor/ci_shard_probe_test.go diff --git a/sensor/common/sensor/ci_shard_probe_test.go b/sensor/common/sensor/ci_shard_probe_test.go deleted file mode 100644 index 50711a2785295..0000000000000 --- a/sensor/common/sensor/ci_shard_probe_test.go +++ /dev/null @@ -1,8 +0,0 @@ -package sensor - -import "testing" - -// Temporary test to validate CI shard isolation - will be reverted. -func TestCIShardIsolationProbe(t *testing.T) { - t.Fatal("deliberate failure to prove shard isolation") -} From f4e33dd45da4b20224f63a7213f78219faaae4cb Mon Sep 17 00:00:00 2001 From: Guzman Date: Sun, 5 Apr 2026 17:26:16 +0200 Subject: [PATCH 4/4] fix: add build-prep and test-prep prerequisites to go-log-level-tests target When go-log-level-tests was extracted from go-unit-tests into its own target, it lost the inherited build-prep and test-prep prerequisites. Running `make go-log-level-tests` standalone would fail due to missing setup. Restore these prerequisites so the target works independently. Addresses review feedback from sourcery-ai. Partially generated by AI. Co-Authored-By: Claude Opus 4.6 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 70afb292262b2..d8d3085d39eac 100644 --- a/Makefile +++ b/Makefile @@ -544,7 +544,7 @@ go-unit-tests: build-prep test-prep # Exercise the logging package for all supported logging levels to make sure that initialization works properly. .PHONY: go-log-level-tests -go-log-level-tests: +go-log-level-tests: build-prep test-prep @echo "Run log tests" for encoding in console json; do \ for level in debug info warn error fatal panic; do \