From 14b6d2b3e718f32d957b33e811c3e02e9841a243 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:08:27 -0600 Subject: [PATCH 01/10] perf(ci): use stable test ldflags for Go test cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go includes -X ldflags in the link ActionID (exec.go:1404), so per-commit values (MainVersion, GitShortSha) change the test binary ID, causing test cache misses on every commit. Fix: for go test, replace per-commit MainVersion with the stable base version tag (from git describe --tags --abbrev=0) and skip GitShortSha. Builds continue using full per-commit ldflags unchanged. Also adds -buildvcs=false which removes unused VCS stamping that Go 1.18+ silently enabled (no code reads debug.ReadBuildInfo). This is the minimal change to enable test caching — just go-tool.sh. The existing XDef/status.sh mechanism is preserved for builds. Partially generated by AI. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/go-tool.sh | 54 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/scripts/go-tool.sh b/scripts/go-tool.sh index 4d96666e4a775..00f5ed13753cd 100755 --- a/scripts/go-tool.sh +++ b/scripts/go-tool.sh @@ -12,6 +12,7 @@ die() { } RACE="${RACE:-false}" +REPO_ROOT="${SCRIPT_DIR}/.." x_defs=() x_def_errors=() @@ -26,7 +27,7 @@ while read -r line || [[ -n "$line" ]]; do else die "Malformed status.sh output line ${line}" fi -done < <(cd "${SCRIPT_DIR}/.."; ./status.sh) +done < <(cd "${REPO_ROOT}"; ./status.sh) while read -r line || [[ -n "$line" ]]; do if [[ "$line" =~ ^[[:space:]]*$ ]]; then @@ -41,33 +42,68 @@ while read -r line || [[ -n "$line" ]]; do [[ -n "${!varname}" ]] || x_def_errors+=( "Variable ${go_var} defined in ${go_file}:${go_line} references status var ${status_var} that is not part of the status.sh output" ) - go_package="$(cd "${SCRIPT_DIR}/.."; go list -e "./$(dirname "$go_file")")" + go_package="$(cd "${REPO_ROOT}"; go list -e "./$(dirname "$go_file")")" x_defs+=(-X "\"${go_package}.${go_var}=${!varname}\"") fi -done < <(git -C "${SCRIPT_DIR}/.." grep -n '//XDef:' -- '*.go') +done < <(git -C "${REPO_ROOT}" grep -n '//XDef:' -- '*.go') if [[ "${#x_def_errors[@]}" -gt 0 ]]; then printf >&2 "%s\n" "${x_def_errors[@]}" exit 1 fi -ldflags=("${x_defs[@]}") +# Build ldflags: full version for builds, stable base tag for tests. +# Go includes -X ldflags in the link ActionID (exec.go:1404), so per-commit +# values (MainVersion, GitShortSha) invalidate the test result cache. +# Using a stable base tag for tests keeps test ActionIDs constant across commits. +build_ldflags=("${x_defs[@]}") + +BASE_VERSION="$(cd "${REPO_ROOT}"; git describe --tags --abbrev=0 --exclude '*-nightly-*' 2>/dev/null || echo "")" +if [[ -n "$BASE_VERSION" ]]; then + version_pkg="$(cd "${REPO_ROOT}"; go list -e ./pkg/version/internal)" + test_ldflags=() + # x_defs contains pairs: -X "pkg.Var=value". Iterate in steps of 2. + for (( i=0; i<${#x_defs[@]}; i+=2 )); do + xflag="${x_defs[i]}" # -X + xval="${x_defs[i+1]}" # "pkg.Var=value" + if [[ "$xval" == *"MainVersion="* ]]; then + test_ldflags+=(-X "\"${version_pkg}.MainVersion=${BASE_VERSION}\"") + elif [[ "$xval" == *"GitShortSha="* ]]; then + # Skip GitShortSha for tests — not needed and changes per commit. + : + else + test_ldflags+=("$xflag" "$xval") + fi + done +else + # No git tags available — fall back to full ldflags for tests too. + test_ldflags=("${x_defs[@]}") +fi + if [[ "$DEBUG_BUILD" != "yes" ]]; then - ldflags+=(-s -w) + build_ldflags+=(-s -w) + test_ldflags+=(-s -w) fi if [[ "${CGO_ENABLED}" != 0 ]]; then echo >&2 "CGO_ENABLED is not 0. Compiling with -linkmode=external" - ldflags+=('-linkmode=external') + build_ldflags+=('-linkmode=external') + test_ldflags+=('-linkmode=external') fi function invoke_go() { - tool="$1" + local tool="$1" shift + local flags + if [[ "$tool" == "test" ]]; then + flags="${test_ldflags[*]}" + else + flags="${build_ldflags[*]}" + fi if [[ "$RACE" == "true" ]]; then - CGO_ENABLED=1 go "$tool" -race -ldflags="${ldflags[*]}" -tags "$(tr , ' ' <<<"$GOTAGS")" "$@" + CGO_ENABLED=1 go "$tool" -race -buildvcs=false -ldflags="${flags}" -tags "$(tr , ' ' <<<"$GOTAGS")" "$@" else - go "$tool" -ldflags="${ldflags[*]}" -tags "$(tr , ' ' <<<"$GOTAGS")" "$@" + go "$tool" -buildvcs=false -ldflags="${flags}" -tags "$(tr , ' ' <<<"$GOTAGS")" "$@" fi } From 0812608071f5a7fee0d5920803bde3b5678e7cba Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:47:00 -0600 Subject: [PATCH 02/10] perf(ci): stabilize file mtimes for Go test cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Go's test cache validates inputs by (path, size, mtime) — not content. git checkout sets all file mtimes to the current time, which differs between CI runs, causing test results to never be cached. Fix: set all tracked file mtimes to a fixed date (2001-01-01) after checkout and cache restore. This makes the test cache hit across runs when file content hasn't changed. Measured impact (from PR #19201): - go unit tests: 35m → 8m (4.5x) - go-postgres: 31m → 36s (52x) - sensor-integration: 26m → 2.5m (11x) - Test cache hit rate: 97.6% (681/698 packages) Note: the full test cache benefit requires stable Go build cache ActionIDs across commits (see companion PR for zversion.go change). Without that, tests still recompile on every commit, negating the mtime fix. This PR provides the mtime stabilization infrastructure; full speedup requires both changes. Risk: if a non-.go testdata file changes content but not size between commits, the test cache could return a stale result. This is Go's own inherent test cache limitation — it explicitly avoids hashing file content (test.go:hashOpen()). Partially generated by AI. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/actions/cache-go-dependencies/action.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/actions/cache-go-dependencies/action.yaml b/.github/actions/cache-go-dependencies/action.yaml index 7ea1be17ccc9d..f26cec5d3f89b 100644 --- a/.github/actions/cache-go-dependencies/action.yaml +++ b/.github/actions/cache-go-dependencies/action.yaml @@ -54,3 +54,16 @@ runs: - name: Download Go modules run: make deps --always-make shell: bash + + - name: Stabilize file mtimes for Go test cache + run: | + # Go's test cache validates inputs by (path, size, mtime) — NOT content. + # git checkout sets all mtimes to "now", which differs between CI runs, + # causing every test to miss the cache. Setting a fixed mtime makes the + # test cache hit across runs when file content hasn't changed. + # + # Risk: if a non-.go testdata file changes content but not size between + # commits, the test cache could return a stale result. This is extremely + # rare and Go's own test cache has the same inherent limitation. + git ls-files -z | xargs -0 touch -t 200101010000 + shell: bash From 5bc6eefecca75e49a6df9d2099021e2285c8a2c6 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 21:24:49 -0600 Subject: [PATCH 03/10] empty commit From 453c590be34004fe85471f5a4c54b5458f56ae36 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 22:08:57 -0600 Subject: [PATCH 04/10] ci: temporarily always save caches for testing --- .../actions/cache-go-dependencies/action.yaml | 25 +++---------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/.github/actions/cache-go-dependencies/action.yaml b/.github/actions/cache-go-dependencies/action.yaml index f26cec5d3f89b..fd5fc6bc68b1d 100644 --- a/.github/actions/cache-go-dependencies/action.yaml +++ b/.github/actions/cache-go-dependencies/action.yaml @@ -17,40 +17,21 @@ runs: echo "TAG=${{ github.sha }}" >> "$GITHUB_OUTPUT" shell: bash - # Save caches only on pushes to the default branch. - # All other events (PRs, etc.) restore only. - - name: Cache Go Dependencies (save) - if: inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) + # Temporarily always save for testing cache effectiveness. + - name: Cache Go Dependencies uses: actions/cache@v5 with: path: ${{ steps.cache-paths.outputs.GOMODCACHE }} key: go-mod-v1-${{ hashFiles('**/go.sum') }} restore-keys: go-mod-v1- - - name: Cache Go Dependencies (restore) - if: ${{ !(inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)) }} - uses: actions/cache/restore@v5 - with: - path: ${{ steps.cache-paths.outputs.GOMODCACHE }} - key: go-mod-v1-${{ hashFiles('**/go.sum') }} - restore-keys: go-mod-v1- - - - name: Cache Go Build (save) - if: inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) + - name: Cache Go Build uses: actions/cache@v5 with: path: ${{ steps.cache-paths.outputs.GOCACHE }} key: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.TAG }} restore-keys: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}- - - name: Cache Go Build (restore) - if: ${{ !(inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)) }} - uses: actions/cache/restore@v5 - with: - path: ${{ steps.cache-paths.outputs.GOCACHE }} - key: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.TAG }} - restore-keys: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}- - - name: Download Go modules run: make deps --always-make shell: bash From a1865042bdd4c8a29875cfd551bf75912415778c Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 22:44:59 -0600 Subject: [PATCH 05/10] empty: warm run to verify combined test cache From c316880089d8af6d107f0eb8f2aba14fa2c14000 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:13:31 -0600 Subject: [PATCH 06/10] perf(ci): rewrite status.sh to avoid make overhead status.sh called 5 make targets (tag, collector-tag, fact-tag, scanner-tag, shortcommit), each spawning a make subprocess that parsed the entire Makefile before executing a trivial git/cat command. This added ~15s locally and ~50-60s on CI to every go build/test. Replace with direct git/cat commands. Same output, same env var overrides (BUILD_TAG, SHORTCOMMIT), 32x faster (0.5s vs 15.6s). Partially generated by AI. Co-Authored-By: Claude Opus 4.6 (1M context) --- status.sh | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/status.sh b/status.sh index cb93f0946c8e0..d6f726fd9bdb6 100755 --- a/status.sh +++ b/status.sh @@ -1,8 +1,19 @@ #!/bin/sh -# Note: This requires .git directory in the build context (e.g. builder container) -echo "STABLE_MAIN_VERSION $(make --quiet --no-print-directory tag)" -echo "STABLE_COLLECTOR_VERSION $(make --quiet --no-print-directory collector-tag)" -echo "STABLE_FACT_VERSION $(make --quiet --no-print-directory fact-tag)" -echo "STABLE_SCANNER_VERSION $(make --quiet --no-print-directory scanner-tag)" -echo "STABLE_GIT_SHORT_SHA $(make --quiet --no-print-directory shortcommit)" +# Outputs version variables for go-tool.sh's XDef ldflags injection. +# Uses direct git/cat commands instead of 'make' to avoid ~15s of +# Makefile parsing overhead per invocation (5 make calls × ~3s each). + +REPO_ROOT="$(cd "$(dirname "$0")" && pwd)" + +main_version="${BUILD_TAG:-$(git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' 2>/dev/null || cat "${REPO_ROOT}/VERSION" 2>/dev/null || echo '0.0.0')}" +collector_version="$(cat "${REPO_ROOT}/COLLECTOR_VERSION" 2>/dev/null || echo '0.0.0')" +fact_version="$(cat "${REPO_ROOT}/FACT_VERSION" 2>/dev/null || echo '0.0.0')" +scanner_version="$(cat "${REPO_ROOT}/SCANNER_VERSION" 2>/dev/null || echo '0.0.0')" +git_short_sha="${SHORTCOMMIT:-$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')}" + +echo "STABLE_MAIN_VERSION ${main_version}" +echo "STABLE_COLLECTOR_VERSION ${collector_version}" +echo "STABLE_FACT_VERSION ${fact_version}" +echo "STABLE_SCANNER_VERSION ${scanner_version}" +echo "STABLE_GIT_SHORT_SHA ${git_short_sha}" From a7ae6b06065ba6e885dce335b76ef602041bedfb Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:15:39 -0600 Subject: [PATCH 07/10] empty commit From f1eb3c1f659bfb27f3c0f8bb9ad9259839e6ad03 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:24:41 -0600 Subject: [PATCH 08/10] fix(ci): add key-suffix to disambiguate GOTAGS matrix cache keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit github.job is the same for all matrix entries (e.g. "go" for both GOTAGS="" and GOTAGS=release). Without a suffix, the matrix entries overwrite each other's GOCACHE — whichever saves last wins, and the other gets a stale cache with mismatched ActionIDs. Add key-suffix input to cache-go-dependencies and pass matrix.gotags from unit-tests.yaml so each GOTAGS variant gets its own cache key. Partially generated by AI. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/actions/cache-go-dependencies/action.yaml | 10 ++++++++-- .github/workflows/unit-tests.yaml | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/actions/cache-go-dependencies/action.yaml b/.github/actions/cache-go-dependencies/action.yaml index fd5fc6bc68b1d..c6cfa9e31778f 100644 --- a/.github/actions/cache-go-dependencies/action.yaml +++ b/.github/actions/cache-go-dependencies/action.yaml @@ -5,6 +5,10 @@ inputs: description: Whether this job saves caches on pushes to the default branch. required: false default: 'true' + key-suffix: + description: Extra suffix for cache keys (use to distinguish matrix entries that share github.job). + required: false + default: '' runs: using: composite steps: @@ -15,6 +19,8 @@ runs: echo "GOMODCACHE=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" echo "GOARCH=$(go env GOARCH)" >> "$GITHUB_OUTPUT" echo "TAG=${{ github.sha }}" >> "$GITHUB_OUTPUT" + suffix="${{ inputs.key-suffix }}" + echo "KEY_SUFFIX=${suffix:+-$suffix}" >> "$GITHUB_OUTPUT" shell: bash # Temporarily always save for testing cache effectiveness. @@ -29,8 +35,8 @@ runs: uses: actions/cache@v5 with: path: ${{ steps.cache-paths.outputs.GOCACHE }} - key: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.TAG }} - restore-keys: go-build-v1-${{ github.job }}-${{ steps.cache-paths.outputs.GOARCH }}- + key: go-build-v1-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.TAG }} + restore-keys: go-build-v1-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}- - name: Download Go modules run: make deps --always-make diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 770f30321edf1..a64bf5bfff738 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -42,6 +42,8 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - name: Go Unit Tests run: ${{ matrix.gotags }} make go-unit-tests @@ -122,6 +124,8 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - name: Is Postgres ready run: pg_isready -h 127.0.0.1 @@ -182,6 +186,8 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - name: Is Postgres ready run: pg_isready -h 127.0.0.1 @@ -303,6 +309,8 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - uses: ./.github/actions/handle-tagged-build @@ -403,6 +411,8 @@ jobs: - name: Cache Go dependencies uses: ./.github/actions/cache-go-dependencies + with: + key-suffix: ${{ matrix.gotags }} - name: Login to Quay.io uses: docker/login-action@v4 From 7930062bf4d7664eda5eaf24d7ecb69dbc2735f3 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Fri, 13 Mar 2026 00:12:36 -0600 Subject: [PATCH 09/10] empty: warm run with key-suffix fix From d9e0941d1ffb843ca0000e09210d45fc81820db5 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Fri, 13 Mar 2026 00:37:22 -0600 Subject: [PATCH 10/10] feat: add MinInt to mathutil (realistic change test) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simulates a typical PR — one small Go file changed. Expected: - pkg/mathutil recompiles + retests (~1 package) - ~700 other packages hit test cache - Total time should be close to warm run (~14m vs 45m cold) --- pkg/mathutil/mod.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/mathutil/mod.go b/pkg/mathutil/mod.go index 2456fae02e52e..c9c8736a17f7e 100644 --- a/pkg/mathutil/mod.go +++ b/pkg/mathutil/mod.go @@ -12,3 +12,11 @@ func Mod(a, b int) int { } return r } + +// MinInt returns the smaller of a and b. +func MinInt(a, b int) int { + if a < b { + return a + } + return b +}