diff --git a/.github/workflows/test-bundle-helpers.yaml b/.github/workflows/test-bundle-helpers.yaml new file mode 100644 index 0000000000000..43e608d403a66 --- /dev/null +++ b/.github/workflows/test-bundle-helpers.yaml @@ -0,0 +1,248 @@ +name: Test Bundle Helpers + +on: + push: + tags: + - '*' + branches: + - master + - release-* + pull_request: + types: + - opened + - reopened + - synchronize + +env: + ROX_PRODUCT_BRANDING: RHACS_BRANDING + RELATED_IMAGE_MAIN: foo + RELATED_IMAGE_SCANNER: foo + RELATED_IMAGE_SCANNER_SLIM: foo + RELATED_IMAGE_SCANNER_DB: foo + RELATED_IMAGE_SCANNER_DB_SLIM: foo + RELATED_IMAGE_COLLECTOR: foo + RELATED_IMAGE_ROXCTL: foo + RELATED_IMAGE_CENTRAL_DB: foo + RELATED_IMAGE_SCANNER_V4_DB: foo + RELATED_IMAGE_SCANNER_V4: foo + +jobs: + test-python-implementation: + strategy: + matrix: + related_img_mode: + - konflux + - omit + - downstream + name: Test Python Implementation + runs-on: ubuntu-latest + defaults: + run: + working-directory: operator + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r bundle_helpers/requirements.txt + + - name: Run Python unit tests + run: make test-bundle-helpers + + - name: Generate bundle with Python + env: + USE_GO_BUNDLE_HELPER: false + RELATED_IMAGES_MODE: ${{ matrix.related_img_mode }} + run: | + set -euo pipefail + go mod tidy + make bundle bundle-post-process + mkdir -p /tmp/artifacts + cp -r bundle /tmp/artifacts/bundle-python + if [ -d build/bundle ]; then + cp -r build/bundle /tmp/artifacts/build-bundle-python + fi + git diff || true + + - name: Upload Python bundle artifacts + uses: actions/upload-artifact@v4 + with: + name: bundle-python-${{ matrix.related_img_mode }} + path: /tmp/artifacts/ + retention-days: 1 + + test-go-implementation: + strategy: + fail-fast: false + matrix: + related_img_mode: + - konflux + - omit + - downstream + name: Test Go Implementation + runs-on: ubuntu-latest + defaults: + run: + working-directory: operator + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run Go unit tests + run: | + cd bundle_helpers + go test ./... + + # For python fallback + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + # For python fallback + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r bundle_helpers/requirements.txt + + - name: Generate bundle with Go + env: + USE_GO_BUNDLE_HELPER: true + RELATED_IMAGES_MODE: ${{ matrix.related_img_mode }} + run: | + set -euo pipefail + go mod tidy + make bundle bundle-post-process + mkdir -p /tmp/artifacts + cp -r bundle /tmp/artifacts/bundle-go + if [ -d build/bundle ]; then + cp -r build/bundle /tmp/artifacts/build-bundle-go + fi + git diff || true + + - name: Upload Go bundle artifacts + uses: actions/upload-artifact@v4 + with: + name: bundle-go-${{ matrix.related_img_mode }} + path: /tmp/artifacts/ + retention-days: 1 + + compare-implementations: + strategy: + fail-fast: false + matrix: + related_img_mode: + - konflux + - omit + - downstream + name: Compare Python and Go Outputs + runs-on: ubuntu-latest + needs: [test-python-implementation, test-go-implementation] + # Only run comparison if Go implementation exists + if: always() && needs.test-go-implementation.result == 'success' + + steps: + - name: Download Python bundle + uses: actions/download-artifact@v4 + with: + name: bundle-python-${{ matrix.related_img_mode }} + path: /tmp/python + + - name: Download Go bundle + uses: actions/download-artifact@v4 + with: + name: bundle-go-${{ matrix.related_img_mode }} + path: /tmp/go + continue-on-error: true + + - name: Install comparison tools + run: | + sudo apt-get update + sudo apt-get install -y diffutils + + - name: Compare bundle outputs + run: | + echo "=== Comparing Python and Go bundle outputs ===" + + if diff -ruN /tmp/python/bundle-python /tmp/go/bundle-go; then + echo "✓ SUCCESS: Bundle outputs are identical" + else + echo "✗ FAILURE: Bundle outputs differ" + exit 1 + fi + echo "" + echo "=== Pruning createdAt lines..." + sed -i '/^ createdAt:/d' /tmp/*/build-bundle-*/manifests/rhacs-operator.clusterserviceversion.yaml + + echo "" + echo "=== Comparing build/bundle outputs ===" + if diff -ruN /tmp/python/build-bundle-python /tmp/go/build-bundle-go; then + echo "✓ SUCCESS: Build bundle outputs are identical" + else + echo "✗ FAILURE: Build bundle outputs differ" + exit 1 + fi + echo "" + echo "=== Listing contents" + find /tmp/python /tmp/go -ls + find /tmp/python /tmp/go -type f -print0 | xargs -0 md5sum | sort + + status-check: + strategy: + matrix: + related_img_mode: + - konflux + - omit + - downstream + name: Bundle Helper Tests Status + runs-on: ubuntu-latest + needs: [test-python-implementation, test-go-implementation, compare-implementations] + if: always() + + steps: + - name: Check test results + run: | + python_result="${{ needs.test-python-implementation.result }}" + go_result="${{ needs.test-go-implementation.result }}" + compare_result="${{ needs.compare-implementations.result }}" + + echo "Python tests: $python_result" + echo "Go tests: $go_result" + echo "Comparison: $compare_result" + + if [ "$python_result" != "success" ]; then + echo "✗ Python tests failed" + exit 1 + fi + + if [ "$go_result" != "success" ]; then + echo "✗ Go tests failed" + exit 1 + fi + + if [ "$compare_result" != "success" ]; then + echo "✗ Go/Python comparison failed" + exit 1 + fi + + echo "✓ All required tests passed" diff --git a/operator/Makefile b/operator/Makefile index a25efe522b009..68e3e29f2a256 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -462,22 +462,24 @@ bundle: yq manifests kustomize operator-sdk ## Generate bundle manifests and met # Yet we want most of the contents autogenerated from the Makefile variables as a single source of truth. # Therefore we append ".extra" file to the end of bundle's dockerfile. cat bundle.Dockerfile.extra >> bundle.Dockerfile -# Run a python script to fix the orders in the specDescriptors (children must not appear before their parents). +# Fix the orders in the specDescriptors (children must not appear before their parents). set -euo pipefail ;\ $(ACTIVATE_PYTHON) ;\ - bundle_helpers/fix-spec-descriptor-order.py \ + ./bundle_helpers/dispatch.sh fix-spec-descriptor-order \ bundle/manifests/rhacs-operator.clusterserviceversion.yaml.fixed mv bundle/manifests/rhacs-operator.clusterserviceversion.yaml.fixed \ bundle/manifests/rhacs-operator.clusterserviceversion.yaml $(OPERATOR_SDK) bundle validate ./bundle --select-optional suite=operatorframework +RELATED_IMAGES_MODE ?= omit + .PHONY: bundle-post-process bundle-post-process: test-bundle-helpers operator-sdk ## Post-process CSV file to include correct operator versions, etc. set -euo pipefail ;\ $(ACTIVATE_PYTHON) ;\ first_version=3.62.0 `# 3.62.0 is the first operator version ever released` ;\ - candidate_version=$$(./bundle_helpers/patch-csv.py \ + candidate_version=$$(./bundle_helpers/dispatch.sh patch-csv \ --use-version $(VERSION) \ --first-version $${first_version} \ --operator-image $(IMG) \ @@ -495,7 +497,7 @@ bundle-post-process: test-bundle-helpers operator-sdk ## Post-process CSV file t --use-version=$(VERSION) \ --first-version=$${first_version} \ --operator-image=$(IMG) \ - --related-images-mode=omit \ + --related-images-mode=$(RELATED_IMAGES_MODE) \ $${unreleased_opt:-} # Check that the resulting bundle still passes validations. $(OPERATOR_SDK) bundle validate ./build/bundle --select-optional suite=operatorframework diff --git a/operator/bundle_helpers/cmd/fix_descriptors.go b/operator/bundle_helpers/cmd/fix_descriptors.go new file mode 100644 index 0000000000000..21db9b8a30df0 --- /dev/null +++ b/operator/bundle_helpers/cmd/fix_descriptors.go @@ -0,0 +1,77 @@ +package cmd + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "github.com/stackrox/rox/operator/bundle_helpers/pkg/descriptor" + "gopkg.in/yaml.v3" +) + +// FixSpecDescriptorOrder fixes the ordering of specDescriptors in a CSV file. +// It reads from stdin and writes to stdout, matching the Python script behavior. +func FixSpecDescriptorOrder(args []string) error { + if len(args) > 0 && (args[0] == "-h" || args[0] == "--help") { + fmt.Println("Usage: bundle-helper fix-spec-descriptor-order < input.yaml > output.yaml") + fmt.Println() + fmt.Println("Fixes the ordering of specDescriptors in a ClusterServiceVersion YAML file.") + fmt.Println("Ensures parent descriptors appear before their children.") + return nil + } + + // Read CSV from stdin + data, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read stdin: %w", err) + } + + // Parse YAML into a map (like Python's yaml.safe_load) + var csvDoc map[string]interface{} + if err := yaml.Unmarshal(data, &csvDoc); err != nil { + return fmt.Errorf("failed to parse YAML: %w", err) + } + + // Process descriptors + if err := descriptor.FixCSVDescriptorsMap(csvDoc); err != nil { + return fmt.Errorf("failed to fix descriptors: %w", err) + } + + // Encode to YAML using Go's yaml.v3 + var buf bytes.Buffer + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + if err := encoder.Encode(csvDoc); err != nil { + return fmt.Errorf("failed to encode YAML: %w", err) + } + if err := encoder.Close(); err != nil { + return fmt.Errorf("failed to close encoder: %w", err) + } + + // Normalize through Python to match PyYAML's exact formatting + // This is the "escape hatch" mentioned in the migration plan + return normalizeYAMLOutput(buf.Bytes(), os.Stdout) +} + +// normalizeYAMLOutput pipes YAML through the Python normalizer to match PyYAML formatting. +// This handles formatting quirks (quote styles, line wrapping, etc.) while keeping +// all business logic in Go. +func normalizeYAMLOutput(goYAML []byte, w io.Writer) error { + wd, _ := os.Getwd() + normalizerPath := filepath.Join(wd, "bundle_helpers", "yaml-normalizer.py") + + // Run the normalizer + cmd := exec.Command(normalizerPath) + cmd.Stdin = bytes.NewReader(goYAML) + cmd.Stdout = w + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to normalize YAML: %w", err) + } + + return nil +} diff --git a/operator/bundle_helpers/cmd/patch_csv.go b/operator/bundle_helpers/cmd/patch_csv.go new file mode 100644 index 0000000000000..ce6139c6a909d --- /dev/null +++ b/operator/bundle_helpers/cmd/patch_csv.go @@ -0,0 +1,7 @@ +package cmd + +// PatchCSV patches a ClusterServiceVersion YAML file with version and image information. +// This is a placeholder implementation - to be implemented in future phases. +func PatchCSV(args []string) error { + panic("not yet implemented") +} diff --git a/operator/bundle_helpers/dispatch.sh b/operator/bundle_helpers/dispatch.sh new file mode 100755 index 0000000000000..83ac70d66b090 --- /dev/null +++ b/operator/bundle_helpers/dispatch.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Wrapper script for bundle helper tools. +# +# Provides an abstraction layer that allows switching between Python and Go +# implementations of bundle helper scripts without changing Makefile or Dockerfiles. +# The implementation is selected via the USE_GO_BUNDLE_HELPER environment variable. +# +# Usage: dispatch.sh [args...] + +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [args...]" >&2 + exit 1 +fi + +script_name="$1" +shift + +script_dir="$(dirname "$0")" + +if [[ "${USE_GO_BUNDLE_HELPER:-false}" == "true" ]]; then + case "$script_name" in + fix-spec-descriptor-order) + exec go run "${script_dir}/main.go" "$script_name" "$@" + ;; + *) + echo >&2 "No Go implementation of $script_name yet, falling back to Python one!" + exec "${script_dir}/${script_name}.py" "$@" + ;; + esac +else + exec "${script_dir}/${script_name}.py" "$@" +fi diff --git a/operator/bundle_helpers/main.go b/operator/bundle_helpers/main.go new file mode 100644 index 0000000000000..c7570a2123409 --- /dev/null +++ b/operator/bundle_helpers/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "os" + + "github.com/stackrox/rox/operator/bundle_helpers/cmd" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "Usage: %s [args...]\n", os.Args[0]) + fmt.Fprint(os.Stderr, "Available commands:\n") + fmt.Fprint(os.Stderr, " fix-spec-descriptor-order Fix specDescriptor ordering\n") + fmt.Fprint(os.Stderr, " patch-csv Patch ClusterServiceVersion file (not yet implemented)\n") + os.Exit(1) + } + + command := os.Args[1] + args := os.Args[2:] + + switch command { + case "fix-spec-descriptor-order": + if err := cmd.FixSpecDescriptorOrder(args); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + case "patch-csv": + fmt.Fprint(os.Stderr, "patch-csv command not yet implemented\n") + os.Exit(1) + default: + fmt.Fprintf(os.Stderr, "Unknown command: %s\n", command) + os.Exit(1) + } +} diff --git a/operator/bundle_helpers/pkg/csv/patcher.go b/operator/bundle_helpers/pkg/csv/patcher.go new file mode 100644 index 0000000000000..59855dc53ee8a --- /dev/null +++ b/operator/bundle_helpers/pkg/csv/patcher.go @@ -0,0 +1,3 @@ +package csv + +// Placeholder for CSV patching logic diff --git a/operator/bundle_helpers/pkg/csv/related_images.go b/operator/bundle_helpers/pkg/csv/related_images.go new file mode 100644 index 0000000000000..8806b3533e5e1 --- /dev/null +++ b/operator/bundle_helpers/pkg/csv/related_images.go @@ -0,0 +1,3 @@ +package csv + +// Placeholder for related images handling logic diff --git a/operator/bundle_helpers/pkg/csv/replaces.go b/operator/bundle_helpers/pkg/csv/replaces.go new file mode 100644 index 0000000000000..c48d358a31ad0 --- /dev/null +++ b/operator/bundle_helpers/pkg/csv/replaces.go @@ -0,0 +1,3 @@ +package csv + +// Placeholder for replace calculation logic diff --git a/operator/bundle_helpers/pkg/csv/version.go b/operator/bundle_helpers/pkg/csv/version.go new file mode 100644 index 0000000000000..e2c3c1f0fdc2a --- /dev/null +++ b/operator/bundle_helpers/pkg/csv/version.go @@ -0,0 +1,3 @@ +package csv + +// Placeholder for version handling logic diff --git a/operator/bundle_helpers/pkg/descriptor/sorter.go b/operator/bundle_helpers/pkg/descriptor/sorter.go new file mode 100644 index 0000000000000..e7d76d4141fe4 --- /dev/null +++ b/operator/bundle_helpers/pkg/descriptor/sorter.go @@ -0,0 +1,164 @@ +package descriptor + +import ( + "sort" + "strings" + + "github.com/pkg/errors" +) + +// FixCSVDescriptorsMap processes all CRDs in a CSV and fixes their specDescriptors. +// This function works with map[string]interface{} to match Python's behavior. +func FixCSVDescriptorsMap(csvDoc map[string]interface{}) error { + // Navigate to spec.customresourcedefinitions.owned + spec, ok := csvDoc["spec"].(map[string]interface{}) + if !ok { + return errors.New("spec not found or not a map") + } + + crds, ok := spec["customresourcedefinitions"].(map[string]interface{}) + if !ok { + return errors.New("customresourcedefinitions not found or not a map") + } + + owned, ok := crds["owned"].([]interface{}) + if !ok { + return errors.New("owned not found or not a list") + } + + // Process each CRD + for _, crdItem := range owned { + crd, ok := crdItem.(map[string]interface{}) + if !ok { + continue + } + + if err := processSpecDescriptorsMap(crd); err != nil { + return err + } + } + + return nil +} + +// processSpecDescriptorsMap processes specDescriptors for a single CRD. +func processSpecDescriptorsMap(crd map[string]interface{}) error { + descs, ok := crd["specDescriptors"] + if !ok { + // No specDescriptors, that's OK + return nil + } + + descriptors, ok := descs.([]interface{}) + if !ok { + return errors.New("specDescriptors is not a list") + } + + // Fix descriptor order + fixDescriptorOrderMap(descriptors) + + // Allow relative field dependencies + allowRelativeFieldDependenciesMap(descriptors) + + return nil +} + +// fixDescriptorOrderMap performs a stable sort based on the parent path. +// This ensures children always come after their parents. +// Mimics Python: descriptors.sort(key=lambda d: f'.{d["path"]}'.rsplit('.', 1)[0]) +func fixDescriptorOrderMap(descriptors []interface{}) { + sort.SliceStable(descriptors, func(i, j int) bool { + pathI := getDescriptorPathMap(descriptors[i]) + pathJ := getDescriptorPathMap(descriptors[j]) + parentI := getParentPath(pathI) + parentJ := getParentPath(pathJ) + return parentI < parentJ + }) +} + +// getParentPath extracts the parent path from a descriptor path. +// Mimics Python: f'.{d["path"]}'.rsplit('.', 1)[0] +func getParentPath(path string) string { + // Add a '.' in front for simplicity + fullPath := "." + path + // Split by last '.' and take the first part + lastDot := strings.LastIndex(fullPath, ".") + if lastDot == -1 { + return "" + } + return fullPath[:lastDot] +} + +// getDescriptorPathMap extracts the 'path' field from a descriptor map. +func getDescriptorPathMap(desc interface{}) string { + descMap, ok := desc.(map[string]interface{}) + if !ok { + return "" + } + + path, ok := descMap["path"].(string) + if !ok { + return "" + } + + return path +} + +// allowRelativeFieldDependenciesMap converts relative field dependency paths to absolute. +func allowRelativeFieldDependenciesMap(descriptors []interface{}) { + for _, desc := range descriptors { + descMap, ok := desc.(map[string]interface{}) + if !ok { + continue + } + + path, _ := descMap["path"].(string) + xDescsRaw, ok := descMap["x-descriptors"] + if !ok { + continue + } + + xDescs, ok := xDescsRaw.([]interface{}) + if !ok { + continue + } + + // Process each x-descriptor + for i, xDescRaw := range xDescs { + xDesc, ok := xDescRaw.(string) + if !ok { + continue + } + + if !strings.HasPrefix(xDesc, "urn:alm:descriptor:com.tectonic.ui:fieldDependency:") { + continue + } + + // Split by ':' and get the last two parts (field and value) + parts := strings.Split(xDesc, ":") + if len(parts) < 2 { + continue + } + + field := parts[len(parts)-2] + val := parts[len(parts)-1] + + // Check if field starts with '.' (relative path) + if !strings.HasPrefix(field, ".") { + continue + } + + // Convert relative to absolute + // Mimics Python: f'.{d["path"]}'.rsplit('.', 1)[0][1:] + field + parentPath := getParentPath(path) + if len(parentPath) > 0 { + parentPath = parentPath[1:] // Remove leading '.' + } + absoluteField := parentPath + field + + // Reconstruct the x-descriptor + prefix := "urn:alm:descriptor:com.tectonic.ui:fieldDependency:" + xDescs[i] = prefix + absoluteField + ":" + val + } + } +} diff --git a/operator/bundle_helpers/pkg/rewrite/rewriter.go b/operator/bundle_helpers/pkg/rewrite/rewriter.go new file mode 100644 index 0000000000000..0b5684a4b8671 --- /dev/null +++ b/operator/bundle_helpers/pkg/rewrite/rewriter.go @@ -0,0 +1,3 @@ +package rewrite + +// Placeholder for generic dictionary rewriter diff --git a/operator/bundle_helpers/pkg/yamlformat/pyaml_compat.go b/operator/bundle_helpers/pkg/yamlformat/pyaml_compat.go new file mode 100644 index 0000000000000..ca2e9918ab0c8 --- /dev/null +++ b/operator/bundle_helpers/pkg/yamlformat/pyaml_compat.go @@ -0,0 +1,41 @@ +package yamlformat + +import ( + "bytes" + "io" + + "gopkg.in/yaml.v3" +) + +// EncodePyYAMLStyle encodes data to YAML matching PyYAML's safe_dump() style. +// This includes: +// - Single quotes for simple string values +// - Empty strings using single quotes +// - Flow style for arrays/maps where appropriate +func EncodePyYAMLStyle(w io.Writer, data interface{}) error { + var buf bytes.Buffer + + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + + if err := encoder.Encode(data); err != nil { + return err + } + if err := encoder.Close(); err != nil { + return err + } + + // Post-process to match PyYAML style + output := buf.Bytes() + output = normalizeToPyYAMLStyle(output) + + _, err := w.Write(output) + return err +} + +// normalizeToPyYAMLStyle converts Go yaml.v3 output to match PyYAML style. +func normalizeToPyYAMLStyle(input []byte) []byte { + // For now, just return as-is + // We'll implement specific transformations if needed + return input +} diff --git a/operator/bundle_helpers/prepare-bundle-manifests.sh b/operator/bundle_helpers/prepare-bundle-manifests.sh index d08bfc9f8a61a..b363fac844030 100755 --- a/operator/bundle_helpers/prepare-bundle-manifests.sh +++ b/operator/bundle_helpers/prepare-bundle-manifests.sh @@ -9,6 +9,6 @@ mkdir -p build/ rm -rf build/bundle cp -a bundle build/ -"$(dirname "$0")/patch-csv.py" "$@" \ +"$(dirname "$0")/dispatch.sh" patch-csv "$@" \ < bundle/manifests/rhacs-operator.clusterserviceversion.yaml \ > build/bundle/manifests/rhacs-operator.clusterserviceversion.yaml diff --git a/operator/bundle_helpers/yaml-normalizer.py b/operator/bundle_helpers/yaml-normalizer.py new file mode 100755 index 0000000000000..7d8cb21a14297 --- /dev/null +++ b/operator/bundle_helpers/yaml-normalizer.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +""" +Thin YAML reformatter that normalizes Go-generated YAML to match PyYAML output. +This script has NO knowledge of CSV/bundle structure - it only normalizes formatting. + +This is the "escape hatch" mentioned in the migration plan (Section 2.2). +All business logic remains in Go; this only handles YAML formatting quirks. +""" +import sys +import yaml + +# Read YAML from stdin +doc = yaml.safe_load(sys.stdin) + +# Write YAML to stdout with PyYAML's formatting +# Note: yaml.safe_dump() adds a trailing newline, and print() adds another +print(yaml.safe_dump(doc))