Skip to content
Closed
51 changes: 51 additions & 0 deletions tests/e2e/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,17 @@ deploy_central() {
DEPLOY_DIR="deploy/${ORCHESTRATOR_FLAVOR}"
CENTRAL_NAMESPACE="${central_namespace}" "${ROOT}/${DEPLOY_DIR}/central.sh"
fi

if [[ -n "${IS_RACE_BUILD:-}" && "${DEPLOY_STACKROX_VIA_OPERATOR}" != "true" ]]; then
# The race detector's ~5-10x memory multiplier causes OOMKills for
# components with tight memory limits. For operator deployments, limits
# are set in the CR template; for Helm/roxctl, patch after deploy.
info "Race build detected: increasing memory limits for central-namespace components"
retrying_kubectl </dev/null -n "${central_namespace}" set resources deploy/config-controller -c manager --limits 'memory=512Mi'
retrying_kubectl </dev/null -n "${central_namespace}" set resources deploy/scanner-v4-indexer -c indexer --limits 'memory=6Gi' 2>/dev/null || true
retrying_kubectl </dev/null -n "${central_namespace}" set resources deploy/scanner-v4-matcher -c matcher --limits 'memory=6Gi' 2>/dev/null || true
retrying_kubectl </dev/null -n "${central_namespace}" set resources deploy/scanner-v4-db -c db --limits 'memory=4Gi' 2>/dev/null || true
fi
}

# shellcheck disable=SC2120
Expand Down Expand Up @@ -365,6 +376,19 @@ deploy_central_via_operator() {
false) scannerV4ScannerComponent="Disabled" ;;
esac

# Resource limits — increased for race-detector builds to avoid OOMKills.
local configControllerMemoryLimit="128Mi"
local scannerV4IndexerMemoryLimit="2Gi"
local scannerV4MatcherMemoryLimit="2000Mi"
local scannerV4DbMemoryLimit="1000Mi"
if [[ -n "${IS_RACE_BUILD:-}" ]]; then
info "Race build detected: using increased memory limits for Central CR"
configControllerMemoryLimit="512Mi"
scannerV4IndexerMemoryLimit="6Gi"
scannerV4MatcherMemoryLimit="6Gi"
scannerV4DbMemoryLimit="4Gi"
fi

CENTRAL_YAML_PATH="tests/e2e/yaml/central-cr.envsubst.yaml"
# Different yaml for midstream images
if [[ "${USE_MIDSTREAM_IMAGES}" == "true" ]]; then
Expand All @@ -380,6 +404,10 @@ deploy_central_via_operator() {
central_exposure_route_enabled="$central_exposure_route_enabled" \
customize_envVars="$customize_envVars" \
scannerV4ScannerComponent="$scannerV4ScannerComponent" \
configControllerMemoryLimit="$configControllerMemoryLimit" \
scannerV4IndexerMemoryLimit="$scannerV4IndexerMemoryLimit" \
scannerV4MatcherMemoryLimit="$scannerV4MatcherMemoryLimit" \
scannerV4DbMemoryLimit="$scannerV4DbMemoryLimit" \
"${envsubst}" \
< "${CENTRAL_YAML_PATH}" | retrying_kubectl apply -n "${central_namespace}" -f -

Expand Down Expand Up @@ -425,6 +453,15 @@ deploy_sensor() {
# et al.
retrying_kubectl </dev/null -n "${sensor_namespace}" set resources deploy/sensor -c sensor --requests 'cpu=2' --limits 'cpu=4'
fi

if [[ -n "${IS_RACE_BUILD:-}" && "${DEPLOY_STACKROX_VIA_OPERATOR}" != "true" ]]; then
# For operator deployments, limits are set in the CR template;
# for Helm/roxctl, patch after deploy.
info "Race build detected: increasing memory limits for sensor-namespace components"
retrying_kubectl </dev/null -n "${sensor_namespace}" set resources deploy/admission-control -c admission-control --limits 'memory=2Gi'
retrying_kubectl </dev/null -n "${sensor_namespace}" set resources deploy/scanner-v4-indexer -c indexer --limits 'memory=6Gi' 2>/dev/null || true
retrying_kubectl </dev/null -n "${sensor_namespace}" set resources deploy/scanner-v4-db -c db --limits 'memory=4Gi' 2>/dev/null || true
fi
}

# shellcheck disable=SC2120
Expand Down Expand Up @@ -475,11 +512,25 @@ deploy_sensor_via_operator() {
customize_envVars+=$'\n value: "'"${ROX_NETFLOW_CACHE_LIMITING}"'"'
fi

# Resource limits — increased for race-detector builds to avoid OOMKills.
local admissionControlMemoryLimit="500Mi"
local scannerV4IndexerMemoryLimit="2Gi"
local scannerV4DbMemoryLimit="2500Mi"
if [[ -n "${IS_RACE_BUILD:-}" ]]; then
info "Race build detected: using increased memory limits for SecuredCluster CR"
admissionControlMemoryLimit="2Gi"
scannerV4IndexerMemoryLimit="6Gi"
scannerV4DbMemoryLimit="4Gi"
fi

env - \
scanner_component_setting="$scanner_component_setting" \
fam_mode_setting="$fam_mode_setting" \
central_endpoint="$central_endpoint" \
customize_envVars="$customize_envVars" \
admissionControlMemoryLimit="$admissionControlMemoryLimit" \
scannerV4IndexerMemoryLimit="$scannerV4IndexerMemoryLimit" \
scannerV4DbMemoryLimit="$scannerV4DbMemoryLimit" \
"${envsubst}" \
< "${secured_cluster_yaml_path}" | retrying_kubectl apply -n "${sensor_namespace}" --validate="${validate}" -f -

Expand Down
29 changes: 27 additions & 2 deletions tests/e2e/run-scanner-v4-install.bats
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,24 @@ EOT
sleep 60
"${ORCH_CMD}" </dev/null -n rhacs-operator-system wait --for=condition=Ready --timeout=3m pods -l app=rhacs-operator

if [[ -n "${IS_RACE_BUILD:-}" ]]; then
# After upgrading from the old operator, patch the CRs with increased
# memory limits for race-detector builds. The old operator doesn't
# support all resource fields, so we must patch after the upgrade.
info "Race build detected: patching CRs with increased memory limits after operator upgrade"
"${ORCH_CMD}" </dev/null -n "${CUSTOM_CENTRAL_NAMESPACE}" patch central stackrox-central-services --type=merge \
-p '{"spec":{"configAsCode":{"resources":{"limits":{"memory":"512Mi"}}},"scannerV4":{"indexer":{"resources":{"limits":{"memory":"6Gi"}}},"matcher":{"resources":{"limits":{"memory":"6Gi"}}},"db":{"resources":{"limits":{"memory":"4Gi"}}}}}}'
"${ORCH_CMD}" </dev/null -n "${CUSTOM_SENSOR_NAMESPACE}" patch securedcluster stackrox-secured-cluster-services --type=merge \
-p '{"spec":{"admissionControl":{"resources":{"limits":{"memory":"2Gi"}}},"scannerV4":{"indexer":{"resources":{"limits":{"memory":"6Gi"}}},"db":{"resources":{"limits":{"memory":"4Gi"}}}}}}'
# config-controller is created by the new operator with default 128Mi.
# The configAsCode CR field may not control its deployment resources,
# so patch the deployment directly as a fallback.
info "Race build detected: patching config-controller deployment directly"
retrying_kubectl </dev/null -n "${CUSTOM_CENTRAL_NAMESPACE}" set resources deploy/config-controller -c manager --limits 'memory=512Mi' 2>/dev/null || true
# Give the operator time to reconcile the new resource limits.
sleep 30
fi

_begin "verify"

verify_scannerV2_deployed "${CUSTOM_CENTRAL_NAMESPACE}"
Expand Down Expand Up @@ -912,8 +930,15 @@ EOT

_begin "verify"

verify_deployment_deletion_with_timeout 4m "${CUSTOM_CENTRAL_NAMESPACE}" scanner-v4-indexer scanner-v4-matcher scanner-v4-db
verify_deployment_deletion_with_timeout 4m "${CUSTOM_SENSOR_NAMESPACE}" scanner-v4-indexer scanner-v4-db
# Race-detector builds OOMKill the config-controller (operator), slowing
# reconciliation. The operator reconciles config-controller back to 128Mi
# so we can't increase its limit — just give it more time.
local deletion_timeout="4m"
if [[ -n "${IS_RACE_BUILD:-}" ]]; then
deletion_timeout="10m"
fi
verify_deployment_deletion_with_timeout "$deletion_timeout" "${CUSTOM_CENTRAL_NAMESPACE}" scanner-v4-indexer scanner-v4-matcher scanner-v4-db
verify_deployment_deletion_with_timeout "$deletion_timeout" "${CUSTOM_SENSOR_NAMESPACE}" scanner-v4-indexer scanner-v4-db
! verify_deployment_scannerV4_env_var_set "${CUSTOM_CENTRAL_NAMESPACE}" "central"
! verify_deployment_scannerV4_env_var_set "${CUSTOM_SENSOR_NAMESPACE}" "sensor"

Expand Down
16 changes: 16 additions & 0 deletions tests/e2e/yaml/central-cr.envsubst.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ $centralAdditionalCAIndented
scaling:
autoScaling: Disabled
replicas: 1
configAsCode:
resources:
limits:
memory: "$configControllerMemoryLimit"
scannerV4:
scannerComponent: "$scannerV4ScannerComponent"
indexer:
Expand All @@ -48,17 +52,29 @@ $centralAdditionalCAIndented
resources:
requests:
cpu: "400m"
memory: "1Gi"
limits:
cpu: "1000m"
memory: "$scannerV4IndexerMemoryLimit"
matcher:
scaling:
autoScaling: Disabled
replicas: 1
resources:
requests:
cpu: "400m"
memory: "2000Mi"
limits:
cpu: "6000m"
memory: "$scannerV4MatcherMemoryLimit"
db:
resources:
requests:
cpu: "500m"
memory: "500Mi"
limits:
cpu: "1000m"
memory: "$scannerV4DbMemoryLimit"
---
apiVersion: v1
kind: Secret
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ spec:
requests:
memory: "100Mi"
cpu: "100m"
limits:
memory: "$admissionControlMemoryLimit"
sensor:
resources:
requests:
Expand Down Expand Up @@ -65,14 +67,14 @@ spec:
memory: "1500Mi"
limits:
cpu: "2000m"
memory: "2Gi"
memory: "$scannerV4IndexerMemoryLimit"
db:
resources:
requests:
cpu: "200m"
memory: "2Gi"
limits:
cpu: "1000m"
memory: "2500Mi"
memory: "$scannerV4DbMemoryLimit"
customize:
envVars:$customize_envVars
Loading