stackrox · davdhacs · Dec 16, 2022 · Dec 15, 2022 · Dec 16, 2022 · Dec 16, 2022
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,21 @@ Entries in this file should be limited to:
 -  Obscure side-effects that are not obviously apparent based on the JIRA associated with the changes.
 Please avoid adding duplicate information across this changelog and JIRA/doc input pages.
 
+## [3.73.1]
+
+### Added Features
+
+### Removed Features
+
+### Deprecated Features
+
+### Technical Changes
+3.73.0 introduced a change to ACS autogenerated image integration workflows.
+However, this change in workflow caused Central to take too long on startup (details [here](https://access.redhat.com/node/6990153)).
+To fix the issue introduced in 3.73.0, 3.73.1 will reinstate the old workflow.
+Therefore, autogenerated integrations may not work successfully in environments with various credentials
+used for multiple repos within a global registry.
+
 ## [3.73.0]
 ### Removed Features
 - ROX-12839: we will stop shipping the docs embedded in the product, starting with the release following this one (docs will still be available online)

diff --git a/central/enrichment/singleton.go b/central/enrichment/singleton.go
@@ -17,6 +17,7 @@ import (
 	v1 "github.com/stackrox/rox/generated/api/v1"
 	"github.com/stackrox/rox/pkg/env"
 	"github.com/stackrox/rox/pkg/expiringcache"
+	"github.com/stackrox/rox/pkg/features"
 	imageEnricher "github.com/stackrox/rox/pkg/images/enricher"
 	"github.com/stackrox/rox/pkg/metrics"
 	nodeEnricher "github.com/stackrox/rox/pkg/nodes/enricher"
@@ -70,6 +71,10 @@ func initializeManager() {
 		return
 	}
 	for _, ii := range integrations {
+		// Only upsert autogenerated integrations with a source if the feature is enabled.
+		if !features.SourcedAutogeneratedIntegrations.Enabled() && ii.GetAutogenerated() && ii.GetSource() != nil {
+			continue
+		}
 		if err := manager.Upsert(ii); err != nil {
 			log.Errorf("unable to use previous integration %s: %v", ii.GetName(), err)
 		}

diff --git a/central/image/service/service_impl.go b/central/image/service/service_impl.go
@@ -21,6 +21,7 @@ import (
 	"github.com/stackrox/rox/pkg/env"
 	"github.com/stackrox/rox/pkg/errox"
 	"github.com/stackrox/rox/pkg/expiringcache"
+	"github.com/stackrox/rox/pkg/features"
 	"github.com/stackrox/rox/pkg/grpc/authz"
 	"github.com/stackrox/rox/pkg/grpc/authz/idcheck"
 	"github.com/stackrox/rox/pkg/grpc/authz/or"
@@ -232,19 +233,18 @@ func (s *serviceImpl) ScanImageInternal(ctx context.Context, request *v1.ScanIma
 
 	img := types.ToImage(request.GetImage())
 
-	var source *enricher.RequestSource
-	if request.GetSource() != nil {
-		source = &enricher.RequestSource{
-			ClusterID:        request.GetSource().GetClusterId(),
-			Namespace:        request.GetSource().GetNamespace(),
-			ImagePullSecrets: set.NewStringSet(request.GetSource().GetImagePullSecrets()...),
-		}
-	}
-
 	enrichmentContext := enricher.EnrichmentContext{
 		FetchOpt: fetchOpt,
 		Internal: true,
-		Source:   source,
+	}
+
+	requestSource := request.GetSource()
+	if features.SourcedAutogeneratedIntegrations.Enabled() && requestSource != nil {
+		enrichmentContext.Source = &enricher.RequestSource{
+			ClusterID:        requestSource.GetClusterId(),
+			Namespace:        requestSource.GetNamespace(),
+			ImagePullSecrets: set.NewStringSet(requestSource.GetImagePullSecrets()...),
+		}
 	}
 
 	if _, err := s.enricher.EnrichImage(ctx, enrichmentContext, img); err != nil {

diff --git a/central/imageintegration/datastore/datastore_impl.go b/central/imageintegration/datastore/datastore_impl.go
@@ -10,6 +10,7 @@ import (
 	v1 "github.com/stackrox/rox/generated/api/v1"
 	"github.com/stackrox/rox/generated/storage"
 	"github.com/stackrox/rox/pkg/env"
+	"github.com/stackrox/rox/pkg/features"
 	"github.com/stackrox/rox/pkg/sac"
 	searchPkg "github.com/stackrox/rox/pkg/search"
 	"github.com/stackrox/rox/pkg/uuid"
@@ -78,7 +79,7 @@ func (ds *datastoreImpl) AddImageIntegration(ctx context.Context, integration *s
 		return "", sac.ErrResourceAccessDenied
 	}
 
-	if integration.GetId() == "" {
+	if !features.SourcedAutogeneratedIntegrations.Enabled() || integration.GetId() == "" {
 		integration.Id = uuid.NewV4().String()
 	}
 	err := ds.storage.Upsert(ctx, integration)

diff --git a/central/imageintegration/datastore/singleton.go b/central/imageintegration/datastore/singleton.go
@@ -11,6 +11,7 @@ import (
 	"github.com/stackrox/rox/central/imageintegration/store/bolt"
 	"github.com/stackrox/rox/central/imageintegration/store/postgres"
 	"github.com/stackrox/rox/pkg/env"
+	"github.com/stackrox/rox/pkg/features"
 	"github.com/stackrox/rox/pkg/sac"
 	"github.com/stackrox/rox/pkg/sync"
 	"github.com/stackrox/rox/pkg/utils"
@@ -22,16 +23,40 @@ var (
 	dataStore DataStore
 )
 
-func initializeDefaultIntegrations(storage store.Store) {
+func initializeIntegrations(storage store.Store) {
 	ctx := sac.WithGlobalAccessScopeChecker(context.Background(), sac.AllowAllAccessScopeChecker())
 	iis, err := storage.GetAll(ctx)
 	utils.CrashOnError(err)
+	// If we are starting from scratch in online-mode, add the default image integrations.
 	if !env.OfflineModeEnv.BooleanSetting() && len(iis) == 0 {
 		// Add default integrations
 		for _, ii := range store.DefaultImageIntegrations {
 			utils.Must(storage.Upsert(ctx, ii))
 		}
 	}
+
+	// If the feature flag is disabled, remove all "sourced" autogenerated registries from the datastore.
+	if !features.SourcedAutogeneratedIntegrations.Enabled() {
+		if len(iis) > 0 {
+			log.Infof("[STARTUP] Starting deletion of 'sourced' image integrations")
+		}
+
+		var attempted, deleted int
+		for _, ii := range iis {
+			if ii.GetAutogenerated() && ii.GetSource() != nil {
+				attempted++
+				// Use Should so release versions do not panic.
+				if err := utils.Should(storage.Delete(ctx, ii.GetId())); err != nil {
+					deleted++
+				}
+			}
+		}
+		if attempted > 0 {
+			log.Infof("Successfully deleted %d out of %d image integration(s)", deleted, attempted)
+		}
+
+		log.Info("Completed deletion of 'sourced' image integrations")
+	}
 }
 
 func initialize() {
@@ -46,7 +71,7 @@ func initialize() {
 		storage = bolt.New(globaldb.GetGlobalDB())
 		indexer = index.New(globalindex.GetGlobalTmpIndex())
 	}
-	initializeDefaultIntegrations(storage)
+	initializeIntegrations(storage)
 	searcher := search.New(storage, indexer)
 	dataStore = New(storage, indexer, searcher)
 }

diff --git a/central/sensor/service/pipeline/imageintegrations/pipeline.go b/central/sensor/service/pipeline/imageintegrations/pipeline.go
@@ -21,6 +21,7 @@ import (
 	"github.com/stackrox/rox/pkg/centralsensor"
 	"github.com/stackrox/rox/pkg/env"
 	"github.com/stackrox/rox/pkg/errox"
+	"github.com/stackrox/rox/pkg/features"
 	"github.com/stackrox/rox/pkg/logging"
 	"github.com/stackrox/rox/pkg/metrics"
 	"github.com/stackrox/rox/pkg/set"
@@ -65,6 +66,10 @@ type pipelineImpl struct {
 }
 
 func (s *pipelineImpl) Reconcile(ctx context.Context, clusterID string, storeMap *reconciliation.StoreMap) error {
+	if !features.SourcedAutogeneratedIntegrations.Enabled() {
+		return nil
+	}
+
 	existingIDs := set.NewStringSet()
 
 	conn := connection.FromContext(ctx)
@@ -248,7 +253,8 @@ func (s *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M
 	}
 	defer countMetrics.IncrementResourceProcessedCounter(pipeline.ActionToOperation(msg.GetEvent().GetAction()), metrics.ImageIntegration)
 
-	if msg.GetEvent().GetAction() == central.ResourceAction_REMOVE_RESOURCE {
+	if features.SourcedAutogeneratedIntegrations.Enabled() && msg.GetEvent().GetAction() == central.ResourceAction_REMOVE_RESOURCE {
+		// Remove is only supported if the feature flag is enabled.
 		return s.runRemove(ctx, msg.GetEvent().GetImageIntegration().GetId())
 	}
 
@@ -270,7 +276,8 @@ func (s *pipelineImpl) Run(ctx context.Context, clusterID string, msg *central.M
 		return errors.Wrapf(err, "setting up integration params for %q", imageIntegration.GetId())
 	}
 	source := imageIntegration.GetSource()
-	if source == nil {
+	if !features.SourcedAutogeneratedIntegrations.Enabled() || source == nil {
+		imageIntegration.Name = fmt.Sprintf("Autogenerated %s for cluster %s", description, clusterName)
 		return s.legacyRun(ctx, imageIntegration, matches)
 	}
 	imageIntegration.Name = fmt.Sprintf("Autogenerated %s for cluster %s from %s/%s",

diff --git a/pkg/features/list.go b/pkg/features/list.go
@@ -51,4 +51,7 @@ var (
 
 	// ResyncDisabled disables the resync behavior of the kubernetes listeners in sensor
 	ResyncDisabled = registerFeature("Disable the re-sync", "ROX_RESYNC_DISABLED", false)
+
+	// SourcedAutogeneratedIntegrations enables adding a "source" to autogenerated integrations.
+	SourcedAutogeneratedIntegrations = registerFeature("Enable autogenerated integrations with cluster/namespace/secret source", "ROX_SOURCED_AUTOGENERATED_INTEGRATIONS", false)
 )
diff --git a/pkg/images/enricher/enricher_impl.go b/pkg/images/enricher/enricher_impl.go
@@ -14,6 +14,7 @@ import (
 	"github.com/stackrox/rox/pkg/errorhelpers"
 	"github.com/stackrox/rox/pkg/errox"
 	"github.com/stackrox/rox/pkg/expiringcache"
+	"github.com/stackrox/rox/pkg/features"
 	"github.com/stackrox/rox/pkg/images/integration"
 	"github.com/stackrox/rox/pkg/images/utils"
 	"github.com/stackrox/rox/pkg/integrationhealth"
@@ -296,9 +297,13 @@ func (e *enricherImpl) enrichWithMetadata(ctx context.Context, enrichmentContext
 					e.errorsPerRegistry[registry] = 0
 				})
 			}
+			id, name := registry.DataSource().GetId(), registry.DataSource().GetName()
+			if features.SourcedAutogeneratedIntegrations.Enabled() {
+				id, name = registry.Source().GetId(), registry.Source().GetName()
+			}
 			e.integrationHealthReporter.UpdateIntegrationHealthAsync(&storage.IntegrationHealth{
-				Id:            registry.Source().GetId(),
-				Name:          registry.Source().GetName(),
+				Id:            id,
+				Name:          name,
 				Type:          storage.IntegrationHealth_IMAGE_INTEGRATION,
 				Status:        storage.IntegrationHealth_HEALTHY,
 				LastTimestamp: timestamp.TimestampNow(),
@@ -343,7 +348,10 @@ func (e *enricherImpl) enrichImageWithRegistry(ctx context.Context, image *stora
 	if err != nil {
 		return false, errors.Wrapf(err, "getting metadata from registry: %q", registry.Name())
 	}
-	metadata.DataSource = imageIntegrationToDataSource(registry.Source())
+	metadata.DataSource = registry.DataSource()
+	if features.SourcedAutogeneratedIntegrations.Enabled() {
+		metadata.DataSource = imageIntegrationToDataSource(registry.Source())
+	}
 	metadata.Version = metadataVersion
 	image.Metadata = metadata
 
@@ -666,6 +674,9 @@ func isOpenshiftGlobalPullSecret(source *storage.ImageIntegration_Source) bool {
 func (e *enricherImpl) getRegistriesForContext(ctx EnrichmentContext) ([]registryTypes.ImageRegistry, error) {
 	registries := e.integrations.RegistrySet().GetAll()
 	if ctx.Internal {
+		if !features.SourcedAutogeneratedIntegrations.Enabled() {
+			return registries, nil
+		}
 		if ctx.Source == nil {
 			return registries, nil
 		}
@@ -684,6 +695,10 @@ func (e *enricherImpl) getRegistriesForContext(ctx EnrichmentContext) ([]registr
 // 2. If the integration's source matches with the EnrichmentContext.Source
 // Note that this function WILL modify the input array.
 func filterRegistriesBySource(requestSource *RequestSource, registries []registryTypes.ImageRegistry) {
+	if !features.SourcedAutogeneratedIntegrations.Enabled() {
+		return
+	}
+
 	filteredRegistries := registries[:0]
 	for _, registry := range registries {
 		integration := registry.Source()

diff --git a/pkg/registries/factory_impl.go b/pkg/registries/factory_impl.go
@@ -13,7 +13,12 @@ type factoryImpl struct {
 
 type registryWithDataSource struct {
 	types.Registry
-	source *storage.ImageIntegration
+	datasource *storage.DataSource
+	source     *storage.ImageIntegration
+}
+
+func (r *registryWithDataSource) DataSource() *storage.DataSource {
+	return r.datasource
 }
 
 func (r *registryWithDataSource) Source() *storage.ImageIntegration {
@@ -32,6 +37,10 @@ func (e *factoryImpl) CreateRegistry(source *storage.ImageIntegration) (types.Im
 
 	return &registryWithDataSource{
 		Registry: integration,
-		source:   source,
+		datasource: &storage.DataSource{
+			Id:   source.GetId(),
+			Name: source.GetName(),
+		},
+		source: source,
 	}, nil
 }
diff --git a/pkg/registries/set_impl_test.go b/pkg/registries/set_impl_test.go
@@ -41,6 +41,10 @@ func (f fakeRegistry) Name() string {
 	return f.name
 }
 
+func (f fakeRegistry) DataSource() *storage.DataSource {
+	return nil
+}
+
 func (f fakeRegistry) Source() *storage.ImageIntegration {
 	return nil
 }

diff --git a/pkg/registries/types/types.go b/pkg/registries/types/types.go
@@ -27,6 +27,7 @@ type Registry interface {
 // integration formed the interface
 type ImageRegistry interface {
 	Registry
+	DataSource() *storage.DataSource
 	Source() *storage.ImageIntegration
 }
 

diff --git a/qa-tests-backend/src/test/groovy/ImageScanningTest.groovy b/qa-tests-backend/src/test/groovy/ImageScanningTest.groovy
@@ -620,7 +620,8 @@ class ImageScanningTest extends BaseSpecification {
     }
 
     private static String source(String server) {
-        return "Autogenerated ${server} for cluster ${DEFAULT_CLUSTER_NAME} from.*"
+        // TODO: append " from .*" once SourcedAutogeneratedIntegrations is enabled.
+        return "Autogenerated ${server} for cluster ${DEFAULT_CLUSTER_NAME}"
     }
 
     @Unroll
@@ -801,8 +802,9 @@ class ImageScanningTest extends BaseSpecification {
     private static String expectAutoGeneratedRegistry(Secret secret) {
         ImageIntegrationOuterClass.ImageIntegration autoGenerated = null
         withRetry(5, 2) {
+            // TODO: append " from ${secret.namespace}/${secret.name}" once SourcedAutogeneratedIntegrations is enabled.
             autoGenerated = ImageIntegrationService.getImageIntegrationByName(
-                "Autogenerated ${secret.server} for cluster ${DEFAULT_CLUSTER_NAME} from ${secret.namespace}/${secret.name}"
+                "Autogenerated ${secret.server} for cluster ${DEFAULT_CLUSTER_NAME}"
             )
             assert autoGenerated
         }

diff --git a/sensor/common/detector/enricher.go b/sensor/common/detector/enricher.go
@@ -11,6 +11,7 @@ import (
 	"github.com/stackrox/rox/pkg/booleanpolicy/augmentedobjs"
 	"github.com/stackrox/rox/pkg/concurrency"
 	"github.com/stackrox/rox/pkg/expiringcache"
+	"github.com/stackrox/rox/pkg/features"
 	"github.com/stackrox/rox/pkg/images/types"
 	"github.com/stackrox/rox/pkg/set"
 	"github.com/stackrox/rox/sensor/common/clusterid"
@@ -60,14 +61,18 @@ func scanImage(ctx context.Context, svc v1.ImageServiceClient, req *scanImageReq
 	ctx, cancel := context.WithTimeout(ctx, scanTimeout)
 	defer cancel()
 
-	return svc.ScanImageInternal(ctx, &v1.ScanImageInternalRequest{
+	internalReq := &v1.ScanImageInternalRequest{
 		Image: req.containerImage,
-		Source: &v1.ScanImageInternalRequest_Source{
+	}
+	if features.SourcedAutogeneratedIntegrations.Enabled() {
+		internalReq.Source = &v1.ScanImageInternalRequest_Source{
 			ClusterId:        req.clusterID,
 			Namespace:        req.namespace,
 			ImagePullSecrets: req.pullSecrets,
-		},
-	})
+		}
+	}
+
+	return svc.ScanImageInternal(ctx, internalReq)
 }
 
 func scanImageLocal(ctx context.Context, svc v1.ImageServiceClient, req *scanImageRequest) (*v1.ScanImageInternalResponse, error) {
@@ -202,15 +207,19 @@ func (e *enricher) runImageScanAsync(imageChan chan<- imageChanResult, req *scan
 func (e *enricher) getImages(deployment *storage.Deployment) []*storage.Image {
 	imageChan := make(chan imageChanResult, len(deployment.GetContainers()))
 
-	pullSecretSet := set.NewStringSet(e.serviceAccountStore.GetImagePullSecrets(deployment.GetNamespace(), deployment.GetServiceAccount())...)
-	pullSecretSet.AddAll(deployment.GetImagePullSecrets()...)
+	var pullSecrets []string
+	if features.SourcedAutogeneratedIntegrations.Enabled() {
+		pullSecretsSet := set.NewStringSet(e.serviceAccountStore.GetImagePullSecrets(deployment.GetNamespace(), deployment.GetServiceAccount())...)
+		pullSecretsSet.AddAll(deployment.GetImagePullSecrets()...)
+		pullSecrets = pullSecretsSet.AsSlice()
+	}
 	for idx, container := range deployment.GetContainers() {
 		e.runImageScanAsync(imageChan, &scanImageRequest{
 			containerIdx:   idx,
 			containerImage: container.GetImage(),
 			clusterID:      clusterid.Get(),
 			namespace:      deployment.GetNamespace(),
-			pullSecrets:    pullSecretSet.AsSlice(),
+			pullSecrets:    pullSecrets,
 		})
 	}
 	images := make([]*storage.Image, len(deployment.GetContainers()))