stackrox · c-du · May 22, 2023 · Mar 14, 2023 · Mar 14, 2023 · Mar 14, 2023
@@ -0,0 +1,125 @@
+package m180tom181
+
+import (
+	"context"
+	"database/sql"
+	"os"
+
+	timestamp "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/stackrox/rox/generated/storage"
+	"github.com/stackrox/rox/migrator/migrations"
+	"github.com/stackrox/rox/migrator/migrations/m_180_to_m_181_move_to_blobstore/schema"
+	"github.com/stackrox/rox/migrator/types"
+	"github.com/stackrox/rox/pkg/logging"
+	"github.com/stackrox/rox/pkg/postgres/gorm/largeobject"
+	"github.com/stackrox/rox/pkg/postgres/pgutils"
+	"github.com/stackrox/rox/pkg/sac"
+	"github.com/stackrox/rox/pkg/utils"
+	"gorm.io/gorm"
+)
+
+const (
+	scannerDefBlobName = "/offline/scanner/scanner-defs.zip"
+)
+
+var (
+	scannerDefPath = "/var/lib/stackrox/scannerdefinitions/scanner-defs.zip"
+)
+
+var (
+	migration = types.Migration{
+		StartingSeqNum: 180,
+		VersionAfter:   &storage.Version{SeqNum: 181},
+		Run: func(databases *types.Databases) error {
+			err := moveToBlobs(databases.GormDB)
+			if err != nil {
+				return errors.Wrap(err, "moving persistent files to blobs")
+			}
+			return nil
+		},
+	}
+	log = logging.LoggerForModule()
+)
+
+func moveToBlobs(db *gorm.DB) (err error) {
+	ctx := sac.WithAllAccess(context.Background())
+	db = db.WithContext(ctx).Table(schema.BlobsTableName)
+	pgutils.CreateTableFromModel(context.Background(), db, schema.CreateTableBlobsStmt)
+
+	tx := db.Begin(&sql.TxOptions{Isolation: sql.LevelRepeatableRead})
+	if err = moveScannerDefinitions(tx); err != nil {
+		result := tx.Rollback()
+		if result.Error != nil {
+			log.Warnf("failed to rollback with error %v", result.Error)
+		}
+		return errors.Wrap(err, "failed to move scanner definition to blob store.")
+	}
+
+	return tx.Commit().Error
+}
+
+func moveScannerDefinitions(tx *gorm.DB) error {
+	fd, err := os.Open(scannerDefPath)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "failed to open %s", scannerDefPath)
+	}
+	defer utils.IgnoreError(fd.Close)
+	stat, err := fd.Stat()
+	if err != nil {
+		return err
+	}
+	if stat.IsDir() {
+		return nil
+	}
+	modTime, err := timestamp.TimestampProto(stat.ModTime())
+	if err != nil {
+		return errors.Wrapf(err, "invalid timestamp %v", stat.ModTime())
+	}
+
+	// Prepare blob
+	blob := &storage.Blob{
+		Name:         scannerDefBlobName,
+		Length:       stat.Size(),
+		LastUpdated:  timestamp.TimestampNow(),
+		ModifiedTime: modTime,
+	}
+	los := largeobject.LargeObjects{DB: tx}
+
+	// Find the blob if it exists
+	var targets []schema.Blobs
+	result := tx.Limit(1).Where(&schema.Blobs{Name: scannerDefBlobName}).Find(&targets)
+	if result.Error != nil {
+		return result.Error
+	}
+
+	if len(targets) == 0 {
+		blob.Oid, err = los.Create()
+		if err != nil {
+			return errors.Wrap(err, "failed to create large object")
+		}
+	} else {
+		// Update
+		existingBlob, err := schema.ConvertBlobToProto(&targets[0])
+		if err != nil {
+			return errors.Wrapf(err, "existing blob is not valid %+v", targets[0])
+		}
+		blob.Oid = existingBlob.Oid
+	}
+	blobModel, err := schema.ConvertBlobFromProto(blob)
+	if err != nil {
+		return errors.Wrapf(err, "failed to convert blob to blob model %+v", blob)
+	}
+	tx = tx.FirstOrCreate(blobModel)
+	if tx.Error != nil {
+		return errors.Wrap(tx.Error, "failed to create blob metadata")
+	}
+	return los.Upsert(blob.Oid, fd)
+}
+
+func init() {
+	migrations.MustRegisterMigration(migration)
+}
@@ -0,0 +1,98 @@
+//go:build sql_integration
+
+package m180tom181
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stackrox/rox/migrator/migrations/m_180_to_m_181_move_to_blobstore/schema"
+	pghelper "github.com/stackrox/rox/migrator/migrations/postgreshelper"
+	"github.com/stackrox/rox/pkg/postgres/gorm/largeobject"
+	"github.com/stackrox/rox/pkg/postgres/pgutils"
+	"github.com/stretchr/testify/suite"
+)
+
+type blobMigrationTestSuite struct {
+	suite.Suite
+
+	db *pghelper.TestPostgres
+}
+
+func TestMigration(t *testing.T) {
+	suite.Run(t, new(blobMigrationTestSuite))
+}
+
+func (s *blobMigrationTestSuite) SetupTest() {
+	s.db = pghelper.ForT(s.T(), true)
+}
+
+func (s *blobMigrationTestSuite) TearDownTest() {
+	s.db.Teardown(s.T())
+}
+
+func (s *blobMigrationTestSuite) TestMigration() {
+	// Nothing to migrate
+	s.Require().NoError(moveToBlobs(s.db.GetGormDB()))
+
+	// Prepare persistent file
+	size := 90000
+	randomData := make([]byte, size)
+	_, err := rand.Read(randomData)
+	s.Require().NoError(err)
+	reader := bytes.NewBuffer(randomData)
+
+	file, err := os.CreateTemp("", "move-blob")
+	s.Require().NoError(err)
+	defer func() {
+		s.NoError(file.Close())
+		s.NoError(os.Remove(file.Name()))
+	}()
+	scannerDefPath = file.Name()
+	n, err := io.Copy(file, reader)
+	s.Require().NoError(err)
+
+	s.Require().EqualValues(size, n)
+	fileInfo, err := file.Stat()
+	s.Require().NoError(err)
+
+	// Migrate
+	s.Require().NoError(moveToBlobs(s.db.GetGormDB()))
+
+	// Verify Blob
+	blobModel := &schema.Blobs{Name: scannerDefBlobName}
+	s.Require().NoError(s.db.GetGormDB().First(&blobModel).Error)
+
+	blob, err := schema.ConvertBlobToProto(blobModel)
+	s.Require().NoError(err)
+	s.Equal(scannerDefBlobName, blob.GetName())
+	s.EqualValues(size, blob.GetLength())
+
+	modTime := pgutils.NilOrTime(blob.GetModifiedTime())
+	s.Equal(fileInfo.ModTime().UTC(), modTime.UTC())
+
+	// Verify Data
+	buf := bytes.NewBuffer([]byte{})
+
+	tx := s.db.GetGormDB().Begin()
+	s.Require().NoError(err)
+	los := &largeobject.LargeObjects{DB: tx}
+	s.Require().NoError(los.Get(blob.Oid, buf))
+	s.Equal(len(randomData), buf.Len())
+	s.Equal(randomData, buf.Bytes())
+	s.NoError(tx.Commit().Error)
+
+	// Test re-entry
+	s.Require().NoError(moveToBlobs(s.db.GetGormDB()))
+	buf.Reset()
+	tx = s.db.GetGormDB().Begin()
+	los = &largeobject.LargeObjects{DB: tx}
+	s.Require().NoError(err)
+	s.Require().NoError(los.Get(blob.Oid, buf))
+	s.Equal(len(randomData), buf.Len())
+	s.Equal(randomData, buf.Bytes())
+	s.NoError(tx.Commit().Error)
+}
@@ -0,0 +1,6 @@
+package schema
+
+// TODO(ROX-17180): Remove this auto-generation at the beginning of 4.2 or at least
+// before we made schema change to Blob store after first release.
+
+//go:generate pg-schema-migration-helper --type=storage.Blob
@@ -136,4 +136,5 @@ import (
 	_ "github.com/stackrox/rox/migrator/migrations/m_177_to_m_178_group_permissions"
 	_ "github.com/stackrox/rox/migrator/migrations/m_178_to_m_179_embedded_collections_search_label"
 	_ "github.com/stackrox/rox/migrator/migrations/m_179_to_m_180_openshift_policy_exclusions"
+	_ "github.com/stackrox/rox/migrator/migrations/m_180_to_m_181_move_to_blobstore"
 )
diff --git a/pkg/migrations/internal/seq_num.go b/pkg/migrations/internal/seq_num.go
@@ -4,13 +4,13 @@ var (
 	// CurrentDBVersionSeqNum is the current DB version number.
 	// This must be incremented every time we write a migration.
 	// It is a shared constant between central and the migrator binary.
-	CurrentDBVersionSeqNum = 180
+	CurrentDBVersionSeqNum = 181
 
 	// MinimumSupportedDBVersionSeqNum is the minimum DB version number
 	// that is supported by this database.  This is used in case of rollbacks in
 	// the event that a major change introduced an incompatible schema update we
 	// can inform that a rollback below this is not supported by the database
-	MinimumSupportedDBVersionSeqNum = 180
+	MinimumSupportedDBVersionSeqNum = 181
 
 	// LastRocksDBVersionSeqNum is the sequence number for the last RocksDB version.
 	LastRocksDBVersionSeqNum = 112