diff --git a/sensor/kubernetes/localscanner/service_certificates_repository.go b/sensor/kubernetes/localscanner/service_certificates_repository.go new file mode 100644 index 0000000000000..ea677811c0e5f --- /dev/null +++ b/sensor/kubernetes/localscanner/service_certificates_repository.go @@ -0,0 +1,237 @@ +package localscanner + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/mtls" + v1 "k8s.io/api/core/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sTypes "k8s.io/apimachinery/pkg/types" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" +) + +var ( + // ErrUnexpectedSecretsOwner indicates that this repository should not be updating the certificates in + // the secrets they do not have the expected owner. + ErrUnexpectedSecretsOwner = errors.New("unexpected owner for certificate secrets") + + // ErrDifferentCAForDifferentServiceTypes indicates that different service types have different values + // for CA stored in their secrets. + ErrDifferentCAForDifferentServiceTypes = errors.New("found different CA PEM in secret Data for different service types") + + // ErrMissingSecretData indicates some secret has no data. + ErrMissingSecretData = errors.New("missing secret data") + + errForServiceFormat = "for service type %q" +) + +// serviceCertificatesRepoSecretsImpl is a ServiceCertificatesRepo that uses k8s secrets for persistence. +type serviceCertificatesRepoSecretsImpl struct { + secrets map[storage.ServiceType]serviceCertSecretSpec + ownerReference metav1.OwnerReference + namespace string + secretsClient corev1.SecretInterface +} + +// serviceCertSecretSpec specifies the name of the secret where certificates for a service are stored, and +// the secret data keys where each certificate file is stored. +type serviceCertSecretSpec struct { + secretName string + caCertFileName string + serviceCertFileName string + serviceKeyFileName string +} + +// newServiceCertificatesRepo creates a new serviceCertificatesRepoSecretsImpl that persists certificates for +// scanner and scanner DB in k8s secrets that are expected to have ownerReference as the only owner reference. +func newServiceCertificatesRepo(ownerReference metav1.OwnerReference, namespace string, + secretsClient corev1.SecretInterface) *serviceCertificatesRepoSecretsImpl { + + return &serviceCertificatesRepoSecretsImpl{ + secrets: map[storage.ServiceType]serviceCertSecretSpec{ + storage.ServiceType_SCANNER_SERVICE: { + secretName: "scanner-slim-tls", + caCertFileName: mtls.CACertFileName, + serviceCertFileName: mtls.ServiceCertFileName, + serviceKeyFileName: mtls.ServiceKeyFileName, + }, + storage.ServiceType_SCANNER_DB_SERVICE: { + secretName: "scanner-db-slim-tls", + caCertFileName: mtls.CACertFileName, + serviceCertFileName: mtls.ServiceCertFileName, + serviceKeyFileName: mtls.ServiceKeyFileName, + }, + }, + ownerReference: ownerReference, + namespace: namespace, + secretsClient: secretsClient, + } +} + +// getServiceCertificates fails as soon as the context is cancelled. Otherwise it returns a multierror that can contain +// the following errors: +// - ErrUnexpectedSecretsOwner in case the owner specified in the constructor is not the sole owner of all secrets. +// - ErrMissingSecretData in case any secret has no data. +// - ErrDifferentCAForDifferentServiceTypes in case the CA is not the same in all secrets. +// If the data for a secret is missing some expecting key then the corresponding field in the TypedServiceCertificate. +// for that secret will contain a zero value. +func (r *serviceCertificatesRepoSecretsImpl) getServiceCertificates(ctx context.Context) (*storage.TypedServiceCertificateSet, error) { + certificates := &storage.TypedServiceCertificateSet{} + certificates.ServiceCerts = make([]*storage.TypedServiceCertificate, 0) + var getErr error + for serviceType, secretSpec := range r.secrets { + // on context cancellation abort getting other secrets. + if ctx.Err() != nil { + return nil, ctx.Err() + } + + certificate, ca, err := r.getServiceCertificate(ctx, serviceType, secretSpec) + if err != nil { + getErr = multierror.Append(getErr, err) + continue + } + if certificates.GetCaPem() == nil { + certificates.CaPem = ca + } else { + if !bytes.Equal(certificates.GetCaPem(), ca) { + getErr = multierror.Append(getErr, ErrDifferentCAForDifferentServiceTypes) + continue + } + } + certificates.ServiceCerts = append(certificates.ServiceCerts, certificate) + } + + if getErr != nil { + return nil, getErr + } + + return certificates, nil +} + +func (r *serviceCertificatesRepoSecretsImpl) getServiceCertificate(ctx context.Context, serviceType storage.ServiceType, + secretSpec serviceCertSecretSpec) (cert *storage.TypedServiceCertificate, ca []byte, err error) { + + secret, getErr := r.secretsClient.Get(ctx, secretSpec.secretName, metav1.GetOptions{}) + if getErr != nil { + return nil, nil, getErr + } + + ownerReferences := secret.GetOwnerReferences() + if len(ownerReferences) != 1 { + return nil, nil, ErrUnexpectedSecretsOwner + } + + if ownerReferences[0].UID != r.ownerReference.UID { + return nil, nil, ErrUnexpectedSecretsOwner + } + + secretData := secret.Data + if secretData == nil { + return nil, nil, ErrMissingSecretData + } + + return &storage.TypedServiceCertificate{ + ServiceType: serviceType, + Cert: &storage.ServiceCertificate{ + CertPem: secretData[secretSpec.serviceCertFileName], + KeyPem: secretData[secretSpec.serviceKeyFileName], + }, + }, secretData[secretSpec.caCertFileName], nil +} + +// ensureServiceCertificates ensures the services for certificates exists, and that they contain the certificates +// in their data. +// This operation is idempotent but not atomic in sense that on error some secrets might be created and updated, +// while others are not. +// Each missing secret is created with the owner specified in the constructor as owner. +// This only creates secrets for the service types that appear in certificates, missing service types are just skipped. +// Fails for certificates with a service type that doesn't appear in r.secrets, as we don't know where to store them. +func (r *serviceCertificatesRepoSecretsImpl) ensureServiceCertificates(ctx context.Context, + certificates *storage.TypedServiceCertificateSet) error { + + caPem := certificates.GetCaPem() + var serviceErrors error + for _, cert := range certificates.GetServiceCerts() { + // on context cancellation abort putting other secrets. + if ctx.Err() != nil { + return ctx.Err() + } + + secretSpec, ok := r.secrets[cert.GetServiceType()] + if !ok { + // we don't know how to persist this. + err := errors.Errorf("unkown service type %q", cert.GetServiceType()) + serviceErrors = multierror.Append(serviceErrors, err) + continue + } + if err := r.ensureServiceCertificate(ctx, caPem, cert, secretSpec); err != nil { + serviceErrors = multierror.Append(serviceErrors, err) + } + } + + return serviceErrors +} + +func (r *serviceCertificatesRepoSecretsImpl) ensureServiceCertificate(ctx context.Context, caPem []byte, + cert *storage.TypedServiceCertificate, secretSpec serviceCertSecretSpec) error { + patchErr := r.patchServiceCertificate(ctx, caPem, cert, secretSpec) + if k8sErrors.IsNotFound(patchErr) { + _, createErr := r.createSecret(ctx, caPem, cert, secretSpec) + return createErr + } + return patchErr +} + +func (r *serviceCertificatesRepoSecretsImpl) patchServiceCertificate(ctx context.Context, caPem []byte, + cert *storage.TypedServiceCertificate, secretSpec serviceCertSecretSpec) error { + patch := []patchSecretDataByteMap{{ + Op: "replace", + Path: "/data", + Value: r.secretDataForCertificate(secretSpec, caPem, cert), + }} + patchBytes, marshallingErr := json.Marshal(patch) + if marshallingErr != nil { + return errors.Wrapf(marshallingErr, errForServiceFormat, cert.GetServiceType()) + } + if _, patchErr := r.secretsClient.Patch(ctx, secretSpec.secretName, k8sTypes.JSONPatchType, patchBytes, + metav1.PatchOptions{}); patchErr != nil { + return errors.Wrapf(patchErr, errForServiceFormat, cert.GetServiceType()) + } + + return nil +} + +type patchSecretDataByteMap struct { + Op string `json:"op"` + Path string `json:"path"` + Value map[string][]byte `json:"value"` +} + +func (r *serviceCertificatesRepoSecretsImpl) createSecret(ctx context.Context, caPem []byte, + certificate *storage.TypedServiceCertificate, secretSpec serviceCertSecretSpec) (*v1.Secret, error) { + + return r.secretsClient.Create(ctx, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretSpec.secretName, + Namespace: r.namespace, + OwnerReferences: []metav1.OwnerReference{r.ownerReference}, + }, + Data: r.secretDataForCertificate(secretSpec, caPem, certificate), + }, metav1.CreateOptions{}) +} + +func (r *serviceCertificatesRepoSecretsImpl) secretDataForCertificate(secretSpec serviceCertSecretSpec, caPem []byte, + cert *storage.TypedServiceCertificate) map[string][]byte { + + return map[string][]byte{ + secretSpec.caCertFileName: caPem, + secretSpec.serviceCertFileName: cert.GetCert().GetCertPem(), + secretSpec.serviceKeyFileName: cert.GetCert().GetKeyPem(), + } +} diff --git a/sensor/kubernetes/localscanner/service_certificates_repository_test.go b/sensor/kubernetes/localscanner/service_certificates_repository_test.go new file mode 100644 index 0000000000000..314bf00797639 --- /dev/null +++ b/sensor/kubernetes/localscanner/service_certificates_repository_test.go @@ -0,0 +1,364 @@ +package localscanner + +import ( + "context" + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/stackrox/rox/generated/storage" + "github.com/stackrox/rox/pkg/mtls" + "github.com/stretchr/testify/suite" + appsApiv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + fakecorev1 "k8s.io/client-go/kubernetes/typed/core/v1/fake" + k8sTesting "k8s.io/client-go/testing" +) + +const ( + namespace = "stackrox-ns" +) + +var ( + errForced = errors.New("forced error") + serviceType = storage.ServiceType_SCANNER_SERVICE + anotherServiceType = storage.ServiceType_SENSOR_SERVICE + serviceCertificate = &storage.TypedServiceCertificate{ + ServiceType: serviceType, + Cert: &storage.ServiceCertificate{ + CertPem: make([]byte, 0), + KeyPem: make([]byte, 1), + }, + } + certificates = &storage.TypedServiceCertificateSet{ + CaPem: make([]byte, 2), + ServiceCerts: []*storage.TypedServiceCertificate{ + serviceCertificate, + }, + } + sensorDeployment = &appsApiv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sensor-deployment", + Namespace: namespace, + }, + } +) + +func TestServiceCertificatesRepoSecretsImpl(t *testing.T) { + suite.Run(t, new(serviceCertificatesRepoSecretsImplSuite)) +} + +type serviceCertificatesRepoSecretsImplSuite struct { + suite.Suite +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestGet() { + testCases := map[string]struct { + expectedErr error + fixture *certSecretsRepoFixture + }{ + "successful get": {expectedErr: nil, fixture: s.newFixture(certSecretsRepoFixtureConfig{})}, + "failed get due to k8s API error": { + expectedErr: errForced, + fixture: s.newFixture(certSecretsRepoFixtureConfig{k8sAPIVerbToError: "get"}), + }, + "cancelled get": {expectedErr: context.Canceled, fixture: s.newFixture(certSecretsRepoFixtureConfig{})}, + } + for tcName, tc := range testCases { + s.Run(tcName, func() { + getCtx, cancelGetCtx := context.WithCancel(context.Background()) + defer cancelGetCtx() + if tc.expectedErr == context.Canceled { + cancelGetCtx() + } + + certificates, err := tc.fixture.repo.getServiceCertificates(getCtx) + + if tc.expectedErr == nil { + s.Equal(tc.fixture.certificates, certificates) + } + s.ErrorIs(err, tc.expectedErr) + }) + } +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestGetDifferentCAsFailure() { + testCases := map[string]struct { + expectedErr error + secondCASize int + }{ + "same CAs successful get": {expectedErr: nil, secondCASize: 0}, + "different CAs failed get": {expectedErr: ErrDifferentCAForDifferentServiceTypes, secondCASize: 1}, + } + for tcName, tc := range testCases { + s.Run(tcName, func() { + secret1 := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret1", + Namespace: namespace, + OwnerReferences: sensorOwnerReference(), + }, + Data: map[string][]byte{ + mtls.CACertFileName: make([]byte, 0), + }, + } + secret2 := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret2", + Namespace: namespace, + OwnerReferences: sensorOwnerReference(), + }, + Data: map[string][]byte{ + mtls.CACertFileName: make([]byte, tc.secondCASize), + }, + } + secrets := map[storage.ServiceType]*v1.Secret{serviceType: secret1, anotherServiceType: secret2} + clientSet := fake.NewSimpleClientset(secret1, secret2) + secretsClient := clientSet.CoreV1().Secrets(namespace) + repo := newTestRepo(secrets, secretsClient) + + _, err := repo.getServiceCertificates(context.Background()) + + s.ErrorIs(err, tc.expectedErr) + }) + } +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestPatch() { + testCases := map[string]struct { + expectedErr error + fixture *certSecretsRepoFixture + }{ + "successful patch": {expectedErr: nil, fixture: s.newFixture(certSecretsRepoFixtureConfig{})}, + "failed patch due to k8s API error": { + expectedErr: errForced, + fixture: s.newFixture(certSecretsRepoFixtureConfig{k8sAPIVerbToError: "patch"}), + }, + "cancelled patch": {expectedErr: context.Canceled, fixture: s.newFixture(certSecretsRepoFixtureConfig{})}, + } + for tcName, tc := range testCases { + s.Run(tcName, func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tc.expectedErr == context.Canceled { + cancel() + } + + err := tc.fixture.repo.ensureServiceCertificates(ctx, tc.fixture.certificates) + + s.ErrorIs(err, tc.expectedErr) + }) + } +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestGetNoSecretDataFailure() { + fixture := s.newFixture(certSecretsRepoFixtureConfig{emptySecretData: true}) + + _, err := fixture.repo.getServiceCertificates(context.Background()) + + s.ErrorIs(err, ErrMissingSecretData) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestGetUnexpectedSecretsOwnerFailure() { + fixture := s.newFixture(certSecretsRepoFixtureConfig{secretOwnerRefUID: "wrong owner"}) + + _, err := fixture.repo.getServiceCertificates(context.Background()) + + s.ErrorIs(err, ErrUnexpectedSecretsOwner) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestGetSecretDataMissingKeysSuccess() { + testCases := map[string]struct { + missingSecretDataKey string + setExpectedCertsFunc func(certificates *storage.TypedServiceCertificateSet) + }{ + "missing CA": { + missingSecretDataKey: mtls.CACertFileName, + setExpectedCertsFunc: func(certificates *storage.TypedServiceCertificateSet) { + certificates.CaPem = nil + }}, + "missing Cert": { + missingSecretDataKey: mtls.ServiceCertFileName, + setExpectedCertsFunc: func(certificates *storage.TypedServiceCertificateSet) { + s.getFirstServiceCertificate(certificates).Cert.CertPem = nil + }, + }, + "missing Key": { + missingSecretDataKey: mtls.ServiceKeyFileName, + setExpectedCertsFunc: func(certificates *storage.TypedServiceCertificateSet) { + s.getFirstServiceCertificate(certificates).Cert.KeyPem = nil + }, + }, + } + for tcName, tc := range testCases { + s.Run(tcName, func() { + fixture := s.newFixture(certSecretsRepoFixtureConfig{missingSecretDataKeys: []string{tc.missingSecretDataKey}}) + + certificates, err := fixture.repo.getServiceCertificates(context.Background()) + + s.Require().NoError(err) + tc.setExpectedCertsFunc(fixture.certificates) + s.Equal(fixture.certificates, certificates) + }) + } +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestEnsureCertsUnknownServiceTypeFailure() { + fixture := s.newFixture(certSecretsRepoFixtureConfig{}) + s.getFirstServiceCertificate(fixture.certificates).ServiceType = anotherServiceType + + err := fixture.repo.ensureServiceCertificates(context.Background(), fixture.certificates) + + s.Error(err) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestEnsureCertsMissingServiceTypeSuccess() { + fixture := s.newFixture(certSecretsRepoFixtureConfig{}) + fixture.certificates.ServiceCerts = make([]*storage.TypedServiceCertificate, 0) + + err := fixture.repo.ensureServiceCertificates(context.Background(), fixture.certificates) + + s.NoError(err) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestCreateSecretsNoCertificatesSuccess() { + clientSet := fake.NewSimpleClientset(sensorDeployment) + secretsClient := clientSet.CoreV1().Secrets(namespace) + repo := newServiceCertificatesRepo(sensorOwnerReference()[0], namespace, secretsClient) + + s.NoError(repo.ensureServiceCertificates(context.Background(), nil)) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestCreateSecretsCancelFailure() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + clientSet := fake.NewSimpleClientset(sensorDeployment) + secretsClient := clientSet.CoreV1().Secrets(namespace) + + repo := newServiceCertificatesRepo(sensorOwnerReference()[0], namespace, secretsClient) + + s.Error(repo.ensureServiceCertificates(ctx, certificates.Clone())) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) TestEnsureServiceCertificateMissingSecretSuccess() { + clientSet := fake.NewSimpleClientset(sensorDeployment) + secretsClient := clientSet.CoreV1().Secrets(namespace) + repo := newServiceCertificatesRepo(sensorOwnerReference()[0], namespace, secretsClient) + + err := repo.ensureServiceCertificates(context.Background(), certificates) + + s.NoError(err) +} + +func (s *serviceCertificatesRepoSecretsImplSuite) getFirstServiceCertificate( + certificates *storage.TypedServiceCertificateSet) *storage.TypedServiceCertificate { + serviceCerts := certificates.GetServiceCerts() + s.Require().Len(serviceCerts, 1) + return serviceCerts[0] +} + +type certSecretsRepoFixture struct { + repo *serviceCertificatesRepoSecretsImpl + secretsClient corev1.SecretInterface + certificates *storage.TypedServiceCertificateSet +} + +// newFixture creates a certSecretsRepoFixture that contains: +// - A secrets client corresponding to a fake k8s client set such that: +// - It is initialized to represent a cluster with sensorDeployment and a secret that contains certificates +// on its data, or partial data according to spec. +// - The client set will fail all operations on the HTTP verb indicated in spec. +// - The certificates used to initialize the data of the aforementioned secret. +// - A repository that uses that secrets client, sensorDeployment as owner, and with a single serviceCertSecretSpec +// for the aforementioned secret in its secrets. +func (s *serviceCertificatesRepoSecretsImplSuite) newFixture(config certSecretsRepoFixtureConfig) *certSecretsRepoFixture { + certificates := certificates.Clone() + ownerRef := sensorOwnerReference() + if config.secretOwnerRefUID != "" { + ownerRef[0].UID = types.UID(config.secretOwnerRefUID) + } + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-secret", serviceType), + Namespace: namespace, + OwnerReferences: ownerRef, + }, + } + if !config.emptySecretData { + secret.Data = map[string][]byte{ + mtls.CACertFileName: certificates.GetCaPem(), + mtls.ServiceCertFileName: serviceCertificate.GetCert().GetCertPem(), + mtls.ServiceKeyFileName: serviceCertificate.GetCert().GetKeyPem(), + } + } + for _, secretDataKey := range config.missingSecretDataKeys { + delete(secret.Data, secretDataKey) + } + secrets := map[storage.ServiceType]*v1.Secret{serviceType: secret} + clientSet := fake.NewSimpleClientset(sensorDeployment, secret) + secretsClient := clientSet.CoreV1().Secrets(namespace) + clientSet.CoreV1().(*fakecorev1.FakeCoreV1).PrependReactor(config.k8sAPIVerbToError, "secrets", func(action k8sTesting.Action) (handled bool, ret runtime.Object, err error) { + return true, &v1.Secret{}, errForced + }) + repo := newTestRepo(secrets, secretsClient) + return &certSecretsRepoFixture{ + repo: repo, + secretsClient: secretsClient, + certificates: certificates, + } +} + +type certSecretsRepoFixtureConfig struct { + // HTTP verb of the k8s API should for which all operations will fail in the fake k8s client set. + // Use the zero value so all operations work. + k8sAPIVerbToError string + // If true then the data of the secret used to initialize the fake k8s client set will be empty. + emptySecretData bool + // These keys will be removed from the data keys of the secret used to initialize the fake k8s client set. + missingSecretDataKeys []string + // If set to a non-zero value, then the UID of the owner of the secret used to initialize the fake k8s client + // set will take this value. + secretOwnerRefUID string +} + +func sensorOwnerReference() []metav1.OwnerReference { + sensorDeploymentGVK := sensorDeployment.GroupVersionKind() + blockOwnerDeletion := false + isController := false + return []metav1.OwnerReference{ + { + APIVersion: sensorDeploymentGVK.GroupVersion().String(), + Kind: sensorDeploymentGVK.Kind, + Name: sensorDeployment.GetName(), + UID: sensorDeployment.GetUID(), + BlockOwnerDeletion: &blockOwnerDeletion, + Controller: &isController, + }, + } +} + +func newTestRepo(secrets map[storage.ServiceType]*v1.Secret, + secretsClient corev1.SecretInterface) *serviceCertificatesRepoSecretsImpl { + + secretsSpec := make(map[storage.ServiceType]serviceCertSecretSpec) + for serviceType, secret := range secrets { + secretsSpec[serviceType] = serviceCertSecretSpec{ + secretName: secret.Name, + caCertFileName: mtls.CACertFileName, + serviceCertFileName: mtls.ServiceCertFileName, + serviceKeyFileName: mtls.ServiceKeyFileName, + } + } + + return &serviceCertificatesRepoSecretsImpl{ + secrets: secretsSpec, + ownerReference: sensorOwnerReference()[0], + namespace: namespace, + secretsClient: secretsClient, + } +}