diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index 23c250cc535..cbb505c3fe8 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -13,6 +13,7 @@ on: jobs: operator-e2e-tests: + timeout-minutes: 40 if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && @@ -38,7 +39,7 @@ jobs: - name: Create KIND cluster run: | - kind create cluster --name $KIND_CLUSTER --wait 5m + kind create cluster --name $KIND_CLUSTER --wait 10m - name: Set up kubernetes context run: | @@ -51,8 +52,16 @@ jobs: cd infra/feast-operator/ make test-e2e + - name: Debug KIND Cluster when there is a failure + if: failure() + run: | + kubectl get pods --all-namespaces + kubectl describe nodes + - name: Clean up if: always() run: | # Delete the KIND cluster after tests kind delete cluster --name kind-$KIND_CLUSTER + + diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 310d64afaaa..6984ac66e7f 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -117,7 +117,7 @@ test: build-installer fmt vet lint envtest ## Run tests. # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: - go test ./test/e2e/ -v -ginkgo.v + go test -timeout 30m ./test/e2e/ -v -ginkgo.v .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index 7d9fb9af056..23637ff224b 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -28,145 +28,188 @@ import ( ) const feastControllerNamespace = "feast-operator-system" +const timeout = 2 * time.Minute +const controllerDeploymentName = "feast-operator-controller-manager" var _ = Describe("controller", Ordered, func() { BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace) _, _ = utils.Run(cmd) + var err error + // projectimage stores the name of the image used in the example + var projectimage = "localhost/feast-operator:v0.0.1" + + By("building the manager(Operator) image") + cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectimage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("building the feast image") + cmd = exec.Command("make", "feast-ci-dev-docker-img") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + // this image will be built in above make target. + var feastImage = "feastdev/feature-server:dev" + var feastLocalImage = "localhost/feastdev/feature-server:dev" + + By("Tag the local feast image for the integration tests") + cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the feast image on Kind cluster") + err = utils.LoadImageToKindClusterWithName(feastLocalImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("Validating that the controller-manager deployment is in available state") + err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) + Expect(err).To(BeNil(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + controllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) }) AfterAll(func() { //Add any post clean up code here. + By("Uninstalling the feast CRD") + cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - Context("Operator", func() { + Context("Operator E2E Tests", func() { It("Should be able to deploy and run a default feature store CR successfully", func() { - //var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "localhost/feast-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("building the feast image") - cmd = exec.Command("make", "feast-ci-dev-docker-img") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - // this image will be built in above make target. - var feastImage = "feastdev/feature-server:dev" - var feastLocalImage = "localhost/feastdev/feature-server:dev" - - By("Tag the local feast image for the integration tests") - cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the feast image on Kind cluster") - err = utils.LoadImageToKindClusterWithName(feastLocalImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - timeout := 2 * time.Minute - - controllerDeploymentName := "feast-operator-controller-manager" - By("Validating that the controller-manager deployment is in available state") - err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - controllerDeploymentName, err, - )) - fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) - By("deploying the Simple Feast Custom Resource to Kubernetes") - cmd = exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) _, cmdOutputerr := utils.Run(cmd) ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - namespace := "default" - - deploymentNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, deploymentName := range deploymentNames { - By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) - err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - deploymentName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", deploymentName) - } - - By("Check if the feast client - kubernetes config map exists.") - configMapName := "feast-simple-feast-setup-client" - err = checkIfConfigMapExists(namespace, configMapName) - Expect(err).To(BeNil(), fmt.Sprintf( - "config map %s is not available but expected to be available. \nError: %v\n", - configMapName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", configMapName) - - serviceAccountNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, serviceAccountName := range serviceAccountNames { - By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) - err = checkIfServiceAccountExists(namespace, serviceAccountName) - Expect(err).To(BeNil(), fmt.Sprintf( - "Service account %s does not exist in namespace %s. Error: %v", - serviceAccountName, namespace, err, - )) - fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) - } - - serviceNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, serviceName := range serviceNames { - By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) - err = checkIfKubernetesServiceExists(namespace, serviceName) - Expect(err).To(BeNil(), fmt.Sprintf( - "kubernetes service %s is not available but expected to be available. \nError: %v\n", - serviceName, err, - )) - fmt.Printf("kubernetes service %s is available\n", serviceName) - } - - By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", "simple-feast-setup")) - err = checkIfFeatureStoreCustomResourceConditionsInReady("simple-feast-setup", namespace) - Expect(err).To(BeNil(), fmt.Sprintf( - "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", - "simple-feast-setup", err, - )) - fmt.Printf("FeatureStore customer resource %s conditions are in Ready State\n", "simple-feast-setup") + featureStoreName := "simple-feast-setup" + validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) By("deleting the feast deployment") cmd = exec.Command("kubectl", "delete", "-f", "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") _, cmdOutputerr = utils.Run(cmd) ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + }) + + It("Should be able to deploy and run a feature store with remote registry CR successfully", func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) + _, cmdOutputerr := utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + + featureStoreName := "simple-feast-setup" + validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) + + var remoteRegistryNs = "remote-registry" + cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) + _, _ = utils.Run(cmd) + + By("deploying the Simple Feast remote registry Custom Resource to Kubernetes") + cmd = exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - By("Uninstalling the feast CRD") - cmd = exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + remoteFeatureStoreName := "simple-feast-remote-setup" + + validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout) + + By("deleting the feast remote registry deployment") + cmd = exec.Command("kubectl", "delete", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) }) }) }) + +func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) { + hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) + Expect(err).To(BeNil(), fmt.Sprintf( + "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", + featureStoreName, err)) + + k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName), + fmt.Sprintf("feast-%s-offline", featureStoreName), + } + + if !hasRemoteRegistry { + k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName)) + } + + for _, deploymentName := range k8ResourceNames { + By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) + err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) + Expect(err).To(BeNil(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + deploymentName, err, + )) + fmt.Printf("Feast Deployment %s is available\n", deploymentName) + } + + By("Check if the feast client - kubernetes config map exists.") + configMapName := fmt.Sprintf("feast-%s-client", featureStoreName) + err = checkIfConfigMapExists(namespace, configMapName) + Expect(err).To(BeNil(), fmt.Sprintf( + "config map %s is not available but expected to be available. \nError: %v\n", + configMapName, err, + )) + fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) + + for _, serviceAccountName := range k8ResourceNames { + By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) + err = checkIfServiceAccountExists(namespace, serviceAccountName) + Expect(err).To(BeNil(), fmt.Sprintf( + "Service account %s does not exist in namespace %s. Error: %v", + serviceAccountName, namespace, err, + )) + fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) + } + + for _, serviceName := range k8ResourceNames { + By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) + err = checkIfKubernetesServiceExists(namespace, serviceName) + Expect(err).To(BeNil(), fmt.Sprintf( + "kubernetes service %s is not available but expected to be available. \nError: %v\n", + serviceName, err, + )) + fmt.Printf("kubernetes service %s is available\n", serviceName) + } + + By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) + err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) + Expect(err).To(BeNil(), fmt.Sprintf( + "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", + featureStoreName, err, + )) + fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) +} diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index f30d8cbebf5..7d44ac1296a 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -3,10 +3,13 @@ package e2e import ( "bytes" "encoding/json" + "errors" "fmt" "os/exec" "strings" "time" + + "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" ) // dynamically checks if all conditions of custom resource featurestore are in "Ready" state. @@ -184,3 +187,46 @@ func checkIfKubernetesServiceExists(namespace, serviceName string) error { return nil } + +func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, + "-o=jsonpath='{.spec.services.registry}'") + + // Capture the output + output, err := cmd.Output() + if err != nil { + return false, err // Return false on command execution failure + } + + // Convert output to string and trim any extra spaces + result := strings.TrimSpace(string(output)) + + // Remove single quotes if present + if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { + result = strings.Trim(result, "'") + } + + if result == "" { + return false, errors.New("kubectl get featurestore command returned empty output") + } + + // Parse the JSON into a map + var registryConfig v1alpha1.Registry + if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { + return false, err // Return false on JSON parsing failure + } + + if registryConfig.Remote == nil { + return false, nil + } + + hasHostname := registryConfig.Remote.Hostname != nil + hasValidFeastRef := registryConfig.Remote.FeastRef != nil && + registryConfig.Remote.FeastRef.Name != "" + + if hasHostname || hasValidFeastRef { + return true, nil + } + + return false, nil +} diff --git a/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml b/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml new file mode 100644 index 00000000000..61c010f0576 --- /dev/null +++ b/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml @@ -0,0 +1,16 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: simple-feast-remote-setup +spec: + feastProject: my_project + services: + onlineStore: + image: 'localhost/feastdev/feature-server:dev' + offlineStore: + image: 'localhost/feastdev/feature-server:dev' + registry: + remote: + feastRef: + name: simple-feast-setup + namespace: default \ No newline at end of file