From 5242d95d93f61ee61f78886e81f98f56de23931b Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Thu, 6 Feb 2025 17:12:28 -0500 Subject: [PATCH 01/13] rough working instructions to set up postgres in TLS mode and setting up the feast using operator. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/operator-postgres-ssl-demo/README.md | 84 +++ .../operator-postgres-ssl-demo/feast.yaml | 137 +++++ .../postgres-helm-tls.ipynb | 482 ++++++++++++++++++ .../operator-postgres-ssl-demo/redis.yaml | 39 ++ .../operator-postgres-ssl-demo/values.yaml | 9 + 5 files changed, 751 insertions(+) create mode 100644 examples/operator-postgres-ssl-demo/README.md create mode 100644 examples/operator-postgres-ssl-demo/feast.yaml create mode 100644 examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb create mode 100644 examples/operator-postgres-ssl-demo/redis.yaml create mode 100644 examples/operator-postgres-ssl-demo/values.yaml diff --git a/examples/operator-postgres-ssl-demo/README.md b/examples/operator-postgres-ssl-demo/README.md new file mode 100644 index 00000000000..e717a556bed --- /dev/null +++ b/examples/operator-postgres-ssl-demo/README.md @@ -0,0 +1,84 @@ +# Install and run a Feature Store on Kubernetes with the Feast Operator + +The following notebooks will guide you through how to install and use Feast on Kubernetes with the Feast Go Operator. + +* [01-Install.ipynb](./01-Install.ipynb): Install and configure a Feature Store in Kubernetes with the Operator. +* [02-Demo.ipynb](./02-Demo.ipynb): Validate the feature store with demo application. +* [03-Uninstall.ipynb](./03-Uninstall.ipynb): Clear the installed deployments. + +# Here are the steps to deploy + +* Generate certificates and create kubernetes secrets. +* helm install postgresql bitnami/postgresql -f values.yaml +* kubectl apply -f feast.yaml + + +# Some of the important commands to debug while setting up the postgres in TLS setup. +helm show chart bitnami/postgresql + +helm install postgresql -f values.yaml bitnami/postgresql --version 16.4.6 + +helm uninstall postgresql + +make test +make docker-build docker-push IMG=quay.io/lrangine/feast-operator:0.53.0 +make install +make deploy IMG=quay.io/lrangine/feast-operator:0.53.0 + +kubectl exec -it postgresql-0 -- env PGPASSWORD=secret psql -U admin -d mydb -c "SHOW ssl;" + +kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c "SHOW ssl;" + + +kubectl exec -it my-postgres-postgresql-0 -- bash + + + +kubectl create secret generic postgres-tls-secret \ +--from-file=tls.crt=postgres-certs/server.crt \ +--from-file=tls.key=postgres-certs/server.key \ +--from-file=ca.crt=postgres-certs/ca.crt + +kubectl create secret generic postgres-tls-certs-new \ +--from-file=tls.crt=postgres-tls-certs-new/server.crt \ +--from-file=tls.key=postgres-tls-certs-new/server.key \ +--from-file=ca.crt=postgres-tls-certs-new/ca.crt + + + +kubectl run -it --rm --image=postgres:latest --restart=Never postgresql-client -- psql "host=postgresql.default.svc.cluster.local dbname=mydatabase user=admin password=mypassword sslmode=require" + + +kubectl exec -it postgresql-client -- psql \ +"host=postgresql.default.svc.cluster.local dbname=mydatabase user=admin password=mypassword sslmode=verify-full sslcert=/etc/ssl/postgres/client.crt sslkey=/etc/ssl/postgres/client.key sslrootcert=/etc/ssl/postgres/ca.crt" + +postgresql+psycopg://admin:password@postgresql.default.svc.cluster.local:5432/mydb?sslmode=require&sslrootcert=postgres-tls-certs/ca.crt&sslcert=postgres-tls-certs/client.crt&sslkey=postgres-tls-certs/client.key + + +kubectl exec -it postgresql-0 -- cat /opt/bitnami/postgresql/conf/postgresql.conf | grep ssl + +sslcert=postgres-tls-certs-new/server.crt&sslkey=postgres-tls-certs-new/server.key + +postgresql+psycopg://admin:password@localhost:5432/mydatabase?sslmode=disable&sslrootcert=postgres-tls-certs-new/ca.crt&sslcert=postgres-tls-certs-new/server.crt&sslkey=postgres-tls-certs-new/server.key + + +kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c '\l' + + +helm install postgresql -f values.yaml bitnami/postgresql --version 16.4.6 + +helm uninstall postgresql + +helm template postgresql bitnami/postgresql --values values.yaml --version 16.4.6 > postgresql-export.yaml + + +kubectl create secret generic postgresql-client-certs \ +--from-file=ca.crt=./postgres-tls-certs/ca.crt \ +--from-file=tls.crt=./postgres-tls-certs/client.crt \ +--from-file=tls.key=./postgres-tls-certs/client.key \ +--dry-run=client -o yaml | kubectl apply -f - + + +!kubectl exec deploy/postgres -- psql -h localhost -U feast feast -c '\dt' + +kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c '\dt' diff --git a/examples/operator-postgres-ssl-demo/feast.yaml b/examples/operator-postgres-ssl-demo/feast.yaml new file mode 100644 index 00000000000..3bce0ef2544 --- /dev/null +++ b/examples/operator-postgres-ssl-demo/feast.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + namespace: default + labels: + app: postgres +stringData: + POSTGRES_DB: mydatabase + POSTGRES_USER: admin + POSTGRES_PASSWORD: password +--- +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores + namespace: default +stringData: + redis: | + connection_string: redis.feast.svc.cluster.local:6379 + sql: | + path: postgresql+psycopg://admin:password@postgresql.default.svc.cluster.local:5432/mydatabase?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true +--- +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: example + namespace: default +spec: + feastProject: credit_scoring_local + services: + volumes: + - name: ca-cert + secret: + secretName: postgresql-client-certs + items: + - key: ca.crt + path: ca.crt + mode: 0644 # Readable by all, required by PostgreSQL + + - name: client-cert + secret: + secretName: postgresql-client-certs + items: + - key: tls.crt + path: tls.crt + mode: 0644 # Required for the client certificate + + - name: client-key + secret: + secretName: postgresql-client-certs + items: + - key: tls.key + path: tls.key + mode: 0640 # Required for the private key + offlineStore: + volumeMounts: + - name: ca-cert + mountPath: /var/lib/postgresql/certs/ca.crt + subPath: ca.crt + readOnly: true + + - name: client-cert + mountPath: /var/lib/postgresql/certs/tls.crt + subPath: tls.crt + readOnly: true + + - name: client-key + mountPath: /var/lib/postgresql/certs/tls.key + subPath: tls.key + readOnly: true + persistence: + file: + type: duckdb + envFrom: + - secretRef: + name: postgres-secret + onlineStore: + volumeMounts: + - name: ca-cert + mountPath: /var/lib/postgresql/certs/ca.crt + subPath: ca.crt + readOnly: true + + - name: client-cert + mountPath: /var/lib/postgresql/certs/tls.crt + subPath: tls.crt + readOnly: true + + - name: client-key + mountPath: /var/lib/postgresql/certs/tls.key + subPath: tls.key + readOnly: true + persistence: + store: + type: redis + secretRef: + name: feast-data-stores + envFrom: + - secretRef: + name: postgres-secret + registry: + local: + volumeMounts: + - name: ca-cert + mountPath: /var/lib/postgresql/certs/ca.crt + subPath: ca.crt + readOnly: true + + - name: client-cert + mountPath: /var/lib/postgresql/certs/tls.crt + subPath: tls.crt + readOnly: true + + - name: client-key + mountPath: /var/lib/postgresql/certs/tls.key + subPath: tls.key + readOnly: true + persistence: + store: + type: sql + secretRef: + name: feast-data-stores + envFrom: + - secretRef: + name: postgres-secret + env: + - name: MPLCONFIGDIR + value: /tmp + resources: + requests: + cpu: 150m + memory: 128Mi \ No newline at end of file diff --git a/examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb b/examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb new file mode 100644 index 00000000000..43df716b656 --- /dev/null +++ b/examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb @@ -0,0 +1,482 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f16967ef", + "metadata": {}, + "source": [ + "# Deploy PostgreSQL with Helm in TLS Mode" + ] + }, + { + "cell_type": "markdown", + "id": "cce2278a", + "metadata": {}, + "source": [ + "## Step 1: Install Prerequisites" + ] + }, + { + "cell_type": "markdown", + "id": "3e4102d8", + "metadata": {}, + "source": [ + "Before starting, ensure you have the following installed:\n", + "- `kubectl` (Kubernetes CLI)\n", + "- `helm` (Helm CLI)\n", + "- A Kubernetes cluster (e.g., Minikube, GKE, EKS, or AKS)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e2b40efc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Client Version: v1.31.2\n", + "Kustomize Version: v5.4.2\n", + "version.BuildInfo{Version:\"v3.17.0\", GitCommit:\"301108edc7ac2a8ba79e4ebf5701b0b6ce6a31e4\", GitTreeState:\"clean\", GoVersion:\"go1.23.4\"}\n" + ] + } + ], + "source": [ + "# Verify kubectl and helm are installed\n", + "!kubectl version --client\n", + "!helm version" + ] + }, + { + "cell_type": "markdown", + "id": "4b72fabe", + "metadata": {}, + "source": [ + "## Step 2: Add the Bitnami Helm Repository" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "f439691e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\"bitnami\" already exists with the same configuration, skipping\n", + "Hang tight while we grab the latest from your chart repositories...\n", + "...Successfully got an update from the \"bitnami\" chart repository\n", + "Update Complete. ⎈Happy Helming!⎈\n" + ] + } + ], + "source": [ + "# Add the Bitnami Helm repository\n", + "!helm repo add bitnami https://charts.bitnami.com/bitnami\n", + "!helm repo update" + ] + }, + { + "cell_type": "markdown", + "id": "41f4e8db", + "metadata": {}, + "source": [ + "## Step 3: Generate TLS Certificates" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8e192410", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ".............+.+..+............+...+...................+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+.........+....+.....+.+..+.......+..+...+.+.....+......+.+..+.+...........+.+.........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+...........+...+......................+........+......+....+..+.............+...+............+........+....+........+...+...+......+.+..............+.........+.+..............+....+..+.............+..+...+.+......+.....+...+.......+..................+..+....+.....+.+.....+.+...+.........+...+.....+....+.........+.....+.......+..+............+.........+.+......+...+............+..+............+....+..+...+.+...............+...+..+....+...........+..........+.....+....+..+...+....+........+...+.+......+........+.+......+..+...+....+...+..+.+........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "..............+.+...+..................+..+...+..........+............+...............+......+......+.........+.....+...................+..+...............+...+......+.+...+..+.+.....+....+...+..+.+..................+...........+...+......+.+.........+...........+....+.....+.......+..+......+.....................+....+..+............+...+...............+....+...+...+...+......+...+...........+.........+...+...+.+...........+.......+......+..+...+....+..+...+.+..+............+.............+.....+.............+............+..+......+.+...+.....+......+............+...+.......+.....+.......+............+.........+..+....+......+........+.......+............+.....+.......+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "-----\n", + ".........+..........+.....+....+..+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+...+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.......+........+......+......+....+...+...........+.......+...+.........+...+...+......+.....+.+..+.+.................+...+.+.........+.....+....+..+...+...+....+........+......+.+........+..........+...+..+......+......+....+............+..+.............+............+.....+.......+.........+..+..........+.....+....+...+..+................+..+......+....+..+..........+..................+..+.+.....+.........................+..+......+......+....+.....+.+.....+.+..+............+...+..........+...+........+..........+..+.......+..+...+.............+..................+.....+.+.....+.........+.+...........+....+.....+....+.....+....+..+...+......................+......+...+......+...+..+.+........+.........+.............+.....+.........+.+.....+.......+.....+...+.......+..+..........+...+...+.....+.........+.+.......................+.......+.........+.....+..................+.+......+...+...........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+....+.................+....+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+....+.....+.+...+...+.........+......+...............+.....+...............+...+....+...............+..+..........++++++++++++++++++++++++\n", + "-----\n", + "Certificate request self-signature ok\n", + "subject=CN = postgresql.default.svc.cluster.local\n", + "........+.........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+..+.+..+....+...+..+.........+............+...+.+...+........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.......+..+.......+.....+.............+..+.+...+.................+...............+...+.+.....+.....................+.......+..+.+.........+......+...+..+.......+......+..............+.......+......+........+......+.+...+...........+.+..+......+......+...+......+..........+..+...............+............+.+.........+...+.....+.........+.+...+...............+..+.......+...+........+.+.....+......+.........+...+.+..+....+...+..+...+................+..+...+......+.+..+...............+......+.+..................+...+.....+..........+.................+.+.........+...............+......+.................+....+...........+....+.....+......+.......+..+...............+......+....+......+..+.+.....+...............+.......+.....+..........+........+......+....+.....+......+..........+........+.+..+....+......+........+.+...............+.........+...........+...+...+...+.......+...+.....+....+...+..+...+....+...+............+...+.....+...+...+......+...+.+.....+............+....+.....+......+.........+....+.........+..+...+.+..+....+.....+....+.........+...+.......................+............+....+...+...........+...............+....+..+.+..+.......+...+..+...+......+.+.......................+...............+...+...+....+..+...+.........................+......+..+....+.........+.....+.+.........+......+......+...............+...+...........+.+..+...+.........+....+.................+.......+..+....+......+...+...+.....+..........+........+.......+..+.......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "...........+.+.....+.......+..+.......+......+..+.............+........+....+...+.....+...++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*............+.....+...+............+............+...+.+......+.........+..+....+......+.....+.............+.....+......+...+............+....+......+......+..+...+..................+......+...+............+.+.....+.......+........+...+....+............+.....+.........+.......+...+...........+...+......+.+........+...+....+..................+........+.........+....+...\n", + "-----\n", + "Certificate request self-signature ok\n", + "subject=CN = admin\n" + ] + } + ], + "source": [ + "# Create a directory for certificates\n", + "!mkdir -p postgres-tls-certs\n", + "\n", + "# Generate a CA certificate\n", + "!openssl req -new -x509 -days 365 -nodes -out postgres-tls-certs/ca.crt -keyout postgres-tls-certs/ca.key -subj \"/CN=PostgreSQL CA\"\n", + "\n", + "# Generate a server certificate\n", + "!openssl req -new -nodes -out postgres-tls-certs/server.csr -keyout postgres-tls-certs/server.key -subj \"/CN=postgresql.default.svc.cluster.local\"\n", + "!openssl x509 -req -in postgres-tls-certs/server.csr -days 365 -CA postgres-tls-certs/ca.crt -CAkey postgres-tls-certs/ca.key -CAcreateserial -out postgres-tls-certs/server.crt\n", + "\n", + "# Generate a client certificate\n", + "!openssl req -new -nodes -out postgres-tls-certs/client.csr -keyout postgres-tls-certs/client.key -subj \"/CN=admin\"\n", + "!openssl x509 -req -in postgres-tls-certs/client.csr -days 365 -CA postgres-tls-certs/ca.crt -CAkey postgres-tls-certs/ca.key -CAcreateserial -out postgres-tls-certs/client.crt" + ] + }, + { + "cell_type": "markdown", + "id": "7e39cb28", + "metadata": {}, + "source": [ + "## Step 4: Create Kubernetes Secrets for Certificates" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d728d0d5-2ba6-4d4d-b4be-62fb020530d4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "error: failed to create secret secrets \"postgresql-server-certs\" already exists\n", + "secret/postgresql-client-certs created\n" + ] + } + ], + "source": [ + "# Create a secret for the server certificates\n", + "!kubectl create secret generic postgresql-server-certs --from-file=ca.crt=./postgres-tls-certs/ca.crt --from-file=tls.crt=./postgres-tls-certs/server.crt --from-file=tls.key=./postgres-tls-certs/server.key\n", + "\n", + "# Create a secret for the client certificates\n", + "!kubectl create secret generic postgresql-client-certs --from-file=ca.crt=./postgres-tls-certs/ca.crt --from-file=tls.crt=./postgres-tls-certs/client.crt --from-file=tls.key=./postgres-tls-certs/client.key" + ] + }, + { + "cell_type": "markdown", + "id": "67d62692", + "metadata": {}, + "source": [ + "## Step 5: Deploy PostgreSQL with Helm" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e14cae77", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: postgresql\n", + "LAST DEPLOYED: Thu Feb 6 12:36:33 2025\n", + "NAMESPACE: default\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n", + "NOTES:\n", + "CHART NAME: postgresql\n", + "CHART VERSION: 16.4.6\n", + "APP VERSION: 17.2.0\n", + "\n", + "Did you know there are enterprise versions of the Bitnami catalog? For enhanced secure software supply chain features, unlimited pulls from Docker, LTS support, or application customization, see Bitnami Premium or Tanzu Application Catalog. See https://www.arrow.com/globalecs/na/vendors/bitnami for more information.\n", + "\n", + "** Please be patient while the chart is being deployed **\n", + "\n", + "PostgreSQL can be accessed via port 5432 on the following DNS names from within your cluster:\n", + "\n", + " postgresql.default.svc.cluster.local - Read/Write connection\n", + "\n", + "To get the password for \"postgres\" run:\n", + "\n", + " export POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgresql -o jsonpath=\"{.data.postgres-password}\" | base64 -d)\n", + "\n", + "To connect to your database run the following command:\n", + "\n", + " kubectl run postgresql-client --rm --tty -i --restart='Never' --namespace default --image docker.io/bitnami/postgresql:17.2.0-debian-12-r10 --env=\"PGPASSWORD=$POSTGRES_PASSWORD\" \\\n", + " --command -- psql --host postgresql -U postgres -d postgres -p 5432\n", + "\n", + " > NOTE: If you access the container using bash, make sure that you execute \"/opt/bitnami/scripts/postgresql/entrypoint.sh /bin/bash\" in order to avoid the error \"psql: local user with ID 1001} does not exist\"\n", + "\n", + "To connect to your database from outside the cluster execute the following commands:\n", + "\n", + " kubectl port-forward --namespace default svc/postgresql 5432:5432 &\n", + " PGPASSWORD=\"$POSTGRES_PASSWORD\" psql --host 127.0.0.1 -U postgres -d postgres -p 5432\n", + "\n", + "WARNING: The configured password will be ignored on new installation in case when previous PostgreSQL release was deleted through the helm command. In that case, old PVC will have an old password, and setting it through helm won't take effect. Deleting persistent volumes (PVs) will solve the issue.\n", + "\n", + "WARNING: There are \"resources\" sections in the chart not set. Using \"resourcesPreset\" is not recommended for production. For production installations, please set the following values according to your workload needs:\n", + " - primary.resources\n", + " - readReplicas.resources\n", + " - volumePermissions.resources\n", + "+info https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n" + ] + } + ], + "source": [ + "# Helm values for TLS configuration\n", + "helm_values = \"\"\"\n", + "tls:\n", + " enabled: true\n", + " certificatesSecret: \"postgresql-server-certs\"\n", + " certFilename: \"tls.crt\"\n", + " certKeyFilename: \"tls.key\"\n", + " certCAFilename: \"ca.crt\"\n", + "\n", + "volumePermissions:\n", + " enabled: true\n", + "\"\"\"\n", + "\n", + "# Write the values to a file\n", + "with open(\"values.yaml\", \"w\") as f:\n", + " f.write(helm_values)\n", + "\n", + "# Install PostgreSQL with Helm\n", + "!helm install postgresql bitnami/postgresql -f values.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "5be34ace", + "metadata": {}, + "source": [ + "## Step 6: Verify the Deployment" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "132df785-762e-473a-90d2-5fdb66a59a97", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "postgresql-0 1/1 Running 0 12s\n", + "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", + "ssl = 'on'\n", + "ssl_ca_file = '/opt/bitnami/postgresql/certs/ca.crt'\n", + "ssl_cert_file = '/opt/bitnami/postgresql/certs/tls.crt'\n", + "#ssl_crl_file = ''\n", + "#ssl_crl_dir = ''\n", + "ssl_key_file = '/opt/bitnami/postgresql/certs/tls.key'\n", + "#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL'\t# allowed SSL ciphers\n", + "#ssl_prefer_server_ciphers = on\n", + "#ssl_ecdh_curve = 'prime256v1'\n", + "#ssl_min_protocol_version = 'TLSv1.2'\n", + "#ssl_max_protocol_version = ''\n", + "#ssl_dh_params_file = ''\n", + "#ssl_passphrase_command = ''\n", + "#ssl_passphrase_command_supports_reload = off\n", + "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", + " List of databases\n", + " Name | Owner | Encoding | Locale Provider | Collate | Ctype \n", + "| Locale | ICU Rules | Access privileges \n", + "------------+----------+----------+-----------------+-------------+-------------\n", + "+--------+-----------+-----------------------\n", + " mydatabase | admin | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", + "| | | =Tc/admin +\n", + " | | | | | \n", + "| | | admin=CTc/admin\n", + " postgres | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", + "| | | \n", + " template0 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", + "| | | =c/postgres +\n", + " | | | | | \n", + "| | | postgres=CTc/postgres\n", + " template1 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", + "| | | =c/postgres +\n", + " | | | | | \n", + "| | | postgres=CTc/postgres\n", + "(4 rows)\n", + "\n", + "\n", + "\n", + "\u001b[K(END)\u001b[27m\n", + "\u001b[K(END)\u001b[27m" + ] + } + ], + "source": [ + "# Check the status of the PostgreSQL pod\n", + "!kubectl get pods -l app.kubernetes.io/name=postgresql\n", + "\n", + "# check if the ssl is on and setting the path to certificates.\n", + "!kubectl exec -it postgresql-0 -- cat /opt/bitnami/postgresql/conf/postgresql.conf | grep ssl\n", + "\n", + "# Connect to PostgreSQL using TLS\n", + "!kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c '\\l'" + ] + }, + { + "cell_type": "markdown", + "id": "c921423a-81df-456e-9cca-f689070c44d2", + "metadata": {}, + "source": [ + "## Step 7: Port forwarding in the terminal for the connection testing using python" + ] + }, + { + "cell_type": "markdown", + "id": "88a4a7c1-51c4-4c5a-9472-5cace1c47a1c", + "metadata": {}, + "source": [ + "kubectl port-forward svc/postgresql 5432:5432" + ] + }, + { + "cell_type": "markdown", + "id": "a8777ca3-bf59-4f23-b7d0-60ae8c92d5a5", + "metadata": {}, + "source": [ + "## Step 8: Check the connection using sql alchemy" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5a523f9f-784f-493b-b69d-5a3cb1a830af", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "postgresql+psycopg://admin:password@localhost:5432/mydatabase?sslmode=verify-ca&sslrootcert=postgres-tls-certs/ca.crt&sslcert=postgres-tls-certs/client.crt&sslkey=postgres-tls-certs/client.key\n", + "Connected successfully!\n" + ] + } + ], + "source": [ + "\n", + "\n", + "# Define database connection parameters\n", + "DB_USER = \"admin\"\n", + "DB_PASSWORD = \"password\"\n", + "DB_HOST = \"localhost\"\n", + "DB_PORT = \"5432\"\n", + "DB_NAME = \"mydatabase\"\n", + "\n", + "# TLS Certificate Paths\n", + "SSL_CERT = \"postgres-tls-certs/client.crt\"\n", + "SSL_KEY = \"postgres-tls-certs/client.key\"\n", + "SSL_ROOT_CERT = \"postgres-tls-certs/ca.crt\"\n", + "\n", + "from sqlalchemy import create_engine\n", + "# Create SQLAlchemy connection string\n", + "DATABASE_URL = (\n", + " f\"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}?\"\n", + " f\"sslmode=verify-ca&sslrootcert={SSL_ROOT_CERT}&sslcert={SSL_CERT}&sslkey={SSL_KEY}\"\n", + ")\n", + "\n", + "print(DATABASE_URL)\n", + "\n", + "# Create SQLAlchemy engine\n", + "engine = create_engine(DATABASE_URL)\n", + "\n", + "# Test connection\n", + "try:\n", + " with engine.connect() as connection:\n", + " print(\"Connected successfully!\")\n", + "except Exception as e:\n", + " print(\"Connection failed:\", e)\n" + ] + }, + { + "cell_type": "markdown", + "id": "e5d32266", + "metadata": {}, + "source": [ + "## Step 7: Clean Up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "902cbfe8", + "metadata": {}, + "outputs": [], + "source": [ + "# Uninstall the Helm release\n", + "!helm uninstall postgresql\n", + "\n", + "# Delete the secrets\n", + "!kubectl delete secret postgresql-server-certs\n", + "!kubectl delete secret postgresql-client-certs\n", + "\n", + "# Remove the certificates directory\n", + "!rm -rf postgres-tls-certs" + ] + }, + { + "cell_type": "markdown", + "id": "6947fe4b", + "metadata": {}, + "source": [ + "### Explanation of Key Concepts" + ] + }, + { + "cell_type": "markdown", + "id": "59de375a", + "metadata": {}, + "source": [ + "1. **TLS in PostgreSQL**: TLS ensures secure communication between the PostgreSQL server and clients by encrypting the data.\n", + "2. **Helm**: Helm is a package manager for Kubernetes that simplifies the deployment of applications.\n", + "3. **Kubernetes Secrets**: Secrets are used to store sensitive information like certificates and keys securely." + ] + }, + { + "cell_type": "markdown", + "id": "f9de69cb", + "metadata": {}, + "source": [ + "### Notes" + ] + }, + { + "cell_type": "markdown", + "id": "ca675727", + "metadata": {}, + "source": [ + "- Replace the `CN` values in the certificate generation step with your actual domain names.\n", + "- Ensure the Kubernetes cluster has sufficient resources to run PostgreSQL.\n", + "- For production environments, consider using a managed certificate service (e.g., Let's Encrypt) instead of self-signed certificates." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/operator-postgres-ssl-demo/redis.yaml b/examples/operator-postgres-ssl-demo/redis.yaml new file mode 100644 index 00000000000..a7000d115bc --- /dev/null +++ b/examples/operator-postgres-ssl-demo/redis.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: 'bitnami/redis:latest' + ports: + - containerPort: 6379 + env: + - name: ALLOW_EMPTY_PASSWORD + value: "yes" +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: default + labels: + app: redis +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP + selector: + app: redis \ No newline at end of file diff --git a/examples/operator-postgres-ssl-demo/values.yaml b/examples/operator-postgres-ssl-demo/values.yaml new file mode 100644 index 00000000000..aed87fc7874 --- /dev/null +++ b/examples/operator-postgres-ssl-demo/values.yaml @@ -0,0 +1,9 @@ +tls: + enabled: true + certificatesSecret: "postgresql-server-certs" + certFilename: "tls.crt" + certKeyFilename: "tls.key" + certCAFilename: "ca.crt" + +volumePermissions: + enabled: true From 4b75e78cd10711827bd64fb3fd4787088c754457 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 00:08:36 -0500 Subject: [PATCH 02/13] adding the example of postgres tls setup and deploying feast using postgres tls as well. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/operator-postgres-ssl-demo/README.md | 84 ---- .../operator-postgres-tls-demo/.gitignore | 4 + .../01-Install-postgres-tls-using-helm.ipynb} | 304 ++++++++------ .../02-Install-feast.ipynb | 380 ++++++++++++++++++ .../operator-postgres-tls-demo/03-Demo.ipynb | 237 +++++++++++ .../04-Uninstall.ipynb | 89 ++++ examples/operator-postgres-tls-demo/README.md | 53 +++ ..._featurestore_postgres_db_volumes_ssl.yaml | 10 +- 8 files changed, 955 insertions(+), 206 deletions(-) delete mode 100644 examples/operator-postgres-ssl-demo/README.md create mode 100644 examples/operator-postgres-tls-demo/.gitignore rename examples/{operator-postgres-ssl-demo/postgres-helm-tls.ipynb => operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb} (54%) create mode 100644 examples/operator-postgres-tls-demo/02-Install-feast.ipynb create mode 100644 examples/operator-postgres-tls-demo/03-Demo.ipynb create mode 100644 examples/operator-postgres-tls-demo/04-Uninstall.ipynb create mode 100644 examples/operator-postgres-tls-demo/README.md diff --git a/examples/operator-postgres-ssl-demo/README.md b/examples/operator-postgres-ssl-demo/README.md deleted file mode 100644 index e717a556bed..00000000000 --- a/examples/operator-postgres-ssl-demo/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Install and run a Feature Store on Kubernetes with the Feast Operator - -The following notebooks will guide you through how to install and use Feast on Kubernetes with the Feast Go Operator. - -* [01-Install.ipynb](./01-Install.ipynb): Install and configure a Feature Store in Kubernetes with the Operator. -* [02-Demo.ipynb](./02-Demo.ipynb): Validate the feature store with demo application. -* [03-Uninstall.ipynb](./03-Uninstall.ipynb): Clear the installed deployments. - -# Here are the steps to deploy - -* Generate certificates and create kubernetes secrets. -* helm install postgresql bitnami/postgresql -f values.yaml -* kubectl apply -f feast.yaml - - -# Some of the important commands to debug while setting up the postgres in TLS setup. -helm show chart bitnami/postgresql - -helm install postgresql -f values.yaml bitnami/postgresql --version 16.4.6 - -helm uninstall postgresql - -make test -make docker-build docker-push IMG=quay.io/lrangine/feast-operator:0.53.0 -make install -make deploy IMG=quay.io/lrangine/feast-operator:0.53.0 - -kubectl exec -it postgresql-0 -- env PGPASSWORD=secret psql -U admin -d mydb -c "SHOW ssl;" - -kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c "SHOW ssl;" - - -kubectl exec -it my-postgres-postgresql-0 -- bash - - - -kubectl create secret generic postgres-tls-secret \ ---from-file=tls.crt=postgres-certs/server.crt \ ---from-file=tls.key=postgres-certs/server.key \ ---from-file=ca.crt=postgres-certs/ca.crt - -kubectl create secret generic postgres-tls-certs-new \ ---from-file=tls.crt=postgres-tls-certs-new/server.crt \ ---from-file=tls.key=postgres-tls-certs-new/server.key \ ---from-file=ca.crt=postgres-tls-certs-new/ca.crt - - - -kubectl run -it --rm --image=postgres:latest --restart=Never postgresql-client -- psql "host=postgresql.default.svc.cluster.local dbname=mydatabase user=admin password=mypassword sslmode=require" - - -kubectl exec -it postgresql-client -- psql \ -"host=postgresql.default.svc.cluster.local dbname=mydatabase user=admin password=mypassword sslmode=verify-full sslcert=/etc/ssl/postgres/client.crt sslkey=/etc/ssl/postgres/client.key sslrootcert=/etc/ssl/postgres/ca.crt" - -postgresql+psycopg://admin:password@postgresql.default.svc.cluster.local:5432/mydb?sslmode=require&sslrootcert=postgres-tls-certs/ca.crt&sslcert=postgres-tls-certs/client.crt&sslkey=postgres-tls-certs/client.key - - -kubectl exec -it postgresql-0 -- cat /opt/bitnami/postgresql/conf/postgresql.conf | grep ssl - -sslcert=postgres-tls-certs-new/server.crt&sslkey=postgres-tls-certs-new/server.key - -postgresql+psycopg://admin:password@localhost:5432/mydatabase?sslmode=disable&sslrootcert=postgres-tls-certs-new/ca.crt&sslcert=postgres-tls-certs-new/server.crt&sslkey=postgres-tls-certs-new/server.key - - -kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c '\l' - - -helm install postgresql -f values.yaml bitnami/postgresql --version 16.4.6 - -helm uninstall postgresql - -helm template postgresql bitnami/postgresql --values values.yaml --version 16.4.6 > postgresql-export.yaml - - -kubectl create secret generic postgresql-client-certs \ ---from-file=ca.crt=./postgres-tls-certs/ca.crt \ ---from-file=tls.crt=./postgres-tls-certs/client.crt \ ---from-file=tls.key=./postgres-tls-certs/client.key \ ---dry-run=client -o yaml | kubectl apply -f - - - -!kubectl exec deploy/postgres -- psql -h localhost -U feast feast -c '\dt' - -kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c '\dt' diff --git a/examples/operator-postgres-tls-demo/.gitignore b/examples/operator-postgres-tls-demo/.gitignore new file mode 100644 index 00000000000..debe8880d52 --- /dev/null +++ b/examples/operator-postgres-tls-demo/.gitignore @@ -0,0 +1,4 @@ +postgres-tls-certs +values.yaml +.ipynb_checkpoints +f43b44b.tar.gz \ No newline at end of file diff --git a/examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb similarity index 54% rename from examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb rename to examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb index 43df716b656..27f3aff77b1 100644 --- a/examples/operator-postgres-ssl-demo/postgres-helm-tls.ipynb +++ b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb @@ -8,6 +8,14 @@ "# Deploy PostgreSQL with Helm in TLS Mode" ] }, + { + "cell_type": "markdown", + "id": "1247e2e7-706c-44a3-a45c-fba638e50f31", + "metadata": {}, + "source": [ + "### NOTE: This PostgreSQL setup guide is intended to demonstrate the capabilities of the Feast operator in configuring Feast with PostgreSQL in TLS mode. For ongoing assistance with Postgres setup, we recommend consulting the official Helm PostgreSQL documentation." + ] + }, { "cell_type": "markdown", "id": "cce2278a", @@ -27,6 +35,14 @@ "- A Kubernetes cluster (e.g., Minikube, GKE, EKS, or AKS)" ] }, + { + "cell_type": "markdown", + "id": "44b611ba-097e-4777-b77b-739116e7e4d6", + "metadata": {}, + "source": [ + "**Note:** When deploying PostgreSQL and Feast on a Kubernetes cluster, it's important to ensure that your cluster has sufficient resources to support both applications." + ] + }, { "cell_type": "code", "execution_count": 1, @@ -80,17 +96,82 @@ "!helm repo update" ] }, + { + "cell_type": "markdown", + "id": "6f51e5c8-41ba-417e-a2fc-78cf5951d9dc", + "metadata": {}, + "source": [ + "## Step 3: create kubernetes feast namespace" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d114872a-7a43-4eca-8748-6dc7346dc176", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast created\n", + "Context \"kind-kind\" modified.\n" + ] + } + ], + "source": [ + "!kubectl create ns feast\n", + "!kubectl config set-context --current --namespace feast" + ] + }, { "cell_type": "markdown", "id": "41f4e8db", "metadata": {}, "source": [ - "## Step 3: Generate TLS Certificates" + "## Step 3: Generate Self Signed TLS Certificates" + ] + }, + { + "cell_type": "markdown", + "id": "c34957e4-dd7f-49c1-986c-eefe74dd7e22", + "metadata": {}, + "source": [ + "**Note**: \n", + "- Self signed certificates are used only for demo purpose, consider using a managed certificate service (e.g., Let's Encrypt) instead of self-signed certificates.\n", + "- \"Replace the `CN` values in the certificate generation step with your actual domain names.\"," + ] + }, + { + "cell_type": "markdown", + "id": "500f9010-6329-4868-83d5-9c063d5890f5", + "metadata": {}, + "source": [ + "Delete the directory of existing certificates if you running this demo not first time." ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 4, + "id": "bdc71e19-0fcc-4a1f-ba94-8b5e427e45d9", + "metadata": {}, + "outputs": [], + "source": [ + "# Delete certificates directory if you are running this example not first time.\n", + "!rm -rf postgres-tls-certs" + ] + }, + { + "cell_type": "markdown", + "id": "91dc26c9-cfaa-46f5-8252-7ad463264236", + "metadata": {}, + "source": [ + "Generate the certificates by executing below scripts. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "8e192410", "metadata": {}, "outputs": [ @@ -98,16 +179,16 @@ "name": "stdout", "output_type": "stream", "text": [ - ".............+.+..+............+...+...................+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+.........+....+.....+.+..+.......+..+...+.+.....+......+.+..+.+...........+.+.........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+...........+...+......................+........+......+....+..+.............+...+............+........+....+........+...+...+......+.+..............+.........+.+..............+....+..+.............+..+...+.+......+.....+...+.......+..................+..+....+.....+.+.....+.+...+.........+...+.....+....+.........+.....+.......+..+............+.........+.+......+...+............+..+............+....+..+...+.+...............+...+..+....+...........+..........+.....+....+..+...+....+........+...+.+......+........+.+......+..+...+....+...+..+.+........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", - "..............+.+...+..................+..+...+..........+............+...............+......+......+.........+.....+...................+..+...............+...+......+.+...+..+.+.....+....+...+..+.+..................+...........+...+......+.+.........+...........+....+.....+.......+..+......+.....................+....+..+............+...+...............+....+...+...+...+......+...+...........+.........+...+...+.+...........+.......+......+..+...+....+..+...+.+..+............+.............+.....+.............+............+..+......+.+...+.....+......+............+...+.......+.....+.......+............+.........+..+....+......+........+.......+............+.....+.......+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "..+.............+.....+.+.........+..+.........+....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+..........+...........+....+...+........+.+.....+.+...+.........+..+......+...+......+....+..+...+......+.+...+.........+............+.....................+........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*........+.......+........+......+.......+........+.+..............+..........+......+.........+............+..+......+.......+..................+..+...+.......+.........+...+...........+....+..+......+....+........+.+.....+.+...+........+...+.........+.............+..+.+.................+....+.....+.+...+..+.......+..+....+......+.....+.........+.+..................+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + ".......+...+...+.......+.........+.....+....+..+.........+...................+..............+.......+.....+...+...+....+.........+.....+......+....+......+.....+.+.......................+.+.........+..+..........+.........+........+......+.......+...+....................+.+.....+...+.+.........+...+.....+.......+..+....+.........+...+...+..+....+...+........+...+..........+.........+..+...+......+.+.....+......+....+........+..........+...+.................+.+...........+....+..+.............+.........+...........+.......+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", "-----\n", - ".........+..........+.....+....+..+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+...+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.......+........+......+......+....+...+...........+.......+...+.........+...+...+......+.....+.+..+.+.................+...+.+.........+.....+....+..+...+...+....+........+......+.+........+..........+...+..+......+......+....+............+..+.............+............+.....+.......+.........+..+..........+.....+....+...+..+................+..+......+....+..+..........+..................+..+.+.....+.........................+..+......+......+....+.....+.+.....+.+..+............+...+..........+...+........+..........+..+.......+..+...+.............+..................+.....+.+.....+.........+.+...........+....+.....+....+.....+....+..+...+......................+......+...+......+...+..+.+........+.........+.............+.....+.........+.+.....+.......+.....+...+.......+..+..........+...+...+.....+.........+.+.......................+.......+.........+.....+..................+.+......+...+...........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", - "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+....+.................+....+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+....+.....+.+...+...+.........+......+...............+.....+...............+...+....+...............+..+..........++++++++++++++++++++++++\n", + "......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.+.........+...............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+...+.+.........+.........+.....+...............+.......+...+......+.....+..........+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+.....+....+.....+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*............+...+...+....+..+...+...+...+.+++++++++++++++++++++++++++++++++++\n", "-----\n", "Certificate request self-signature ok\n", "subject=CN = postgresql.default.svc.cluster.local\n", - "........+.........+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+..+.+..+....+...+..+.........+............+...+.+...+........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.......+..+.......+.....+.............+..+.+...+.................+...............+...+.+.....+.....................+.......+..+.+.........+......+...+..+.......+......+..............+.......+......+........+......+.+...+...........+.+..+......+......+...+......+..........+..+...............+............+.+.........+...+.....+.........+.+...+...............+..+.......+...+........+.+.....+......+.........+...+.+..+....+...+..+...+................+..+...+......+.+..+...............+......+.+..................+...+.....+..........+.................+.+.........+...............+......+.................+....+...........+....+.....+......+.......+..+...............+......+....+......+..+.+.....+...............+.......+.....+..........+........+......+....+.....+......+..........+........+.+..+....+......+........+.+...............+.........+...........+...+...+...+.......+...+.....+....+...+..+...+....+...+............+...+.....+...+...+......+...+.+.....+............+....+.....+......+.........+....+.........+..+...+.+..+....+.....+....+.........+...+.......................+............+....+...+...........+...............+....+..+.+..+.......+...+..+...+......+.+.......................+...............+...+...+....+..+...+.........................+......+..+....+.........+.....+.+.........+......+......+...............+...+...........+.+..+...+.........+....+.................+.......+..+....+......+...+...+.....+..........+........+.......+..+.......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", - "...........+.+.....+.......+..+.......+......+..+.............+........+....+...+.....+...++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*............+.....+...+............+............+...+.+......+.........+..+....+......+.....+.............+.....+......+...+............+....+......+......+..+...+..................+......+...+............+.+.....+.......+........+...+....+............+.....+.........+.......+...+...........+...+......+.+........+...+....+..................+........+.........+....+...\n", + ".+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+..+...+.+..+...+.........+......+.+..............+.+..............+.+......+.....+.........+.+.....+...+....+..+...+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "+....+...+..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+.....+....+..+....+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+....+..+..........+.....+....+..............+...\n", "-----\n", "Certificate request self-signature ok\n", "subject=CN = admin\n" @@ -138,6 +219,20 @@ "## Step 4: Create Kubernetes Secrets for Certificates" ] }, + { + "cell_type": "markdown", + "id": "a4775780-3734-40ba-ae43-48f1e47b481a", + "metadata": {}, + "source": [ + "In this step, we will create **two Kubernetes secrets** that reference the certificates generated earlier step:\n", + "\n", + "- **`postgresql-server-certs`** \n", + " This secret contains the server certificates and will be used by the PostgreSQL server.\n", + "\n", + "- **`postgresql-client-certs`** \n", + " This secret contains the client certificates and will be used by the PostgreSQL client. In our case it will be feast application." + ] + }, { "cell_type": "code", "execution_count": 6, @@ -148,7 +243,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "error: failed to create secret secrets \"postgresql-server-certs\" already exists\n", + "secret/postgresql-server-certs created\n", "secret/postgresql-client-certs created\n" ] } @@ -171,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "id": "e14cae77", "metadata": {}, "outputs": [ @@ -180,15 +275,15 @@ "output_type": "stream", "text": [ "NAME: postgresql\n", - "LAST DEPLOYED: Thu Feb 6 12:36:33 2025\n", - "NAMESPACE: default\n", + "LAST DEPLOYED: Sun Feb 16 22:18:39 2025\n", + "NAMESPACE: feast\n", "STATUS: deployed\n", "REVISION: 1\n", "TEST SUITE: None\n", "NOTES:\n", "CHART NAME: postgresql\n", - "CHART VERSION: 16.4.6\n", - "APP VERSION: 17.2.0\n", + "CHART VERSION: 16.4.9\n", + "APP VERSION: 17.3.0\n", "\n", "Did you know there are enterprise versions of the Bitnami catalog? For enhanced secure software supply chain features, unlimited pulls from Docker, LTS support, or application customization, see Bitnami Premium or Tanzu Application Catalog. See https://www.arrow.com/globalecs/na/vendors/bitnami for more information.\n", "\n", @@ -196,23 +291,27 @@ "\n", "PostgreSQL can be accessed via port 5432 on the following DNS names from within your cluster:\n", "\n", - " postgresql.default.svc.cluster.local - Read/Write connection\n", + " postgresql.feast.svc.cluster.local - Read/Write connection\n", "\n", "To get the password for \"postgres\" run:\n", "\n", - " export POSTGRES_PASSWORD=$(kubectl get secret --namespace default postgresql -o jsonpath=\"{.data.postgres-password}\" | base64 -d)\n", + " export POSTGRES_ADMIN_PASSWORD=$(kubectl get secret --namespace feast postgresql -o jsonpath=\"{.data.postgres-password}\" | base64 -d)\n", + "\n", + "To get the password for \"admin\" run:\n", + "\n", + " export POSTGRES_PASSWORD=$(kubectl get secret --namespace feast postgresql -o jsonpath=\"{.data.password}\" | base64 -d)\n", "\n", "To connect to your database run the following command:\n", "\n", - " kubectl run postgresql-client --rm --tty -i --restart='Never' --namespace default --image docker.io/bitnami/postgresql:17.2.0-debian-12-r10 --env=\"PGPASSWORD=$POSTGRES_PASSWORD\" \\\n", - " --command -- psql --host postgresql -U postgres -d postgres -p 5432\n", + " kubectl run postgresql-client --rm --tty -i --restart='Never' --namespace feast --image docker.io/bitnami/postgresql:17.3.0-debian-12-r1 --env=\"PGPASSWORD=$POSTGRES_PASSWORD\" \\\n", + " --command -- psql --host postgresql -U admin -d feast -p 5432\n", "\n", " > NOTE: If you access the container using bash, make sure that you execute \"/opt/bitnami/scripts/postgresql/entrypoint.sh /bin/bash\" in order to avoid the error \"psql: local user with ID 1001} does not exist\"\n", "\n", "To connect to your database from outside the cluster execute the following commands:\n", "\n", - " kubectl port-forward --namespace default svc/postgresql 5432:5432 &\n", - " PGPASSWORD=\"$POSTGRES_PASSWORD\" psql --host 127.0.0.1 -U postgres -d postgres -p 5432\n", + " kubectl port-forward --namespace feast svc/postgresql 5432:5432 &\n", + " PGPASSWORD=\"$POSTGRES_PASSWORD\" psql --host 127.0.0.1 -U admin -d feast -p 5432\n", "\n", "WARNING: The configured password will be ignored on new installation in case when previous PostgreSQL release was deleted through the helm command. In that case, old PVC will have an old password, and setting it through helm won't take effect. Deleting persistent volumes (PVs) will solve the issue.\n", "\n", @@ -236,6 +335,15 @@ "\n", "volumePermissions:\n", " enabled: true\n", + "\n", + "# Set fixed PostgreSQL credentials\n", + "\n", + "global:\n", + " postgresql:\n", + " auth:\n", + " username: admin\n", + " password: password\n", + " database: feast\n", "\"\"\"\n", "\n", "# Write the values to a file\n", @@ -243,7 +351,7 @@ " f.write(helm_values)\n", "\n", "# Install PostgreSQL with Helm\n", - "!helm install postgresql bitnami/postgresql -f values.yaml" + "!helm install postgresql bitnami/postgresql -f values.yaml -n feast" ] }, { @@ -251,12 +359,12 @@ "id": "5be34ace", "metadata": {}, "source": [ - "## Step 6: Verify the Deployment" + "## Step 6: Verify the postgres Deployment" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "id": "132df785-762e-473a-90d2-5fdb66a59a97", "metadata": {}, "outputs": [ @@ -264,8 +372,11 @@ "name": "stdout", "output_type": "stream", "text": [ + "pod/postgresql-0 condition met\n", + "\n", "NAME READY STATUS RESTARTS AGE\n", - "postgresql-0 1/1 Running 0 12s\n", + "postgresql-0 1/1 Running 0 25s\n", + "\n", "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", "ssl = 'on'\n", "ssl_ca_file = '/opt/bitnami/postgresql/certs/ca.crt'\n", @@ -281,44 +392,44 @@ "#ssl_dh_params_file = ''\n", "#ssl_passphrase_command = ''\n", "#ssl_passphrase_command_supports_reload = off\n", + "\n", "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", - " List of databases\n", - " Name | Owner | Encoding | Locale Provider | Collate | Ctype \n", - "| Locale | ICU Rules | Access privileges \n", - "------------+----------+----------+-----------------+-------------+-------------\n", - "+--------+-----------+-----------------------\n", - " mydatabase | admin | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", - "| | | =Tc/admin +\n", - " | | | | | \n", - "| | | admin=CTc/admin\n", - " postgres | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", - "| | | \n", - " template0 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", - "| | | =c/postgres +\n", - " | | | | | \n", - "| | | postgres=CTc/postgres\n", - " template1 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 \n", - "| | | =c/postgres +\n", - " | | | | | \n", - "| | | postgres=CTc/postgres\n", + " List of databases\n", + " Name | Owner | Encoding | Locale Provider | Collate | Ctype | Locale | ICU Rules | Access privileges \n", + "-----------+----------+----------+-----------------+-------------+-------------+--------+-----------+-----------------------\n", + " feast | admin | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =Tc/admin +\n", + " | | | | | | | | admin=CTc/admin\n", + " postgres | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | \n", + " template0 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =c/postgres +\n", + " | | | | | | | | postgres=CTc/postgres\n", + " template1 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =c/postgres +\n", + " | | | | | | | | postgres=CTc/postgres\n", "(4 rows)\n", - "\n", - "\n", - "\n", - "\u001b[K(END)\u001b[27m\n", - "\u001b[K(END)\u001b[27m" + "\n" ] } ], "source": [ - "# Check the status of the PostgreSQL pod\n", + "# Wait for the status of the PostgreSQL pod to be in Ready status.\n", + "!kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=postgresql --timeout=60s\n", + "\n", + "# Insert an empty line in the output for verbocity.\n", + "print()\n", + "\n", + "# display the pod status.\n", "!kubectl get pods -l app.kubernetes.io/name=postgresql\n", "\n", - "# check if the ssl is on and setting the path to certificates.\n", - "!kubectl exec -it postgresql-0 -- cat /opt/bitnami/postgresql/conf/postgresql.conf | grep ssl\n", + "# Insert an empty line in the output for verbocity.\n", + "print()\n", "\n", - "# Connect to PostgreSQL using TLS\n", - "!kubectl exec -it postgresql-0 -- env PGPASSWORD=password psql -U admin -d mydatabase -c '\\l'" + "# check if the ssl is on and the path to certificates is configured.\n", + "!kubectl exec postgresql-0 -- cat /opt/bitnami/postgresql/conf/postgresql.conf | grep ssl\n", + "\n", + "# Insert an empty line in the output for verbocity.\n", + "print()\n", + "\n", + "# Connect to PostgreSQL using TLS (non-interactive mode)\n", + "!kubectl exec postgresql-0 -- env PGPASSWORD=password psql -U admin -d feast -c '\\l'\n" ] }, { @@ -329,6 +440,23 @@ "## Step 7: Port forwarding in the terminal for the connection testing using python" ] }, + { + "cell_type": "markdown", + "id": "d6a26bb4-e0e7-419e-9c91-f0d63db127bc", + "metadata": {}, + "source": [ + "**Note:** If you do not intend to test the PostgreSQL connection from outside the Kubernetes cluster, you can skip the remaining steps." + ] + }, + { + "cell_type": "markdown", + "id": "6fcad5e1-66d2-4353-aba7-3549ef21bc9f", + "metadata": {}, + "source": [ + "**Note:**\n", + "To test a connection to a PostgreSQL database outside of your Kubernetes cluster, you'll need to execute the following command in your system's terminal window. This is necessary because Jupyter Notebook does not support running commands in a separate thread." + ] + }, { "cell_type": "markdown", "id": "88a4a7c1-51c4-4c5a-9472-5cace1c47a1c", @@ -342,12 +470,12 @@ "id": "a8777ca3-bf59-4f23-b7d0-60ae8c92d5a5", "metadata": {}, "source": [ - "## Step 8: Check the connection using sql alchemy" + "## Step 8: Check the connection using Python sql alchemy" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 9, "id": "5a523f9f-784f-493b-b69d-5a3cb1a830af", "metadata": {}, "outputs": [ @@ -355,20 +483,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "postgresql+psycopg://admin:password@localhost:5432/mydatabase?sslmode=verify-ca&sslrootcert=postgres-tls-certs/ca.crt&sslcert=postgres-tls-certs/client.crt&sslkey=postgres-tls-certs/client.key\n", + "postgresql+psycopg://admin:password@localhost:5432/feast?sslmode=verify-ca&sslrootcert=postgres-tls-certs/ca.crt&sslcert=postgres-tls-certs/client.crt&sslkey=postgres-tls-certs/client.key\n", "Connected successfully!\n" ] } ], "source": [ - "\n", - "\n", "# Define database connection parameters\n", "DB_USER = \"admin\"\n", "DB_PASSWORD = \"password\"\n", "DB_HOST = \"localhost\"\n", "DB_PORT = \"5432\"\n", - "DB_NAME = \"mydatabase\"\n", + "DB_NAME = \"feast\"\n", "\n", "# TLS Certificate Paths\n", "SSL_CERT = \"postgres-tls-certs/client.crt\"\n", @@ -392,70 +518,16 @@ " with engine.connect() as connection:\n", " print(\"Connected successfully!\")\n", "except Exception as e:\n", - " print(\"Connection failed:\", e)\n" - ] - }, - { - "cell_type": "markdown", - "id": "e5d32266", - "metadata": {}, - "source": [ - "## Step 7: Clean Up" + " print(\"Connection failed: Make sure that port forwarding step is done in the terminal.\", e)\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "902cbfe8", + "id": "7503e47e-12f1-44dd-8a50-786d744bbf4c", "metadata": {}, "outputs": [], - "source": [ - "# Uninstall the Helm release\n", - "!helm uninstall postgresql\n", - "\n", - "# Delete the secrets\n", - "!kubectl delete secret postgresql-server-certs\n", - "!kubectl delete secret postgresql-client-certs\n", - "\n", - "# Remove the certificates directory\n", - "!rm -rf postgres-tls-certs" - ] - }, - { - "cell_type": "markdown", - "id": "6947fe4b", - "metadata": {}, - "source": [ - "### Explanation of Key Concepts" - ] - }, - { - "cell_type": "markdown", - "id": "59de375a", - "metadata": {}, - "source": [ - "1. **TLS in PostgreSQL**: TLS ensures secure communication between the PostgreSQL server and clients by encrypting the data.\n", - "2. **Helm**: Helm is a package manager for Kubernetes that simplifies the deployment of applications.\n", - "3. **Kubernetes Secrets**: Secrets are used to store sensitive information like certificates and keys securely." - ] - }, - { - "cell_type": "markdown", - "id": "f9de69cb", - "metadata": {}, - "source": [ - "### Notes" - ] - }, - { - "cell_type": "markdown", - "id": "ca675727", - "metadata": {}, - "source": [ - "- Replace the `CN` values in the certificate generation step with your actual domain names.\n", - "- Ensure the Kubernetes cluster has sufficient resources to run PostgreSQL.\n", - "- For production environments, consider using a managed certificate service (e.g., Let's Encrypt) instead of self-signed certificates." - ] + "source": [] } ], "metadata": { diff --git a/examples/operator-postgres-tls-demo/02-Install-feast.ipynb b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb new file mode 100644 index 00000000000..2f8e46dfde5 --- /dev/null +++ b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb @@ -0,0 +1,380 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install Feast on Kubernetes with the Feast Operator\n", + "## Objective\n", + "\n", + "Provide a reference implementation of a runbook to deploy a Feast environment on a Kubernetes cluster using [Kind](https://kind.sigs.k8s.io/docs/user/quick-start) and the [Feast Operator](../../infra/feast-operator/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "* Kubernetes Cluster\n", + "* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) Kubernetes CLI tool." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast Operator" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast-operator-system created\n", + "customresourcedefinition.apiextensions.k8s.io/featurestores.feast.dev created\n", + "serviceaccount/feast-operator-controller-manager created\n", + "role.rbac.authorization.k8s.io/feast-operator-leader-election-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-editor-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-viewer-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-manager-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-auth-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-reader created\n", + "rolebinding.rbac.authorization.k8s.io/feast-operator-leader-election-rolebinding created\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-manager-rolebinding created\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-metrics-auth-rolebinding created\n", + "service/feast-operator-controller-manager-metrics-service created\n", + "deployment.apps/feast-operator-controller-manager created\n", + "deployment.apps/feast-operator-controller-manager condition met\n" + ] + } + ], + "source": [ + "## Use this install command from a release branch (e.g. 'v0.46-branch')\n", + "!kubectl apply -f ../../infra/feast-operator/dist/install.yaml\n", + "\n", + "## OR, for the latest code/builds, use one the following commands from the 'master' branch\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:develop FS_IMG=quay.io/feastdev-ci/feature-server:develop\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:$(git rev-parse HEAD) FS_IMG=quay.io/feastdev-ci/feature-server:$(git rev-parse HEAD)\n", + "\n", + "!kubectl wait --for=condition=available --timeout=5m deployment/feast-operator-controller-manager -n feast-operator-system" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast services via FeatureStore CR\n", + "Next, we'll use the running Feast Operator to install the feast services. Apply the included [reference deployment](feast.yaml) to install and configure Feast." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgres-secret created\n", + "secret/feast-data-stores created\n", + "featurestore.feast.dev/sample-db-ssl created\n" + ] + } + ], + "source": [ + "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml --namespace=feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validate the running FeatureStore deployment\n", + "Validate the deployment status." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment.apps/feast-sample-db-ssl condition met\n", + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-sample-db-ssl-f559fcc9b-kzgjp 1/1 Running 0 100s\n", + "pod/postgresql-0 1/1 Running 0 6m15s\n", + "\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "service/feast-sample-db-ssl-online ClusterIP 10.96.87.130 80/TCP 100s\n", + "service/postgresql ClusterIP 10.96.165.212 5432/TCP 6m15s\n", + "service/postgresql-hl ClusterIP None 5432/TCP 6m15s\n", + "\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "deployment.apps/feast-sample-db-ssl 1/1 1 1 100s\n", + "\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/feast-sample-db-ssl-f559fcc9b 1 1 1 100s\n", + "\n", + "NAME READY AGE\n", + "statefulset.apps/postgresql 1/1 6m15s\n" + ] + } + ], + "source": [ + "!kubectl wait --for=condition=available --timeout=8m deployment/feast-sample-db-ssl -n feast\n", + "!kubectl get all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Validate that the FeatureStore CR is in a `Ready` state." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\n", + "sample-db-ssl Ready 2m9s\n" + ] + } + ], + "source": [ + "!kubectl get feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify that the DB includes the expected tables." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", + " List of relations\n", + " Schema | Name | Type | Owner \n", + "--------+-------------------------+-------+-------\n", + " public | data_sources | table | admin\n", + " public | entities | table | admin\n", + " public | feast_metadata | table | admin\n", + " public | feature_services | table | admin\n", + " public | feature_views | table | admin\n", + " public | managed_infra | table | admin\n", + " public | on_demand_feature_views | table | admin\n", + " public | permissions | table | admin\n", + " public | projects | table | admin\n", + " public | saved_datasets | table | admin\n", + " public | stream_feature_views | table | admin\n", + " public | validation_references | table | admin\n", + "(12 rows)\n", + "\n" + ] + } + ], + "source": [ + "!kubectl exec postgresql-0 -- env PGPASSWORD=password psql -U admin -d feast -c '\\dt'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify the client `feature_store.yaml` and create the sample feature store definitions." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: postgres_tls_sample\n", + "provider: local\n", + "offline_store:\n", + " host: ${POSTGRES_HOST}\n", + " type: postgres\n", + " port: 5432\n", + " database: ${POSTGRES_DB}\n", + " db_schema: public\n", + " password: ${POSTGRES_PASSWORD}\n", + " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", + " sslkey_path: /var/lib/postgresql/certs/tls.key\n", + " sslmode: require\n", + " sslrootcert_path: /var/lib/postgresql/certs/ca.crt\n", + " user: ${POSTGRES_USER}\n", + "online_store:\n", + " type: postgres\n", + " database: ${POSTGRES_DB}\n", + " db_schema: public\n", + " host: ${POSTGRES_HOST}\n", + " password: ${POSTGRES_PASSWORD}\n", + " port: 5432\n", + " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", + " sslkey_path: /var/lib/postgresql/certs/tls.key\n", + " sslmode: require\n", + " sslrootcert_path: /var/lib/postgresql/certs/ca.crt\n", + " user: ${POSTGRES_USER}\n", + "registry:\n", + " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key\n", + " registry_type: sql\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "auth:\n", + " type: no_auth\n", + "entity_key_serialization_version: 3\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/feast-data/postgres_tls_sample/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\n", + "Applying changes for project postgres_tls_sample\n", + "/opt/app-root/src/sdk/python/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", + " warnings.warn(\n", + "Deploying infrastructure for driver_hourly_stats_fresh\n", + "Deploying infrastructure for driver_hourly_stats\n", + " Feast apply is completed. You can go to next step.\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -c online -- cat feature_store.yaml\n", + "!kubectl exec deploy/feast-sample-db-ssl -c online -- feast apply\n", + "print(\" Feast apply is completed. You can go to next step.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List the registered feast projects & feature views." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "NAME DESCRIPTION TAGS OWNER\n", + "postgres_tls_sample A project for driver statistics {}\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "NAME ENTITIES TYPE\n", + "driver_hourly_stats_fresh {'driver'} FeatureView\n", + "driver_hourly_stats {'driver'} FeatureView\n", + "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n", + "transformed_conv_rate {'driver'} OnDemandFeatureView\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -c online -- feast projects list\n", + "!kubectl exec deploy/feast-sample-db-ssl -c online -- feast feature-views list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let's verify the feast version." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "Feast SDK Version: \"0.1.dev1+g6c92447.d20250213\"\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/feast-sample-db-ssl -c online -- feast version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-postgres-tls-demo/03-Demo.ipynb b/examples/operator-postgres-tls-demo/03-Demo.ipynb new file mode 100644 index 00000000000..e7de3ec190a --- /dev/null +++ b/examples/operator-postgres-tls-demo/03-Demo.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run the \"Real-time Credit Scoring\" tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll use the following tutorial as a demonstration.\n", + "\n", + "https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/f43b44b245ae2632b582f14176392cfe31f98da9" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upload the tutorial source code to the running Feast pod." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Upload the tutorial source code to the running feast pod and extract its contents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "![ -f f43b44b.tar.gz ] || wget https://github.com/feast-dev/feast-credit-score-local-tutorial/archive/f43b44b.tar.gz\n", + "!kubectl cp f43b44b.tar.gz $(kubectl get pods -l 'feast.dev/name=sample-db-ssl' -ojsonpath=\"{.items[*].metadata.name}\"):/feast-data -c online\n", + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- rm -rf /feast-data/feast-credit-score-local-tutorial\n", + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- mkdir /feast-data/feast-credit-score-local-tutorial\n", + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- tar vfx /feast-data/f43b44b.tar.gz -C /feast-data/feast-credit-score-local-tutorial --strip-components 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify the client `feature_store.yaml`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copy the `feature_store.yaml` to the tutorial directory and verify its contents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- cp -f /feast-data/postgres_tls_sample/feature_repo/feature_store.yaml /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml\n", + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- cat /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Apply the tutorial feature store definitions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Update the feature store definitions for the tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo apply" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load data from feature views into the online store, beginning from either the previous materialize or materialize-incremental end date, or the beginning of time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial/feature_repo && feast materialize-incremental $(date -u +\"%Y-%m-%dT%H:%M:%S\")'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Execute feast commands inside the client Pod" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List the registered feast feature views & entities." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo feature-views list\n", + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo entities list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train and test the model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install the required packages." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'pip install -r /feast-data/feast-credit-score-local-tutorial/requirements.txt'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Train and test the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && python run.py'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interactive demo (using Streamlit)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In a new terminal, run the following command and leave it active.\n", + "\n", + "```bash\n", + "$ kubectl port-forward deploy/feast-sample-db-ssl 8501:8501\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start the Streamlit application" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && streamlit run --server.port 8501 streamlit_app.py'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then navigate to the local URL on which Streamlit is being served.\n", + "\n", + "http://localhost:8501" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-postgres-tls-demo/04-Uninstall.ipynb b/examples/operator-postgres-tls-demo/04-Uninstall.ipynb new file mode 100644 index 00000000000..e7bb3b46152 --- /dev/null +++ b/examples/operator-postgres-tls-demo/04-Uninstall.ipynb @@ -0,0 +1,89 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Uninstall the Operator and all Feast related objects" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml\n", + "# !kubectl delete -f ../../infra/feast-operator/dist/install.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Uninstall the Postgresql using helm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Uninstall the Helm release\n", + "!helm uninstall postgresql\n", + "\n", + "# Delete the secrets\n", + "!kubectl delete secret postgresql-server-certs\n", + "!kubectl delete secret postgresql-client-certs\n", + "\n", + "# Remove the certificates directory\n", + "!rm -rf postgres-tls-certs\n", + "\n", + "# Remove PV and PVC for clean up. some times those are not deleted automatically and can cause issues.\n", + "# Delete all PVCs in the default namespace\n", + "!kubectl delete pvc --all\n", + "\n", + "# Delete all PVs\n", + "!kubectl delete pv --all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ensure everything has been removed, or is in the process of being terminated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl get all" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-postgres-tls-demo/README.md b/examples/operator-postgres-tls-demo/README.md new file mode 100644 index 00000000000..30dee7bb599 --- /dev/null +++ b/examples/operator-postgres-tls-demo/README.md @@ -0,0 +1,53 @@ +# Installing Feast on Kubernetes with PostgreSQL TLS Demo using feast operator + +This repository contains a series of Jupyter Notebooks that guide you through setting up [Feast](https://feast.dev/) on a Kubernetes cluster. + +In this demo, Feast connects to a PostgreSQL database running in TLS mode, ensuring secure communication between services. Additionally, the example demonstrates how feast application references TLS certificates using Kubernetes volumes and volume mounts. While the focus is on mounting TLS certificates, you can also mount any other resources supported by Kubernetes volumes. + +## Prerequisites + +- A running Kubernetes cluster with sufficient resources. +- [Helm](https://helm.sh/) installed and configured. +- The [Feast Operator](https://docs.feast.dev/) for managing Feast deployments. +- Jupyter Notebook or JupyterLab to run the provided notebooks. +- Basic familiarity with Kubernetes, Helm, and TLS concepts. + +## Notebook Overview + +The following Jupyter Notebooks will walk you through the entire process: + +1. **[01-Install-postgres-tls-using-helm.ipynb](./01-Install-postgres-tls-using-helm.ipynb)** + Installs PostgreSQL in TLS mode using a Helm chart. + +2. **[02-Install-feast.ipynb](02-Install-feast.ipynb)** + Deploys Feast using the Feast Operator. + +3. **[03-demo-feast.ipynb](./03-Demo.ipynb)** + Demonstrates the functionality of Feast by executing [feast-credit-score-local-tutorial](https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/f43b44b245ae2632b582f14176392cfe31f98da9) the tutorial. + +4. **[04-Uninstall.ipynb](./04-Uninstall.ipynb)** + Uninstalls Feast, the Feast Operator, and the PostgreSQL deployments set up in this demo. + +## How to Run the Demo + +1. **Clone the Repository** + + ```shell + https://github.com/feast-dev/feast.git + cd examples/operator-postgres-ssl-demo + ``` +2. Start Jupyter Notebook or JupyterLab from the repository root: + +```shell +jupyter notebook +``` +3. Execute the Notebooks +Run the notebooks in the order listed above. Each notebook contains step-by-step instructions and code to deploy, test, and eventually clean up the demo components. + + +## Troubleshooting +* **Cluster Resources:** +Verify that your Kubernetes cluster has adequate resources before starting the demo. + +* **Logs & Diagnostics:** +If you encounter issues, check the logs for the PostgreSQL and Feast pods. This can help identify problems related to TLS configurations or resource constraints. \ No newline at end of file diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml index 5988a5e942c..33238755cc6 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml @@ -2,28 +2,27 @@ apiVersion: v1 kind: Secret metadata: name: postgres-secret - namespace: default labels: app: postgres stringData: - POSTGRES_DB: mydatabase + POSTGRES_DB: feast POSTGRES_USER: admin POSTGRES_PASSWORD: password + POSTGRES_HOST: postgresql.feast.svc.cluster.local --- apiVersion: v1 kind: Secret metadata: name: feast-data-stores - namespace: default stringData: sql: | - path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgresql.default.svc.cluster.local:5432/${POSTGRES_DB}?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key cache_ttl_seconds: 60 sqlalchemy_config_kwargs: echo: false pool_pre_ping: true postgres: | - host: postgresql.default.svc.cluster.local + host: ${POSTGRES_HOST} port: 5432 database: ${POSTGRES_DB} db_schema: public @@ -38,7 +37,6 @@ apiVersion: feast.dev/v1alpha1 kind: FeatureStore metadata: name: sample-db-ssl - namespace: default spec: feastProject: postgres_tls_sample services: From fe47fd23db14183a8f9ba16f1a7446418017b5f8 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 00:11:59 -0500 Subject: [PATCH 03/13] deleting the files which are not needed. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../operator-postgres-ssl-demo/feast.yaml | 137 ------------------ .../operator-postgres-ssl-demo/redis.yaml | 39 ----- .../operator-postgres-ssl-demo/values.yaml | 9 -- 3 files changed, 185 deletions(-) delete mode 100644 examples/operator-postgres-ssl-demo/feast.yaml delete mode 100644 examples/operator-postgres-ssl-demo/redis.yaml delete mode 100644 examples/operator-postgres-ssl-demo/values.yaml diff --git a/examples/operator-postgres-ssl-demo/feast.yaml b/examples/operator-postgres-ssl-demo/feast.yaml deleted file mode 100644 index 3bce0ef2544..00000000000 --- a/examples/operator-postgres-ssl-demo/feast.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: postgres-secret - namespace: default - labels: - app: postgres -stringData: - POSTGRES_DB: mydatabase - POSTGRES_USER: admin - POSTGRES_PASSWORD: password ---- -apiVersion: v1 -kind: Secret -metadata: - name: feast-data-stores - namespace: default -stringData: - redis: | - connection_string: redis.feast.svc.cluster.local:6379 - sql: | - path: postgresql+psycopg://admin:password@postgresql.default.svc.cluster.local:5432/mydatabase?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key - cache_ttl_seconds: 60 - sqlalchemy_config_kwargs: - echo: false - pool_pre_ping: true ---- -apiVersion: feast.dev/v1alpha1 -kind: FeatureStore -metadata: - name: example - namespace: default -spec: - feastProject: credit_scoring_local - services: - volumes: - - name: ca-cert - secret: - secretName: postgresql-client-certs - items: - - key: ca.crt - path: ca.crt - mode: 0644 # Readable by all, required by PostgreSQL - - - name: client-cert - secret: - secretName: postgresql-client-certs - items: - - key: tls.crt - path: tls.crt - mode: 0644 # Required for the client certificate - - - name: client-key - secret: - secretName: postgresql-client-certs - items: - - key: tls.key - path: tls.key - mode: 0640 # Required for the private key - offlineStore: - volumeMounts: - - name: ca-cert - mountPath: /var/lib/postgresql/certs/ca.crt - subPath: ca.crt - readOnly: true - - - name: client-cert - mountPath: /var/lib/postgresql/certs/tls.crt - subPath: tls.crt - readOnly: true - - - name: client-key - mountPath: /var/lib/postgresql/certs/tls.key - subPath: tls.key - readOnly: true - persistence: - file: - type: duckdb - envFrom: - - secretRef: - name: postgres-secret - onlineStore: - volumeMounts: - - name: ca-cert - mountPath: /var/lib/postgresql/certs/ca.crt - subPath: ca.crt - readOnly: true - - - name: client-cert - mountPath: /var/lib/postgresql/certs/tls.crt - subPath: tls.crt - readOnly: true - - - name: client-key - mountPath: /var/lib/postgresql/certs/tls.key - subPath: tls.key - readOnly: true - persistence: - store: - type: redis - secretRef: - name: feast-data-stores - envFrom: - - secretRef: - name: postgres-secret - registry: - local: - volumeMounts: - - name: ca-cert - mountPath: /var/lib/postgresql/certs/ca.crt - subPath: ca.crt - readOnly: true - - - name: client-cert - mountPath: /var/lib/postgresql/certs/tls.crt - subPath: tls.crt - readOnly: true - - - name: client-key - mountPath: /var/lib/postgresql/certs/tls.key - subPath: tls.key - readOnly: true - persistence: - store: - type: sql - secretRef: - name: feast-data-stores - envFrom: - - secretRef: - name: postgres-secret - env: - - name: MPLCONFIGDIR - value: /tmp - resources: - requests: - cpu: 150m - memory: 128Mi \ No newline at end of file diff --git a/examples/operator-postgres-ssl-demo/redis.yaml b/examples/operator-postgres-ssl-demo/redis.yaml deleted file mode 100644 index a7000d115bc..00000000000 --- a/examples/operator-postgres-ssl-demo/redis.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: redis - namespace: default -spec: - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - labels: - app: redis - spec: - containers: - - name: redis - image: 'bitnami/redis:latest' - ports: - - containerPort: 6379 - env: - - name: ALLOW_EMPTY_PASSWORD - value: "yes" ---- -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: default - labels: - app: redis -spec: - type: ClusterIP - ports: - - port: 6379 - targetPort: 6379 - protocol: TCP - selector: - app: redis \ No newline at end of file diff --git a/examples/operator-postgres-ssl-demo/values.yaml b/examples/operator-postgres-ssl-demo/values.yaml deleted file mode 100644 index aed87fc7874..00000000000 --- a/examples/operator-postgres-ssl-demo/values.yaml +++ /dev/null @@ -1,9 +0,0 @@ -tls: - enabled: true - certificatesSecret: "postgresql-server-certs" - certFilename: "tls.crt" - certKeyFilename: "tls.key" - certCAFilename: "ca.crt" - -volumePermissions: - enabled: true From 806aede3fa8a73f4d127f20cc2db8d24e366bc3f Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 00:15:36 -0500 Subject: [PATCH 04/13] uncommenting the code to uninstall the feast operator. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../04-Uninstall.ipynb | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/examples/operator-postgres-tls-demo/04-Uninstall.ipynb b/examples/operator-postgres-tls-demo/04-Uninstall.ipynb index e7bb3b46152..c97d0e32cb7 100644 --- a/examples/operator-postgres-tls-demo/04-Uninstall.ipynb +++ b/examples/operator-postgres-tls-demo/04-Uninstall.ipynb @@ -9,12 +9,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret \"postgres-secret\" deleted\n", + "secret \"feast-data-stores\" deleted\n", + "featurestore.feast.dev \"sample-db-ssl\" deleted\n" + ] + } + ], "source": [ "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml\n", - "# !kubectl delete -f ../../infra/feast-operator/dist/install.yaml" + "!kubectl delete -f ../../infra/feast-operator/dist/install.yaml" ] }, { @@ -26,9 +36,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"postgresql\" uninstalled\n", + "secret \"postgresql-server-certs\" deleted\n", + "secret \"postgresql-client-certs\" deleted\n", + "persistentvolumeclaim \"data-postgresql-0\" deleted\n", + "persistentvolume \"pvc-a889c734-d921-46d4-968e-67c22542e140\" deleted\n" + ] + } + ], "source": [ "# Uninstall the Helm release\n", "!helm uninstall postgresql\n", @@ -57,12 +79,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No resources found in feast namespace.\n" + ] + } + ], "source": [ "!kubectl get all" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From fadbbf4fca19a8c35e7957af10c86ce4f22b7d60 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 00:22:20 -0500 Subject: [PATCH 05/13] Added the postgres example to the examples README.md index. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index f968b94b5f6..c7a5550e4fd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,7 +6,9 @@ 3. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. -4. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. +4. **[Operator Quickstart with Postgres in TLS](operator-quickstart)**: Demonstrates installing and configuring Feast with PostgreSQL in TLS mode on Kubernetes, using the Feast Go Operator. + +5. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. 5. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. From 9b47b4b8d49b2735feacf06ba3dd118daec6d5f5 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 13:53:21 -0500 Subject: [PATCH 06/13] Adding more details related to volumes and volumeMounts. and also incorporating code review comments. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/README.md | 14 +- .../operator-postgres-tls-demo/.gitignore | 2 +- .../01-Install-postgres-tls-using-helm.ipynb | 37 ++--- .../02-Install-feast.ipynb | 144 +++++++++++++----- .../04-Uninstall.ipynb | 20 ++- ...featurestore_postgres_db_volumes_tls.yaml} | 4 +- ...turestore_postgres_tls_volumes_ca_env.yaml | 84 ++++++++++ 7 files changed, 234 insertions(+), 71 deletions(-) rename infra/feast-operator/config/samples/{v1alpha1_featurestore_postgres_db_volumes_ssl.yaml => v1alpha1_featurestore_postgres_db_volumes_tls.yaml} (90%) create mode 100644 infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml diff --git a/examples/README.md b/examples/README.md index c7a5550e4fd..576c003fe03 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,16 +10,16 @@ 5. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. -5. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. +6. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. -6. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. +7. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. -7. **[RBAC Local](rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment. +8. **[RBAC Local](rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment. -8. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment. +9. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment. -9. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use remote offline server. +10. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use remote offline server. -10. **[Podman/Podman Compose_local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. +11. **[Podman/Podman Compose_local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. -11. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. +12. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. diff --git a/examples/operator-postgres-tls-demo/.gitignore b/examples/operator-postgres-tls-demo/.gitignore index debe8880d52..6eb45f3fbca 100644 --- a/examples/operator-postgres-tls-demo/.gitignore +++ b/examples/operator-postgres-tls-demo/.gitignore @@ -1,4 +1,4 @@ postgres-tls-certs values.yaml .ipynb_checkpoints -f43b44b.tar.gz \ No newline at end of file +*.tar.gz \ No newline at end of file diff --git a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb index 27f3aff77b1..d2e6a5f66a5 100644 --- a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb +++ b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 9, "id": "d114872a-7a43-4eca-8748-6dc7346dc176", "metadata": {}, "outputs": [ @@ -152,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "id": "bdc71e19-0fcc-4a1f-ba94-8b5e427e45d9", "metadata": {}, "outputs": [], @@ -171,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "id": "8e192410", "metadata": {}, "outputs": [ @@ -179,16 +179,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "..+.............+.....+.+.........+..+.........+....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+..........+...........+....+...+........+.+.....+.+...+.........+..+......+...+......+....+..+...+......+.+...+.........+............+.....................+........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*........+.......+........+......+.......+........+.+..............+..........+......+.........+............+..+......+.......+..................+..+...+.......+.........+...+...........+....+..+......+....+........+.+.....+.+...+........+...+.........+.............+..+.+.................+....+.....+.+...+..+.......+..+....+......+.....+.........+.+..................+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", - ".......+...+...+.......+.........+.....+....+..+.........+...................+..............+.......+.....+...+...+....+.........+.....+......+....+......+.....+.+.......................+.+.........+..+..........+.........+........+......+.......+...+....................+.+.....+...+.+.........+...+.....+.......+..+....+.........+...+...+..+....+...+........+...+..........+.........+..+...+......+.+.....+......+....+........+..........+...+.................+.+...........+....+..+.............+.........+...........+.......+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "..+.......+.........+...+.....+......+.......+...+.....+......+.+..+......+.+.....+...+.......+...+..+.+.....+.......+........+.......+......+...........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+...+...+........+....+..+...+...+....+...+......+..+..........+..+...+...+...............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+...........+......+..........+..+.+.....+....+......+.....................+...+...+..+...+.......+..+.........+.......+.....+....+........+.+..+.............+......+....................+.........+.+......+.....+.......+........+......................+......+..+...+....+...+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "..+...+......+.+.........+...+......+..+.......+.....+.+..+...+.+...+......+.....+.........+......+.+...........+....+..................+...+.........+...+.....+.+.....+...............+.+......+...+............+...+......+......+........+.+.....+.............+..+.+..+.+..............+...+...+....+............+...+.....+......+.+.....+.+...+..+...+...................+...........+....+..+.................................+..........+...........+......+.+...+..+...+.......+.....+.......+...........+.......+...+......+.....+..........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", "-----\n", - "......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.+.........+...............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+...+.+.........+.........+.....+...............+.......+...+......+.....+..........+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+.....+....+.....+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*............+...+...+....+..+...+...+...+.+++++++++++++++++++++++++++++++++++\n", + ".+....+......+..+....+...+.....+......+.+........+..........+.....+............+.+...+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.+..............+...............+.+...........+.......+...+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+...............+............+.....+.+......+........+...+...+.+...+.....+......+.+..............+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "............+....+.....+.+...+........+..........+..............+.+..............+.........+.+...+...........+......+......+.......+........+...+.........+.+.....+.+.....+.+........+.+.....................+..+.............+........+......+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", "-----\n", "Certificate request self-signature ok\n", - "subject=CN = postgresql.default.svc.cluster.local\n", - ".+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+..+...+.+..+...+.........+......+.+..............+.+..............+.+......+.....+.........+.+.....+...+....+..+...+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", - "+....+...+..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+.....+....+..+....+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+....+..+..........+.....+....+..............+...\n", + "subject=CN = postgresql.feast.svc.cluster.local\n", + "..+....+...+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+.+..+.......+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+..+.+.....+.+...+..................+.....+...+...................+......+..+...+.+......+..+..........+..+..................+.+..+...+......+.+............+..+....+...........+..........+.....+...+......+.+...+...+..+......+.+...+...+.........+......+.....+..................+.+.....+....+..............+.+..............+.+......+....................+..........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.....+.........+...+..+.......+.....+.+..+.+......+....................+......+.............+......+...+..+...+.+..+...+....+.....+...+...+.........+......+.+.....+.+..+..........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+...+.+...........+....+.....+...................+..+.+..+......+............+..........+.........+...+..+...............+..........+.....+....+............+........+.+........+.+.....+.......+.....++\n", "-----\n", "Certificate request self-signature ok\n", "subject=CN = admin\n" @@ -203,7 +203,7 @@ "!openssl req -new -x509 -days 365 -nodes -out postgres-tls-certs/ca.crt -keyout postgres-tls-certs/ca.key -subj \"/CN=PostgreSQL CA\"\n", "\n", "# Generate a server certificate\n", - "!openssl req -new -nodes -out postgres-tls-certs/server.csr -keyout postgres-tls-certs/server.key -subj \"/CN=postgresql.default.svc.cluster.local\"\n", + "!openssl req -new -nodes -out postgres-tls-certs/server.csr -keyout postgres-tls-certs/server.key -subj \"/CN=postgresql.feast.svc.cluster.local\"\n", "!openssl x509 -req -in postgres-tls-certs/server.csr -days 365 -CA postgres-tls-certs/ca.crt -CAkey postgres-tls-certs/ca.key -CAcreateserial -out postgres-tls-certs/server.crt\n", "\n", "# Generate a client certificate\n", @@ -235,7 +235,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 12, "id": "d728d0d5-2ba6-4d4d-b4be-62fb020530d4", "metadata": {}, "outputs": [ @@ -266,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 13, "id": "e14cae77", "metadata": {}, "outputs": [ @@ -275,7 +275,7 @@ "output_type": "stream", "text": [ "NAME: postgresql\n", - "LAST DEPLOYED: Sun Feb 16 22:18:39 2025\n", + "LAST DEPLOYED: Mon Feb 17 00:43:46 2025\n", "NAMESPACE: feast\n", "STATUS: deployed\n", "REVISION: 1\n", @@ -364,7 +364,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 15, "id": "132df785-762e-473a-90d2-5fdb66a59a97", "metadata": {}, "outputs": [ @@ -375,7 +375,7 @@ "pod/postgresql-0 condition met\n", "\n", "NAME READY STATUS RESTARTS AGE\n", - "postgresql-0 1/1 Running 0 25s\n", + "postgresql-0 1/1 Running 0 14s\n", "\n", "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", "ssl = 'on'\n", @@ -475,7 +475,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "5a523f9f-784f-493b-b69d-5a3cb1a830af", "metadata": {}, "outputs": [ @@ -501,6 +501,9 @@ "SSL_KEY = \"postgres-tls-certs/client.key\"\n", "SSL_ROOT_CERT = \"postgres-tls-certs/ca.crt\"\n", "\n", + "import os\n", + "os.environ[\"FEAST_CA_CERT_FILE_PATH\"] = \"postgres-tls-certs/ca.crt\"\n", + "\n", "from sqlalchemy import create_engine\n", "# Create SQLAlchemy connection string\n", "DATABASE_URL = (\n", diff --git a/examples/operator-postgres-tls-demo/02-Install-feast.ipynb b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb index 2f8e46dfde5..aed9c54c852 100644 --- a/examples/operator-postgres-tls-demo/02-Install-feast.ipynb +++ b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb @@ -69,12 +69,86 @@ "metadata": {}, "source": [ "## Install the Feast services via FeatureStore CR\n", - "Next, we'll use the running Feast Operator to install the feast services. Apply the included [reference deployment](feast.yaml) to install and configure Feast." + "Next, we'll use the running Feast Operator to install the feast services. Before doing that it is important to understand basic understanding of operator support of Volumes and volumeMounts and how to mount TLS certificates." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Mounting TLS Certificates with Volumes in Feast Operator \n", + "\n", + "The Feast operator supports **volumes** and **volumeMounts**, allowing you to mount TLS certificates onto a pod. This approach provides flexibility in how you mount these files, supporting different Kubernetes resources such as **Secrets, ConfigMaps,** and **Persistent Volumes (PVs).** \n", + "\n", + "#### Example: Mounting Certificates Using Kubernetes Secrets \n", + "\n", + "In this example, we demonstrate how to mount TLS certificates using **Kubernetes Secrets** that were created in a previous notebook. \n", + "\n", + "#### PostgreSQL Connection Parameters \n", + "\n", + "When connecting to PostgreSQL with TLS, some important parameters in the connection URL are: \n", + "\n", + "- **`sslrootcert`** – Specifies the path to the **CA certificate** file used to validate trusted certificates. \n", + "- **`sslcert`** – Provides the client certificate for **mutual TLS (mTLS) encryption**. \n", + "- **`sslkey`** – Specifies the private key for the client certificate. \n", + "\n", + "If mutual TLS authentication is not required, you can **omit** the `sslcert` and `sslkey` parameters. However, the `sslrootcert` parameter is still necessary for validating server certificates. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " Note: Please deploy either option 1 or 2 only. Don't deploy both of them at the same time to avoid conflicts in the lateral steps. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Option 1: Directly Setting the CA Certificate Path** \n", + "\n", + "In this approach, we specify the CA certificate path directly in the Feast PostgreSQL URL using the `sslrootcert` parameter. \n", + "\n", + "You can refer to the `v1alpha1_featurestore_postgres_db_volumes_tls.yaml` file for the complete configuration details. " ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 9, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgres-secret created\n", + "secret/feast-data-stores created\n", + "featurestore.feast.dev/sample-db-ssl created\n" + ] + } + ], + "source": [ + "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml --namespace=feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Option 2: Using an Environment Variable for the CA Certificate** \n", + "\n", + "In this approach, you define the CA certificate path as an environment variable. You can refer to the `v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml` file for the complete configuration details. \n", + "\n", + "```bash\n", + "FEAST_CA_CERT_FILE_PATH=\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -88,7 +162,7 @@ } ], "source": [ - "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml --namespace=feast" + "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml --namespace=feast" ] }, { @@ -101,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -109,23 +183,23 @@ "output_type": "stream", "text": [ "deployment.apps/feast-sample-db-ssl condition met\n", - "NAME READY STATUS RESTARTS AGE\n", - "pod/feast-sample-db-ssl-f559fcc9b-kzgjp 1/1 Running 0 100s\n", - "pod/postgresql-0 1/1 Running 0 6m15s\n", + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-sample-db-ssl-5ffc48c774-f6hpg 1/1 Running 0 39s\n", + "pod/postgresql-0 1/1 Running 0 13h\n", "\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "service/feast-sample-db-ssl-online ClusterIP 10.96.87.130 80/TCP 100s\n", - "service/postgresql ClusterIP 10.96.165.212 5432/TCP 6m15s\n", - "service/postgresql-hl ClusterIP None 5432/TCP 6m15s\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "service/feast-sample-db-ssl-online ClusterIP 10.96.158.78 80/TCP 39s\n", + "service/postgresql ClusterIP 10.96.228.3 5432/TCP 13h\n", + "service/postgresql-hl ClusterIP None 5432/TCP 13h\n", "\n", "NAME READY UP-TO-DATE AVAILABLE AGE\n", - "deployment.apps/feast-sample-db-ssl 1/1 1 1 100s\n", + "deployment.apps/feast-sample-db-ssl 1/1 1 1 39s\n", "\n", - "NAME DESIRED CURRENT READY AGE\n", - "replicaset.apps/feast-sample-db-ssl-f559fcc9b 1 1 1 100s\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/feast-sample-db-ssl-5ffc48c774 1 1 1 39s\n", "\n", "NAME READY AGE\n", - "statefulset.apps/postgresql 1/1 6m15s\n" + "statefulset.apps/postgresql 1/1 13h\n" ] } ], @@ -143,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -151,7 +225,7 @@ "output_type": "stream", "text": [ "NAME STATUS AGE\n", - "sample-db-ssl Ready 2m9s\n" + "sample-db-ssl Ready 52s\n" ] } ], @@ -168,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -209,14 +283,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "project: postgres_tls_sample\n", + "project: postgres_tls_sample_env_ca\n", "provider: local\n", "offline_store:\n", " host: ${POSTGRES_HOST}\n", @@ -227,7 +301,7 @@ " password: ${POSTGRES_PASSWORD}\n", " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", " sslkey_path: /var/lib/postgresql/certs/tls.key\n", - " sslmode: require\n", + " sslmode: verify-full\n", " sslrootcert_path: /var/lib/postgresql/certs/ca.crt\n", " user: ${POSTGRES_USER}\n", "online_store:\n", @@ -239,11 +313,11 @@ " port: 5432\n", " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", " sslkey_path: /var/lib/postgresql/certs/tls.key\n", - " sslmode: require\n", + " sslmode: verify-full\n", " sslrootcert_path: /var/lib/postgresql/certs/ca.crt\n", " user: ${POSTGRES_USER}\n", "registry:\n", - " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key\n", + " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=system&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key\n", " registry_type: sql\n", " cache_ttl_seconds: 60\n", " sqlalchemy_config_kwargs:\n", @@ -256,9 +330,9 @@ ": (This is the expected behaviour if you are running under QEMU)\n", "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", " DUMMY_ENTITY = Entity(\n", - "/feast-data/postgres_tls_sample/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + "/feast-data/postgres_tls_sample_env_ca/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\n", - "Applying changes for project postgres_tls_sample\n", + "Applying changes for project postgres_tls_sample_env_ca\n", "/opt/app-root/src/sdk/python/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", " warnings.warn(\n", "Deploying infrastructure for driver_hourly_stats_fresh\n", @@ -282,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -297,8 +371,9 @@ " entity = cls(\n", "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", " entity = cls(\n", - "NAME DESCRIPTION TAGS OWNER\n", - "postgres_tls_sample A project for driver statistics {}\n", + "NAME DESCRIPTION TAGS OWNER\n", + "postgres_tls_sample {}\n", + "postgres_tls_sample_env_ca A project for driver statistics {}\n", ": MADV_DONTNEED does not work (memset will be used instead)\n", ": (This is the expected behaviour if you are running under QEMU)\n", "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", @@ -310,8 +385,8 @@ "NAME ENTITIES TYPE\n", "driver_hourly_stats_fresh {'driver'} FeatureView\n", "driver_hourly_stats {'driver'} FeatureView\n", - "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n", - "transformed_conv_rate {'driver'} OnDemandFeatureView\n" + "transformed_conv_rate {'driver'} OnDemandFeatureView\n", + "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n" ] } ], @@ -329,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -347,13 +422,6 @@ "source": [ "!kubectl exec deployment/feast-sample-db-ssl -c online -- feast version" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/operator-postgres-tls-demo/04-Uninstall.ipynb b/examples/operator-postgres-tls-demo/04-Uninstall.ipynb index c97d0e32cb7..007b8d7bc1a 100644 --- a/examples/operator-postgres-tls-demo/04-Uninstall.ipynb +++ b/examples/operator-postgres-tls-demo/04-Uninstall.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -18,13 +18,21 @@ "text": [ "secret \"postgres-secret\" deleted\n", "secret \"feast-data-stores\" deleted\n", - "featurestore.feast.dev \"sample-db-ssl\" deleted\n" + "featurestore.feast.dev \"sample-db-ssl\" deleted\n", + "Error from server (NotFound): error when deleting \"../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\": secrets \"postgres-secret\" not found\n", + "Error from server (NotFound): error when deleting \"../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\": secrets \"feast-data-stores\" not found\n", + "Error from server (NotFound): error when deleting \"../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\": featurestores.feast.dev \"sample-db-ssl\" not found\n" ] } ], "source": [ - "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml\n", - "!kubectl delete -f ../../infra/feast-operator/dist/install.yaml" + "# If you have choosen the option 1 example earlier.\n", + "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml\n", + "\n", + "# If you have choosen the option 2 example earlier.\n", + "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\n", + "\n", + "#!kubectl delete -f ../../infra/feast-operator/dist/install.yaml" ] }, { @@ -36,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -47,7 +55,7 @@ "secret \"postgresql-server-certs\" deleted\n", "secret \"postgresql-client-certs\" deleted\n", "persistentvolumeclaim \"data-postgresql-0\" deleted\n", - "persistentvolume \"pvc-a889c734-d921-46d4-968e-67c22542e140\" deleted\n" + "persistentvolume \"pvc-d0c961d9-7579-4e30-842a-b46812b71f74\" deleted\n" ] } ], diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml similarity index 90% rename from infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml rename to infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml index 33238755cc6..61add153716 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml @@ -16,7 +16,7 @@ metadata: name: feast-data-stores stringData: sql: | - path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key cache_ttl_seconds: 60 sqlalchemy_config_kwargs: echo: false @@ -28,7 +28,7 @@ stringData: db_schema: public user: ${POSTGRES_USER} password: ${POSTGRES_PASSWORD} - sslmode: require + sslmode: verify-full sslkey_path: /var/lib/postgresql/certs/tls.key sslcert_path: /var/lib/postgresql/certs/tls.crt sslrootcert_path: /var/lib/postgresql/certs/ca.crt diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml new file mode 100644 index 00000000000..1d59cfc387a --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + labels: + app: postgres +stringData: + POSTGRES_DB: feast + POSTGRES_USER: admin + POSTGRES_PASSWORD: password + POSTGRES_HOST: postgresql.feast.svc.cluster.local + FEAST_CA_CERT_FILE_PATH: /var/lib/postgresql/certs/ca.crt +--- +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores +stringData: + sql: | + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=system&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true + postgres: | + host: ${POSTGRES_HOST} + port: 5432 + database: ${POSTGRES_DB} + db_schema: public + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + sslmode: verify-full + sslkey_path: /var/lib/postgresql/certs/tls.key + sslcert_path: /var/lib/postgresql/certs/tls.crt + sslrootcert_path: /var/lib/postgresql/certs/ca.crt +--- +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-db-ssl +spec: + feastProject: postgres_tls_sample_env_ca + services: + volumes: + - name: postgres-certs + secret: + secretName: postgresql-client-certs + items: + - key: ca.crt + path: ca.crt + mode: 0644 # Readable by all, required by PostgreSQL + - key: tls.crt + path: tls.crt + mode: 0644 # Required for the client certificate + - key: tls.key + path: tls.key + mode: 0640 # Required for the private key + offlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + server: + volumeMounts: + - name: postgres-certs + mountPath: /var/lib/postgresql/certs + readOnly: true + envFrom: + - secretRef: + name: postgres-secret + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores From f9a24eb48c515bacce23f228132eac0465aece3c Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:26:05 -0500 Subject: [PATCH 07/13] Removed the demo jupyter notebook which is not adding a lot of value. Also fixed some broken links. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/README.md | 2 +- .../operator-postgres-tls-demo/03-Demo.ipynb | 237 ------------------ ...{04-Uninstall.ipynb => 03-Uninstall.ipynb} | 0 examples/operator-postgres-tls-demo/README.md | 5 +- 4 files changed, 2 insertions(+), 242 deletions(-) delete mode 100644 examples/operator-postgres-tls-demo/03-Demo.ipynb rename examples/operator-postgres-tls-demo/{04-Uninstall.ipynb => 03-Uninstall.ipynb} (100%) diff --git a/examples/README.md b/examples/README.md index 576c003fe03..7c4e8ba6966 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,7 +6,7 @@ 3. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. -4. **[Operator Quickstart with Postgres in TLS](operator-quickstart)**: Demonstrates installing and configuring Feast with PostgreSQL in TLS mode on Kubernetes, using the Feast Go Operator. +4. **[Operator Quickstart with Postgres in TLS](operator-postgres-tls-demo)**: Demonstrates installing and configuring Feast with PostgreSQL in TLS mode on Kubernetes, using the Feast Go Operator. 5. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. diff --git a/examples/operator-postgres-tls-demo/03-Demo.ipynb b/examples/operator-postgres-tls-demo/03-Demo.ipynb deleted file mode 100644 index e7de3ec190a..00000000000 --- a/examples/operator-postgres-tls-demo/03-Demo.ipynb +++ /dev/null @@ -1,237 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run the \"Real-time Credit Scoring\" tutorial" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll use the following tutorial as a demonstration.\n", - "\n", - "https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/f43b44b245ae2632b582f14176392cfe31f98da9" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Upload the tutorial source code to the running Feast pod." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Upload the tutorial source code to the running feast pod and extract its contents." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "![ -f f43b44b.tar.gz ] || wget https://github.com/feast-dev/feast-credit-score-local-tutorial/archive/f43b44b.tar.gz\n", - "!kubectl cp f43b44b.tar.gz $(kubectl get pods -l 'feast.dev/name=sample-db-ssl' -ojsonpath=\"{.items[*].metadata.name}\"):/feast-data -c online\n", - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- rm -rf /feast-data/feast-credit-score-local-tutorial\n", - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- mkdir /feast-data/feast-credit-score-local-tutorial\n", - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- tar vfx /feast-data/f43b44b.tar.gz -C /feast-data/feast-credit-score-local-tutorial --strip-components 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Verify the client `feature_store.yaml`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copy the `feature_store.yaml` to the tutorial directory and verify its contents." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- cp -f /feast-data/postgres_tls_sample/feature_repo/feature_store.yaml /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml\n", - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- cat /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Apply the tutorial feature store definitions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Update the feature store definitions for the tutorial." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo apply" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load data from feature views into the online store, beginning from either the previous materialize or materialize-incremental end date, or the beginning of time." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial/feature_repo && feast materialize-incremental $(date -u +\"%Y-%m-%dT%H:%M:%S\")'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Execute feast commands inside the client Pod" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "List the registered feast feature views & entities." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo feature-views list\n", - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo entities list" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train and test the model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Install the required packages." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'pip install -r /feast-data/feast-credit-score-local-tutorial/requirements.txt'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Train and test the model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && python run.py'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Interactive demo (using Streamlit)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In a new terminal, run the following command and leave it active.\n", - "\n", - "```bash\n", - "$ kubectl port-forward deploy/feast-sample-db-ssl 8501:8501\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Start the Streamlit application" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl exec deploy/feast-sample-db-ssl -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && streamlit run --server.port 8501 streamlit_app.py'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then navigate to the local URL on which Streamlit is being served.\n", - "\n", - "http://localhost:8501" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/operator-postgres-tls-demo/04-Uninstall.ipynb b/examples/operator-postgres-tls-demo/03-Uninstall.ipynb similarity index 100% rename from examples/operator-postgres-tls-demo/04-Uninstall.ipynb rename to examples/operator-postgres-tls-demo/03-Uninstall.ipynb diff --git a/examples/operator-postgres-tls-demo/README.md b/examples/operator-postgres-tls-demo/README.md index 30dee7bb599..f37ad889a93 100644 --- a/examples/operator-postgres-tls-demo/README.md +++ b/examples/operator-postgres-tls-demo/README.md @@ -22,10 +22,7 @@ The following Jupyter Notebooks will walk you through the entire process: 2. **[02-Install-feast.ipynb](02-Install-feast.ipynb)** Deploys Feast using the Feast Operator. -3. **[03-demo-feast.ipynb](./03-Demo.ipynb)** - Demonstrates the functionality of Feast by executing [feast-credit-score-local-tutorial](https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/f43b44b245ae2632b582f14176392cfe31f98da9) the tutorial. - -4. **[04-Uninstall.ipynb](./04-Uninstall.ipynb)** +3. **[03-Uninstall.ipynb](./03-Uninstall.ipynb)** Uninstalls Feast, the Feast Operator, and the PostgreSQL deployments set up in this demo. ## How to Run the Demo From 70e5fb55b79d0d8a0a319ba1f9dba8d098064544 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:36:58 -0500 Subject: [PATCH 08/13] Minor fixes. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../02-Install-feast.ipynb | 86 +++++++++++-------- ...turestore_postgres_tls_volumes_ca_env.yaml | 2 +- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/examples/operator-postgres-tls-demo/02-Install-feast.ipynb b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb index aed9c54c852..16948b3610c 100644 --- a/examples/operator-postgres-tls-demo/02-Install-feast.ipynb +++ b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 17, "metadata": { "scrolled": true }, @@ -148,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -175,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -183,20 +183,20 @@ "output_type": "stream", "text": [ "deployment.apps/feast-sample-db-ssl condition met\n", - "NAME READY STATUS RESTARTS AGE\n", - "pod/feast-sample-db-ssl-5ffc48c774-f6hpg 1/1 Running 0 39s\n", - "pod/postgresql-0 1/1 Running 0 13h\n", + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-sample-db-ssl-86b47d54-hclb9 1/1 Running 0 27s\n", + "pod/postgresql-0 1/1 Running 0 13h\n", "\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "service/feast-sample-db-ssl-online ClusterIP 10.96.158.78 80/TCP 39s\n", - "service/postgresql ClusterIP 10.96.228.3 5432/TCP 13h\n", - "service/postgresql-hl ClusterIP None 5432/TCP 13h\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "service/feast-sample-db-ssl-online ClusterIP 10.96.61.65 80/TCP 27s\n", + "service/postgresql ClusterIP 10.96.228.3 5432/TCP 13h\n", + "service/postgresql-hl ClusterIP None 5432/TCP 13h\n", "\n", "NAME READY UP-TO-DATE AVAILABLE AGE\n", - "deployment.apps/feast-sample-db-ssl 1/1 1 1 39s\n", + "deployment.apps/feast-sample-db-ssl 1/1 1 1 27s\n", "\n", - "NAME DESIRED CURRENT READY AGE\n", - "replicaset.apps/feast-sample-db-ssl-5ffc48c774 1 1 1 39s\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/feast-sample-db-ssl-86b47d54 1 1 1 27s\n", "\n", "NAME READY AGE\n", "statefulset.apps/postgresql 1/1 13h\n" @@ -217,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -225,7 +225,7 @@ "output_type": "stream", "text": [ "NAME STATUS AGE\n", - "sample-db-ssl Ready 52s\n" + "sample-db-ssl Ready 33s\n" ] } ], @@ -242,7 +242,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -250,22 +250,24 @@ "output_type": "stream", "text": [ "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", - " List of relations\n", - " Schema | Name | Type | Owner \n", - "--------+-------------------------+-------+-------\n", - " public | data_sources | table | admin\n", - " public | entities | table | admin\n", - " public | feast_metadata | table | admin\n", - " public | feature_services | table | admin\n", - " public | feature_views | table | admin\n", - " public | managed_infra | table | admin\n", - " public | on_demand_feature_views | table | admin\n", - " public | permissions | table | admin\n", - " public | projects | table | admin\n", - " public | saved_datasets | table | admin\n", - " public | stream_feature_views | table | admin\n", - " public | validation_references | table | admin\n", - "(12 rows)\n", + " List of relations\n", + " Schema | Name | Type | Owner \n", + "--------+------------------------------------------------------+-------+-------\n", + " public | data_sources | table | admin\n", + " public | entities | table | admin\n", + " public | feast_metadata | table | admin\n", + " public | feature_services | table | admin\n", + " public | feature_views | table | admin\n", + " public | managed_infra | table | admin\n", + " public | on_demand_feature_views | table | admin\n", + " public | permissions | table | admin\n", + " public | postgres_tls_sample_env_ca_driver_hourly_stats | table | admin\n", + " public | postgres_tls_sample_env_ca_driver_hourly_stats_fresh | table | admin\n", + " public | projects | table | admin\n", + " public | saved_datasets | table | admin\n", + " public | stream_feature_views | table | admin\n", + " public | validation_references | table | admin\n", + "(14 rows)\n", "\n" ] } @@ -283,7 +285,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -302,7 +304,7 @@ " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", " sslkey_path: /var/lib/postgresql/certs/tls.key\n", " sslmode: verify-full\n", - " sslrootcert_path: /var/lib/postgresql/certs/ca.crt\n", + " sslrootcert_path: system\n", " user: ${POSTGRES_USER}\n", "online_store:\n", " type: postgres\n", @@ -314,7 +316,7 @@ " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", " sslkey_path: /var/lib/postgresql/certs/tls.key\n", " sslmode: verify-full\n", - " sslrootcert_path: /var/lib/postgresql/certs/ca.crt\n", + " sslrootcert_path: system\n", " user: ${POSTGRES_USER}\n", "registry:\n", " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=system&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key\n", @@ -332,11 +334,19 @@ " DUMMY_ENTITY = Entity(\n", "/feast-data/postgres_tls_sample_env_ca/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", "Applying changes for project postgres_tls_sample_env_ca\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", "/opt/app-root/src/sdk/python/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", " warnings.warn(\n", - "Deploying infrastructure for driver_hourly_stats_fresh\n", "Deploying infrastructure for driver_hourly_stats\n", + "Deploying infrastructure for driver_hourly_stats_fresh\n", " Feast apply is completed. You can go to next step.\n" ] } @@ -356,7 +366,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -404,7 +414,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 36, "metadata": {}, "outputs": [ { diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml index 1d59cfc387a..42e1ae4b4a6 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml @@ -32,7 +32,7 @@ stringData: sslmode: verify-full sslkey_path: /var/lib/postgresql/certs/tls.key sslcert_path: /var/lib/postgresql/certs/tls.crt - sslrootcert_path: /var/lib/postgresql/certs/ca.crt + sslrootcert_path: system --- apiVersion: feast.dev/v1alpha1 kind: FeatureStore From bd990be58ba2ec2e25397618a2f0423162fcc719 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:42:31 -0500 Subject: [PATCH 09/13] Minor fixes. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../01-Install-postgres-tls-using-helm.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb index d2e6a5f66a5..62d4b937548 100644 --- a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb +++ b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb @@ -129,7 +129,7 @@ "id": "41f4e8db", "metadata": {}, "source": [ - "## Step 3: Generate Self Signed TLS Certificates" + "## Step 4: Generate Self Signed TLS Certificates" ] }, { @@ -216,7 +216,7 @@ "id": "7e39cb28", "metadata": {}, "source": [ - "## Step 4: Create Kubernetes Secrets for Certificates" + "## Step 5: Create Kubernetes Secrets for Certificates" ] }, { @@ -261,7 +261,7 @@ "id": "67d62692", "metadata": {}, "source": [ - "## Step 5: Deploy PostgreSQL with Helm" + "## Step 6: Deploy PostgreSQL with Helm" ] }, { @@ -359,7 +359,7 @@ "id": "5be34ace", "metadata": {}, "source": [ - "## Step 6: Verify the postgres Deployment" + "## Step 7: Verify the postgres Deployment" ] }, { @@ -437,7 +437,7 @@ "id": "c921423a-81df-456e-9cca-f689070c44d2", "metadata": {}, "source": [ - "## Step 7: Port forwarding in the terminal for the connection testing using python" + "## Step 8: Port forwarding in the terminal for the connection testing using python" ] }, { @@ -470,7 +470,7 @@ "id": "a8777ca3-bf59-4f23-b7d0-60ae8c92d5a5", "metadata": {}, "source": [ - "## Step 8: Check the connection using Python sql alchemy" + "## Step 9: Check the connection using Python sql alchemy" ] }, { From 6725e539365b198bab4fdd8424e2c9dba3d84347 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:00:54 -0500 Subject: [PATCH 10/13] Minor fixes. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/operator-postgres-tls-demo/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/operator-postgres-tls-demo/README.md b/examples/operator-postgres-tls-demo/README.md index f37ad889a93..ed71b18175f 100644 --- a/examples/operator-postgres-tls-demo/README.md +++ b/examples/operator-postgres-tls-demo/README.md @@ -1,6 +1,6 @@ # Installing Feast on Kubernetes with PostgreSQL TLS Demo using feast operator -This repository contains a series of Jupyter Notebooks that guide you through setting up [Feast](https://feast.dev/) on a Kubernetes cluster. +This example folder contains a series of Jupyter Notebooks that guide you through setting up [Feast](https://feast.dev/) on a Kubernetes cluster. In this demo, Feast connects to a PostgreSQL database running in TLS mode, ensuring secure communication between services. Additionally, the example demonstrates how feast application references TLS certificates using Kubernetes volumes and volume mounts. While the focus is on mounting TLS certificates, you can also mount any other resources supported by Kubernetes volumes. From 05339cf62d55c983f41c2b0c5cd0b7f4f0a11f75 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:00:54 -0500 Subject: [PATCH 11/13] Minor fixes. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/operator-postgres-tls-demo/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/operator-postgres-tls-demo/README.md b/examples/operator-postgres-tls-demo/README.md index ed71b18175f..70ae00da6ab 100644 --- a/examples/operator-postgres-tls-demo/README.md +++ b/examples/operator-postgres-tls-demo/README.md @@ -31,7 +31,7 @@ The following Jupyter Notebooks will walk you through the entire process: ```shell https://github.com/feast-dev/feast.git - cd examples/operator-postgres-ssl-demo + cd examples/operator-postgres-tls-demo ``` 2. Start Jupyter Notebook or JupyterLab from the repository root: From 2b960772e1b3d2b949cdf0759bb982f773c72783 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 25 Feb 2025 06:23:04 -0500 Subject: [PATCH 12/13] Modified the examples README.md to make it more readable. 1. Now the numbers are automatically ordered when we render to avoid renumbering when we add an example in between. 2. Added separate section for the feast go operator examples. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- examples/README.md | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/examples/README.md b/examples/README.md index 7c4e8ba6966..ab5288dd5ad 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,25 +1,21 @@ # Feast Examples -1. **[Quickstart Example](quickstart)**: This is a step-by-step guide for getting started with Feast. - -2. **[Java Demo](java-demo)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. - -3. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. - -4. **[Operator Quickstart with Postgres in TLS](operator-postgres-tls-demo)**: Demonstrates installing and configuring Feast with PostgreSQL in TLS mode on Kubernetes, using the Feast Go Operator. - -5. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. - -6. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. +The following examples illustrate various **Feast** use cases to enhance understanding of its functionality. -7. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. - -8. **[RBAC Local](rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment. - -9. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment. - -10. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use remote offline server. - -11. **[Podman/Podman Compose_local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. - -12. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. +1. **[Quickstart Example](quickstart)**: This is a step-by-step guide for getting started with Feast. +1. **[Java Demo](java-demo)**: Demonstrates how to use Feast with Java feature server and deploy it on Kubernetes. +1. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. +1. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deploy it on Kubernetes. +1. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. +1. **[RBAC Local](rbac-local)**: Shows how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization in a local environment. +1. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication in a Kubernetes environment. +1. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use a remote offline store. +1. **[Podman/Podman Compose Local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. +1. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. + +# Feast Operator Examples + +The examples below showcase how to deploy and manage **Feast on Kubernetes** using the **Feast Go Operator**. + +1. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. +1. **[Operator Quickstart with Postgres in TLS](operator-postgres-tls-demo)**: Demonstrates installing and configuring Feast with PostgreSQL in TLS mode on Kubernetes using the Feast Go Operator, with an emphasis on volumes and VolumeMounts support. From 704f0987a20287ff9b7b79fe8418df4b6dfdeeb0 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 25 Feb 2025 08:13:42 -0500 Subject: [PATCH 13/13] Adding the helm chart version to ensure it also works in the future. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- .../01-Install-postgres-tls-using-helm.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb index 62d4b937548..d385f3d8de1 100644 --- a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb +++ b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb @@ -266,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 3, "id": "e14cae77", "metadata": {}, "outputs": [ @@ -275,7 +275,7 @@ "output_type": "stream", "text": [ "NAME: postgresql\n", - "LAST DEPLOYED: Mon Feb 17 00:43:46 2025\n", + "LAST DEPLOYED: Tue Feb 25 08:12:21 2025\n", "NAMESPACE: feast\n", "STATUS: deployed\n", "REVISION: 1\n", @@ -351,7 +351,7 @@ " f.write(helm_values)\n", "\n", "# Install PostgreSQL with Helm\n", - "!helm install postgresql bitnami/postgresql -f values.yaml -n feast" + "!helm install postgresql bitnami/postgresql --version 16.4.9 -f values.yaml -n feast " ] }, {