From 4b846a13ce5f2b9705669e4fe407890fc54c0525 Mon Sep 17 00:00:00 2001 From: tokoko Date: Thu, 21 Mar 2024 21:02:01 +0000 Subject: [PATCH 1/4] add minio tests for registry Signed-off-by: tokoko --- .../integration/registration/test_registry.py | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 57e625e66b8..ae442cc9380 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -14,9 +14,11 @@ import os import time from datetime import timedelta +from unittest import mock import pytest from pytest_lazyfixture import lazy_fixture +from testcontainers.core.container import DockerContainer from feast import FileSource from feast.data_format import ParquetFormat @@ -60,10 +62,52 @@ def s3_registry() -> Registry: return Registry("project", registry_config, None) +@pytest.fixture +def minio_registry() -> Registry: + minio_user = "minio99" + minio_password = "minio123" + bucket_name = "test-bucket" + + container: DockerContainer = ( + DockerContainer("quay.io/minio/minio") + .with_exposed_ports(9000, 9001) + .with_env("MINIO_ROOT_USER", minio_user) + .with_env("MINIO_ROOT_PASSWORD", minio_password) + .with_command('server /data --console-address ":9001"') + .with_exposed_ports() + ) + + container.start() + + exposed_port = container.get_exposed_port("9000") + container_host = container.get_container_host_ip() + + container.exec(f"mkdir /data/{bucket_name}") + + registry_config = RegistryConfig( + path=f"s3://{bucket_name}/registry.db", cache_ttl_seconds=600 + ) + + mock_environ = { + "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}", + "AWS_ACCESS_KEY_ID": minio_user, + "AWS_SECRET_ACCESS_KEY": minio_password, + } + + with mock.patch.dict(os.environ, mock_environ): + yield Registry("project", registry_config, None) + + container.stop() + + @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")], + [ + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), + ], ) def test_apply_entity_integration(test_registry): entity = Entity( @@ -106,7 +150,11 @@ def test_apply_entity_integration(test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")], + [ + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), + ], ) def test_apply_feature_view_integration(test_registry): # Create Feature Views @@ -183,7 +231,11 @@ def test_apply_feature_view_integration(test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [lazy_fixture("gcs_registry"), lazy_fixture("s3_registry")], + [ + lazy_fixture("gcs_registry"), + lazy_fixture("s3_registry"), + lazy_fixture("minio_registry"), + ], ) def test_apply_data_source_integration(test_registry: Registry): validate_registry_data_source_apply(test_registry) From 90f5d198cd9307978ffdbd1df2a04e5259721f59 Mon Sep 17 00:00:00 2001 From: tokoko Date: Fri, 22 Mar 2024 04:14:13 +0000 Subject: [PATCH 2/4] clear out session token in for minio tests Signed-off-by: tokoko --- sdk/python/tests/integration/registration/test_registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index ae442cc9380..498326d2b77 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -92,6 +92,7 @@ def minio_registry() -> Registry: "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}", "AWS_ACCESS_KEY_ID": minio_user, "AWS_SECRET_ACCESS_KEY": minio_password, + "AWS_SESSION_TOKEN": "" } with mock.patch.dict(os.environ, mock_environ): From 07ba6d676a60757e13006a717998882387167296 Mon Sep 17 00:00:00 2001 From: tokoko Date: Fri, 22 Mar 2024 05:34:23 +0000 Subject: [PATCH 3/4] log environ keys to stdout Signed-off-by: tokoko --- sdk/python/tests/integration/registration/test_registry.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 498326d2b77..73eb758b297 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -111,6 +111,9 @@ def minio_registry() -> Registry: ], ) def test_apply_entity_integration(test_registry): + print('Log Environ Keys') + print(dict(os.environ).keys()) + entity = Entity( name="driver_car_id", description="Car driver id", From fc3576d90fb6b73da7db68808106a43e68b77e08 Mon Sep 17 00:00:00 2001 From: tokoko Date: Fri, 22 Mar 2024 19:45:12 +0000 Subject: [PATCH 4/4] skip minio registry from cloud integration tests Signed-off-by: tokoko --- Makefile | 6 +++++- sdk/python/tests/integration/registration/test_registry.py | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 0eac7e03a24..e7ba985eabb 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,11 @@ test-python: python -m pytest -n 8 sdk/python/tests \ test-python-integration: - FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests + FEAST_USAGE=False \ + IS_TEST=True \ + python -m pytest -n 8 --integration \ + -k "not minio_registry" \ + sdk/python/tests test-python-integration-local: @(docker info > /dev/null 2>&1 && \ diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 73eb758b297..3bc2b3fb398 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -92,7 +92,7 @@ def minio_registry() -> Registry: "FEAST_S3_ENDPOINT_URL": f"http://{container_host}:{exposed_port}", "AWS_ACCESS_KEY_ID": minio_user, "AWS_SECRET_ACCESS_KEY": minio_password, - "AWS_SESSION_TOKEN": "" + "AWS_SESSION_TOKEN": "", } with mock.patch.dict(os.environ, mock_environ): @@ -111,9 +111,7 @@ def minio_registry() -> Registry: ], ) def test_apply_entity_integration(test_registry): - print('Log Environ Keys') - print(dict(os.environ).keys()) - + entity = Entity( name="driver_car_id", description="Car driver id",