diff --git a/sdk/python/feast/image_utils.py b/sdk/python/feast/image_utils.py index 88622284e93..6c540473cd4 100644 --- a/sdk/python/feast/image_utils.py +++ b/sdk/python/feast/image_utils.py @@ -241,6 +241,7 @@ def validate_image_format(image_bytes: bytes) -> bool: Returns: True if valid image, False otherwise """ + _check_image_dependencies() try: with Image.open(io.BytesIO(image_bytes)) as img: img.verify() @@ -259,6 +260,7 @@ def get_image_metadata(image_bytes: bytes) -> dict: Raises: ValueError: If image cannot be processed """ + _check_image_dependencies() try: with Image.open(io.BytesIO(image_bytes)) as img: return { diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index d79459c0d0e..f5d5647d9ff 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -6,12 +6,8 @@ markers = universal_online_stores: mark a test as using all online stores. rbac_remote_integration_test: mark a integration test related to rbac and remote functionality. -env = - IS_TEST=True - filterwarnings = error::_pytest.warning_types.PytestConfigWarning - error::_pytest.warning_types.PytestUnhandledCoroutineWarning ignore::DeprecationWarning:pyspark.sql.pandas.*: ignore::DeprecationWarning:pyspark.sql.connect.*: ignore::DeprecationWarning:httpx.*: diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index a57846c7e2e..df2f3fd731d 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -15,10 +15,10 @@ import multiprocessing import os import random +import sys import tempfile from datetime import timedelta from multiprocessing import Process -from sys import platform from textwrap import dedent from typing import Any, Dict, List, Tuple, no_type_check from unittest import mock @@ -36,28 +36,71 @@ create_document_dataset, create_image_dataset, ) -from tests.integration.feature_repos.integration_test_repo_config import ( # noqa: E402 - IntegrationTestRepoConfig, -) -from tests.integration.feature_repos.repo_configuration import ( # noqa: E402 - AVAILABLE_OFFLINE_STORES, - AVAILABLE_ONLINE_STORES, - OFFLINE_STORE_TO_PROVIDER_CONFIG, - Environment, - TestData, - construct_test_environment, - construct_universal_feature_views, - construct_universal_test_data, -) -from tests.integration.feature_repos.universal.data_sources.file import ( # noqa: E402 - FileDataSourceCreator, -) -from tests.integration.feature_repos.universal.entities import ( # noqa: E402 - customer, - driver, - location, -) -from tests.utils.auth_permissions_util import default_store + +try: + from tests.integration.feature_repos.integration_test_repo_config import ( # noqa: E402 + IntegrationTestRepoConfig, + ) + from tests.integration.feature_repos.repo_configuration import ( # noqa: E402 + AVAILABLE_OFFLINE_STORES, + AVAILABLE_ONLINE_STORES, + OFFLINE_STORE_TO_PROVIDER_CONFIG, + Environment, + TestData, + construct_test_environment, + construct_universal_feature_views, + construct_universal_test_data, + ) + from tests.integration.feature_repos.universal.data_sources.file import ( # noqa: E402 + FileDataSourceCreator, + ) + from tests.integration.feature_repos.universal.entities import ( # noqa: E402 + customer, + driver, + location, + ) + + _integration_test_deps_available = True +except ModuleNotFoundError: + _integration_test_deps_available = False + + IntegrationTestRepoConfig = None # type: ignore[assignment] + AVAILABLE_OFFLINE_STORES = [] # type: ignore[assignment] + AVAILABLE_ONLINE_STORES = {} # type: ignore[assignment] + OFFLINE_STORE_TO_PROVIDER_CONFIG = {} # type: ignore[assignment] + Environment = Any # type: ignore[assignment] + TestData = Any # type: ignore[assignment] + + def construct_test_environment(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Integration test dependencies are not available") + + def construct_universal_feature_views(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Integration test dependencies are not available") + + def construct_universal_test_data(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Integration test dependencies are not available") + + class FileDataSourceCreator: # type: ignore[no-redef] + pass + + def customer(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Integration test dependencies are not available") + + def driver(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Integration test dependencies are not available") + + def location(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Integration test dependencies are not available") + + +try: + from tests.utils.auth_permissions_util import default_store +except ModuleNotFoundError: + + def default_store(*args, **kwargs): # type: ignore[no-redef] + raise RuntimeError("Auth test dependencies are not available") + + from tests.utils.http_server import check_port_open, free_port # noqa: E402 from tests.utils.ssl_certifcates_util import ( combine_trust_stores, @@ -67,6 +110,8 @@ logger = logging.getLogger(__name__) +os.environ.setdefault("IS_TEST", "True") + level = logging.INFO logging.basicConfig( format="%(asctime)s %(name)s %(levelname)s: %(message)s", @@ -85,7 +130,7 @@ def pytest_configure(config): - if platform in ["darwin", "windows"]: + if sys.platform == "darwin" or sys.platform.startswith("win"): multiprocessing.set_start_method("spawn", force=True) else: multiprocessing.set_start_method("fork") @@ -239,92 +284,96 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): See more examples at https://docs.pytest.org/en/6.2.x/example/parametrize.html#paramexamples - We also utilize indirect parametrization here. Since `environment` is a fixture, - when we call metafunc.parametrize("environment", ..., indirect=True) we actually - parametrizing this "environment" fixture and not the test itself. - Moreover, by utilizing `_config_cache` we are able to share `environment` fixture between different tests. - In order for pytest to group tests together (and share environment fixture) - parameter should point to the same Python object (hence, we use _config_cache dict to store those objects). """ - if "environment" in metafunc.fixturenames: - markers = {m.name: m for m in metafunc.definition.own_markers} - offline_stores = None - if "universal_offline_stores" in markers: - # Offline stores can be explicitly requested - if "only" in markers["universal_offline_stores"].kwargs: - offline_stores = [ - OFFLINE_STORE_TO_PROVIDER_CONFIG.get(store_name) - for store_name in markers["universal_offline_stores"].kwargs["only"] - if store_name in OFFLINE_STORE_TO_PROVIDER_CONFIG - ] - else: - offline_stores = AVAILABLE_OFFLINE_STORES + if "environment" not in metafunc.fixturenames: + return + + if not _integration_test_deps_available: + pytest.skip("Integration test dependencies are not available") + + own_markers = getattr(metafunc.definition, "own_markers", None) + marker_iter = ( + own_markers if own_markers is not None else metafunc.definition.iter_markers() + ) + markers = {m.name: m for m in marker_iter} + + offline_stores = None + if "universal_offline_stores" in markers: + # Offline stores can be explicitly requested + if "only" in markers["universal_offline_stores"].kwargs: + offline_stores = [ + OFFLINE_STORE_TO_PROVIDER_CONFIG.get(store_name) + for store_name in markers["universal_offline_stores"].kwargs["only"] + if store_name in OFFLINE_STORE_TO_PROVIDER_CONFIG + ] else: - # default offline store for testing online store dimension - offline_stores = [("local", FileDataSourceCreator)] - - online_stores = None - if "universal_online_stores" in markers: - # Online stores can be explicitly requested - if "only" in markers["universal_online_stores"].kwargs: - online_stores = [ - AVAILABLE_ONLINE_STORES.get(store_name) - for store_name in markers["universal_online_stores"].kwargs["only"] - if store_name in AVAILABLE_ONLINE_STORES - ] - else: - online_stores = AVAILABLE_ONLINE_STORES.values() - - if online_stores is None: - # No online stores requested -> setting the default or first available + offline_stores = AVAILABLE_OFFLINE_STORES + else: + # default offline store for testing online store dimension + offline_stores = [("local", FileDataSourceCreator)] + + online_stores = None + if "universal_online_stores" in markers: + # Online stores can be explicitly requested + if "only" in markers["universal_online_stores"].kwargs: online_stores = [ - AVAILABLE_ONLINE_STORES.get( - "redis", - AVAILABLE_ONLINE_STORES.get( - "sqlite", next(iter(AVAILABLE_ONLINE_STORES.values())) - ), - ) + AVAILABLE_ONLINE_STORES.get(store_name) + for store_name in markers["universal_online_stores"].kwargs["only"] + if store_name in AVAILABLE_ONLINE_STORES ] - - extra_dimensions: List[Dict[str, Any]] = [{}] - - if "python_server" in metafunc.fixturenames: - extra_dimensions.extend([{"python_feature_server": True}]) - - configs = [] - if offline_stores: - for provider, offline_store_creator in offline_stores: - for online_store, online_store_creator in online_stores: - for dim in extra_dimensions: - config = { - "provider": provider, - "offline_store_creator": offline_store_creator, - "online_store": online_store, - "online_store_creator": online_store_creator, - **dim, - } - - c = IntegrationTestRepoConfig(**config) - - if c not in _config_cache: - marks = [ - pytest.mark.xdist_group(name=m) - for m in c.offline_store_creator.xdist_groups() - ] - # Check if there are any test markers associated with the creator and add them. - if c.offline_store_creator.test_markers(): - marks.extend(c.offline_store_creator.test_markers()) - - _config_cache[c] = pytest.param(c, marks=marks) - - configs.append(_config_cache[c]) else: - # No offline stores requested -> setting the default or first available - offline_stores = [("local", FileDataSourceCreator)] + online_stores = AVAILABLE_ONLINE_STORES.values() - metafunc.parametrize( - "environment", configs, indirect=True, ids=[str(c) for c in configs] - ) + if online_stores is None: + # No online stores requested -> setting the default or first available + online_stores = [ + AVAILABLE_ONLINE_STORES.get( + "redis", + AVAILABLE_ONLINE_STORES.get( + "sqlite", next(iter(AVAILABLE_ONLINE_STORES.values())) + ), + ) + ] + + extra_dimensions: List[Dict[str, Any]] = [{}] + + if "python_server" in metafunc.fixturenames: + extra_dimensions.extend([{"python_feature_server": True}]) + + configs = [] + if offline_stores: + for provider, offline_store_creator in offline_stores: + for online_store, online_store_creator in online_stores: + for dim in extra_dimensions: + config = { + "provider": provider, + "offline_store_creator": offline_store_creator, + "online_store": online_store, + "online_store_creator": online_store_creator, + **dim, + } + + c = IntegrationTestRepoConfig(**config) + + if c not in _config_cache: + marks = [ + pytest.mark.xdist_group(name=m) + for m in c.offline_store_creator.xdist_groups() + ] + # Check if there are any test markers associated with the creator and add them. + if c.offline_store_creator.test_markers(): + marks.extend(c.offline_store_creator.test_markers()) + + _config_cache[c] = pytest.param(c, marks=marks) + + configs.append(_config_cache[c]) + else: + # No offline stores requested -> setting the default or first available + offline_stores = [("local", FileDataSourceCreator)] + + metafunc.parametrize( + "environment", configs, indirect=True, ids=[str(c) for c in configs] + ) @pytest.fixture diff --git a/sdk/python/tests/unit/test_image_utils_optional_deps.py b/sdk/python/tests/unit/test_image_utils_optional_deps.py new file mode 100644 index 00000000000..3821af52845 --- /dev/null +++ b/sdk/python/tests/unit/test_image_utils_optional_deps.py @@ -0,0 +1,33 @@ +# Copyright 2024 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +def test_validate_image_format_raises_when_deps_missing(monkeypatch): + from feast import image_utils + + monkeypatch.setattr(image_utils, "_image_dependencies_available", False) + + with pytest.raises(ImportError, match="Image processing dependencies are not installed"): + image_utils.validate_image_format(b"anything") + + +def test_get_image_metadata_raises_when_deps_missing(monkeypatch): + from feast import image_utils + + monkeypatch.setattr(image_utils, "_image_dependencies_available", False) + + with pytest.raises(ImportError, match="Image processing dependencies are not installed"): + image_utils.get_image_metadata(b"anything") diff --git a/sdk/python/tests/utils/auth_permissions_util.py b/sdk/python/tests/utils/auth_permissions_util.py index c332a5ab8d3..e04686286a2 100644 --- a/sdk/python/tests/utils/auth_permissions_util.py +++ b/sdk/python/tests/utils/auth_permissions_util.py @@ -1,5 +1,6 @@ import os import subprocess +from pathlib import Path import yaml from keycloak import KeycloakAdmin @@ -36,16 +37,22 @@ def default_store( permissions: list[Permission], ): runner = CliRunner() - result = runner.run(["init", PROJECT_NAME], cwd=temp_dir) + result = runner.run(["init", PROJECT_NAME], cwd=Path(temp_dir)) repo_path = os.path.join(temp_dir, PROJECT_NAME, "feature_repo") - assert result.returncode == 0 + assert result.returncode == 0, ( + f"feast init failed. stdout:\n{result.stdout.decode(errors='ignore')}\n" + f"stderr:\n{result.stderr.decode(errors='ignore')}\n" + ) include_auth_config( file_path=f"{repo_path}/feature_store.yaml", auth_config=auth_config ) - result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_dir) - assert result.returncode == 0 + result = runner.run(["--chdir", repo_path, "apply"], cwd=Path(temp_dir)) + assert result.returncode == 0, ( + f"feast apply failed. stdout:\n{result.stdout.decode(errors='ignore')}\n" + f"stderr:\n{result.stderr.decode(errors='ignore')}\n" + ) fs = FeatureStore(repo_path=repo_path) diff --git a/sdk/python/tests/utils/cli_repo_creator.py b/sdk/python/tests/utils/cli_repo_creator.py index ea1d7fcf10b..7922e7454df 100644 --- a/sdk/python/tests/utils/cli_repo_creator.py +++ b/sdk/python/tests/utils/cli_repo_creator.py @@ -1,3 +1,4 @@ +import os import random import string import subprocess @@ -33,11 +34,18 @@ class CliRunner: """ def run(self, args: List[str], cwd: Path) -> subprocess.CompletedProcess: + env = os.environ.copy() + env.setdefault("IS_TEST", "True") return subprocess.run( - [sys.executable, cli.__file__] + args, cwd=cwd, capture_output=True + [sys.executable, cli.__file__] + args, + cwd=cwd, + capture_output=True, + env=env, ) def run_with_output(self, args: List[str], cwd: Path) -> Tuple[int, bytes]: + env = os.environ.copy() + env.setdefault("IS_TEST", "True") try: return ( 0, @@ -45,6 +53,7 @@ def run_with_output(self, args: List[str], cwd: Path) -> Tuple[int, bytes]: [sys.executable, cli.__file__] + args, cwd=cwd, stderr=subprocess.STDOUT, + env=env, ), ) except subprocess.CalledProcessError as e: