From aa2c8393086d32fc29beef71bd9d542b4752042c Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 11:31:04 +0100 Subject: [PATCH 01/22] Clickhouse offline store - initial working version Signed-off-by: Tomasz Wrona --- Makefile | 23 +- .../clickhouse_offline_store/__init__.py | 0 .../clickhouse_offline_store/clickhouse.py | 617 ++++++++++++++++++ .../clickhouse_source.py | 212 ++++++ .../tests/__init__.py | 1 + .../tests/data_source.py | 142 ++++ .../contrib/clickhouse_repo_configuration.py | 11 + .../feast/infra/utils/clickhouse/__init__.py | 0 .../utils/clickhouse/clickhouse_config.py | 12 + .../utils/clickhouse/connection_utils.py | 18 + .../requirements/py3.11-ci-requirements.txt | 13 +- setup.py | 6 +- 12 files changed, 1050 insertions(+), 5 deletions(-) create mode 100644 sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/__init__.py create mode 100644 sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py create mode 100644 sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py create mode 100644 sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/__init__.py create mode 100644 sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py create mode 100644 sdk/python/feast/infra/offline_stores/contrib/clickhouse_repo_configuration.py create mode 100644 sdk/python/feast/infra/utils/clickhouse/__init__.py create mode 100644 sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py create mode 100644 sdk/python/feast/infra/utils/clickhouse/connection_utils.py diff --git a/Makefile b/Makefile index 30ac86e8919..3eb92c263fa 100644 --- a/Makefile +++ b/Makefile @@ -218,7 +218,28 @@ test-python-universal-postgres-offline: not gcs_registry and \ not s3_registry and \ not test_snowflake and \ - not test_universal_types" \ + not test_spark" \ + sdk/python/tests + + test-python-universal-clickhouse-offline: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.clickhouse_repo_configuration \ + PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.clickhouse_offline_store.tests \ + python -m pytest -v -n 8 --integration \ + -k "not test_historical_retrieval_with_validation and \ + not test_historical_features_persisting and \ + not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_snowflake and \ + not test_spark" \ sdk/python/tests test-python-universal-postgres-online: diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py new file mode 100644 index 00000000000..3ec09601852 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -0,0 +1,617 @@ +import contextlib +import re +from dataclasses import asdict +from datetime import datetime +from typing import Iterator, List, Literal, Optional, Union, cast + +import numpy as np +import pandas as pd +import pyarrow as pa +from pyarrow.compute import cast as pa_cast + +from feast import FeatureView, OnDemandFeatureView, RepoConfig +from feast.data_source import DataSource +from feast.errors import EntitySQLEmptyResults, InvalidEntityType +from feast.feature_view import DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL +from feast.infra.offline_stores import offline_utils +from feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source import ( + ClickhouseSource, + SavedDatasetClickhouseStorage, +) +from feast.infra.offline_stores.contrib.postgres_offline_store.postgres import ( + PostgreSQLRetrievalJob, + build_point_in_time_query, +) +from feast.infra.offline_stores.offline_store import ( + OfflineStore, + RetrievalJob, + RetrievalMetadata, +) +from feast.infra.registry.base_registry import BaseRegistry +from feast.infra.utils.clickhouse.clickhouse_config import ClickhouseConfig +from feast.infra.utils.clickhouse.connection_utils import get_client +from feast.saved_dataset import SavedDatasetStorage + + +class ClickhouseOfflineStoreConfig(ClickhouseConfig): + type: Literal[ + "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" + ] = "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" + + +class ClickhouseOfflineStore(OfflineStore): + @staticmethod + def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> RetrievalJob: + assert isinstance(config.offline_store, ClickhouseOfflineStoreConfig) + for fv in feature_views: + assert isinstance(fv.batch_source, ClickhouseSource) + + entity_schema = _get_entity_schema(entity_df, config) + + entity_df_event_timestamp_col = ( + offline_utils.infer_event_timestamp_from_entity_df(entity_schema) + ) + + entity_df_event_timestamp_range = _get_entity_df_event_timestamp_range( + entity_df, + entity_df_event_timestamp_col, + config, + ) + + @contextlib.contextmanager + def query_generator() -> Iterator[str]: + table_name = offline_utils.get_temp_entity_table_name() + if ( + isinstance(entity_df, pd.DataFrame) + and not config.offline_store.use_temporary_tables_for_entity_df + ): + table_name = f"{config.offline_store.database}.{table_name}" + + _upload_entity_df( + config, + entity_df, + table_name, + entity_df_event_timestamp_col, + ) + + expected_join_keys = offline_utils.get_expected_join_keys( + project, feature_views, registry + ) + + offline_utils.assert_expected_columns_in_entity_df( + entity_schema, expected_join_keys, entity_df_event_timestamp_col + ) + + query_context = offline_utils.get_feature_view_query_context( + feature_refs, + feature_views, + registry, + project, + entity_df_event_timestamp_range, + ) + + query_context_dict = [asdict(context) for context in query_context] + # Hack for query_context.entity_selections to support uppercase in columns + for context in query_context_dict: + context["entity_selections"] = [ + f""""{entity_selection.replace(' AS ', '" AS "')}\"""" + for entity_selection in context["entity_selections"] + ] + + try: + query = build_point_in_time_query( + query_context_dict, + left_table_query_string=table_name, + entity_df_event_timestamp_col=entity_df_event_timestamp_col, + entity_df_columns=entity_schema.keys(), + query_template=MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN, + full_feature_names=full_feature_names, + ) + yield query + finally: + if ( + table_name + and not config.offline_store.use_temporary_tables_for_entity_df + ): + get_client(config.offline_store).command( + f"DROP TABLE IF EXISTS {table_name}" + ) + + return ClickhouseRetrievalJob( + query=query_generator, + config=config, + full_feature_names=full_feature_names, + on_demand_feature_views=OnDemandFeatureView.get_requested_odfvs( + feature_refs, project, registry + ), + metadata=RetrievalMetadata( + features=feature_refs, + keys=list(entity_schema.keys() - {entity_df_event_timestamp_col}), + min_event_timestamp=entity_df_event_timestamp_range[0], + max_event_timestamp=entity_df_event_timestamp_range[1], + ), + ) + + @staticmethod + def pull_latest_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + assert isinstance(config.offline_store, ClickhouseOfflineStoreConfig) + assert isinstance(data_source, ClickhouseSource) + from_expression = data_source.get_table_query_string() + + partition_by_join_key_string = ", ".join(_append_alias(join_key_columns, "a")) + if partition_by_join_key_string != "": + partition_by_join_key_string = ( + "PARTITION BY " + partition_by_join_key_string + ) + timestamps = [timestamp_field] + if created_timestamp_column: + timestamps.append(created_timestamp_column) + timestamp_desc_string = " DESC, ".join(_append_alias(timestamps, "a")) + " DESC" + a_field_string = ", ".join( + _append_alias(join_key_columns + feature_name_columns + timestamps, "a") + ) + b_field_string = ", ".join( + _append_alias(join_key_columns + feature_name_columns + timestamps, "b") + ) + + query = f""" + SELECT + {b_field_string} + {f", {repr(DUMMY_ENTITY_VAL)} AS {DUMMY_ENTITY_ID}" if not join_key_columns else ""} + FROM ( + SELECT {a_field_string}, + ROW_NUMBER() OVER({partition_by_join_key_string} ORDER BY {timestamp_desc_string}) AS _feast_row + FROM ({from_expression}) a + WHERE a."{timestamp_field}" + BETWEEN toDateTime64('{start_date.replace(tzinfo=None)!s}', 6, '{start_date.tzinfo!s}') + AND toDateTime64('{end_date.replace(tzinfo=None)!s}', 6, '{end_date.tzinfo!s}') + ) b + WHERE _feast_row = 1 + """ + + return ClickhouseRetrievalJob( + query=query, + config=config, + full_feature_names=False, + on_demand_feature_views=None, + ) + + @staticmethod + def pull_all_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + raise NotImplementedError("Not tested (not needed?)") + assert isinstance(config.offline_store, ClickhouseOfflineStoreConfig) + assert isinstance(data_source, ClickhouseSource) + from_expression = data_source.get_table_query_string() + + field_string = ", ".join( + join_key_columns + feature_name_columns + [timestamp_field] + ) + + query = f""" + SELECT {field_string} + FROM {from_expression} AS paftoq_alias + WHERE "{timestamp_field}" + BETWEEN toDateTime64('{start_date.replace(tzinfo=None)!s}', 6, '{start_date.tzinfo!s}') + AND toDateTime64('{end_date.replace(tzinfo=None)!s}', 6, '{end_date.tzinfo!s}') + """ + + return ClickhouseRetrievalJob( + query=query, + config=config, + full_feature_names=False, + on_demand_feature_views=None, + ) + + +class ClickhouseRetrievalJob(PostgreSQLRetrievalJob): + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: + with self._query_generator() as query: + results = get_client(self.config.offline_store).query_df(query) + return results + + def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: + with self._query_generator() as query: + results: pa.Table = get_client(self.config.offline_store).query_arrow(query) + # Feast doesn't support native decimal types, so we must convert decimal columns to double + for col_index, (name, dtype) in enumerate( + zip(results.schema.names, results.schema.types) + ): + if pa.types.is_decimal(dtype): + results = results.set_column( + col_index, + name, + pa_cast(results[name], target_type=pa.float64()), + ) + return results + + def persist( + self, + storage: SavedDatasetStorage, + allow_overwrite: Optional[bool] = False, + timeout: Optional[int] = None, + ): + assert isinstance(storage, SavedDatasetClickhouseStorage) + + df_to_clickhouse_table( + config=self.config.offline_store, + df=self.to_df(), + table_name=storage.clickhouse_options._table, + entity_timestamp_col="event_timestamp", + ) + + +def _get_entity_schema( + entity_df: Union[pd.DataFrame, str], + config: RepoConfig, +) -> dict[str, np.dtype]: + if isinstance(entity_df, pd.DataFrame): + return dict(zip(entity_df.columns, entity_df.dtypes)) + elif isinstance(entity_df, str): + query = f"SELECT * FROM ({entity_df}) LIMIT 1" + df = get_client(config.offline_store).query_df(query) + return _get_entity_schema(df, config) + else: + raise InvalidEntityType(type(entity_df)) + + +def _get_entity_df_event_timestamp_range( + entity_df: Union[pd.DataFrame, str], + entity_df_event_timestamp_col: str, + config: RepoConfig, +) -> tuple[datetime, datetime]: + if isinstance(entity_df, pd.DataFrame): + entity_df_event_timestamp = entity_df.loc[ + :, entity_df_event_timestamp_col + ].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime( + entity_df_event_timestamp, utc=True + ) + entity_df_event_timestamp_range = ( + entity_df_event_timestamp.min().to_pydatetime(), + entity_df_event_timestamp.max().to_pydatetime(), + ) + elif isinstance(entity_df, str): + # If the entity_df is a string (SQL query), determine range + # from table + query = f'SELECT MIN("{entity_df_event_timestamp_col}") AS "min_value", MAX("{entity_df_event_timestamp_col}") AS "max_value" FROM ({entity_df})' + results = get_client(config.offline_store).query(query).result_rows + + entity_df_event_timestamp_range = cast(tuple[datetime, datetime], results[0]) + if ( + entity_df_event_timestamp_range[0] is None + or entity_df_event_timestamp_range[1] is None + ): + raise EntitySQLEmptyResults(entity_df) + else: + raise InvalidEntityType(type(entity_df)) + + return entity_df_event_timestamp_range + + +def _upload_entity_df( + config: RepoConfig, + entity_df: Union[pd.DataFrame, str], + table_name: str, + entity_timestamp_col: str, +) -> None: + if isinstance(entity_df, pd.DataFrame): + df_to_clickhouse_table( + config.offline_store, entity_df, table_name, entity_timestamp_col + ) + elif isinstance(entity_df, str): + if config.offline_store.use_temporary_tables_for_entity_df: + query = f'CREATE TEMPORARY TABLE "{table_name}" AS ({entity_df})' + else: + query = f'CREATE TABLE "{table_name}" ENGINE = MergeTree() ORDER BY ({entity_timestamp_col}) AS ({entity_df})' + get_client(config.offline_store).command(query) + else: + raise InvalidEntityType(type(entity_df)) + + +def df_to_clickhouse_table( + config: ClickhouseConfig, + df: pd.DataFrame, + table_name: str, + entity_timestamp_col: str, +) -> None: + # pd_dtypes = df.dtypes.to_dict() + # ch_dtypes = convert_pd_dtypes_to_ch(pd_dtypes) + # table_schema = ", ".join( + # [f'"{col}" {ch_dtype}' for col, ch_dtype in ch_dtypes.items()] + # ) + table_schema = _df_to_create_table_schema(df) + if config.use_temporary_tables_for_entity_df: + query = f"CREATE TEMPORARY TABLE {table_name} ({table_schema})" + else: + query = f""" + CREATE TABLE {table_name} ( + {table_schema} + ) + ENGINE = MergeTree() + ORDER BY ({entity_timestamp_col}) + """ + get_client(config).command(query) + get_client(config).insert_df(table_name, df) + + +def _df_to_create_table_schema(entity_df: pd.DataFrame) -> str: + pa_table = pa.Table.from_pandas(entity_df) + # print(f"{pa_table.schema=}") + # print(f"{entity_df=}") + columns = [ + f""""{f.name}" {arrow_to_ch_type(str(f.type), f.nullable)}""" + for f in pa_table.schema + ] + # print(f"{columns=}") + return ", ".join(columns) + + +def _arrow_to_ch_timestamp_type(t_str: str) -> str: + _ARROW_PRECISION_TO_CH_PRECISION = { + "s": 0, + "ms": 3, + "us": 6, + "ns": 9, + } + + unit, *rest = t_str.removeprefix("timestamp[").removesuffix("]").split(",") + + unit = unit.strip() + precision = _ARROW_PRECISION_TO_CH_PRECISION[unit] + + tz: Optional[str] = None + if len(rest): + tz: str = rest[0] + tz = ( + tz.strip() + .removeprefix("tz=") + .translate( + str.maketrans( + { + "'": None, + '"': None, + } + ) + ) + ) + + if precision > 0: + if tz is not None: + return f"DateTime64({precision}, '{tz}')" + else: + return f"DateTime64({precision})" + else: + if tz is not None: + return f"DateTime('{tz}')" + else: + return "DateTime" + + +def arrow_to_ch_type(t_str: str, nullable: bool) -> str: + list_pattern = r"list" + list_res = re.search(list_pattern, t_str) + if list_res is not None: + item_type_str = list_res.group(1) + return f"Array({arrow_to_ch_type(item_type_str, nullable)})" + + if nullable: + return f"Nullable({arrow_to_ch_type(t_str, nullable=False)})" + + try: + if t_str.startswith("timestamp"): + return _arrow_to_ch_timestamp_type(t_str) + return { + "bool": "Boolean", + "int8": "Int8", + "int16": "Int16", + "int32": "Int32", + "int64": "Int64", + "uint8": "UInt8", + "uint16": "UInt16", + "uint32": "UInt32", + "uint64": "Uint64", + "float": "Float32", + "double": "Float64", + "string": "String", + }[t_str] + except KeyError: + raise ValueError(f"Unsupported type: {t_str}") + + +def _append_alias(field_names: List[str], alias: str) -> List[str]: + return [f'{alias}."{field_name}"' for field_name in field_names] + + +MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN = """ +/* + Compute a deterministic hash for the `left_table_query_string` that will be used throughout + all the logic as the field to GROUP BY the data +*/ +WITH entity_dataframe AS ( + SELECT *, + {{entity_df_event_timestamp_col}} AS entity_timestamp + {% for featureview in featureviews %} + {% if featureview.entities %} + ,( + {% for entity in featureview.entities %} + CAST("{{entity}}" as VARCHAR) || + {% endfor %} + CAST("{{entity_df_event_timestamp_col}}" AS VARCHAR) + ) AS "{{featureview.name}}__entity_row_unique_id" + {% else %} + ,CAST("{{entity_df_event_timestamp_col}}" AS VARCHAR) AS "{{featureview.name}}__entity_row_unique_id" + {% endif %} + {% endfor %} + FROM {{ left_table_query_string }} +), + +{% for featureview in featureviews %} + +"{{ featureview.name }}__entity_dataframe" AS ( + SELECT + {% if featureview.entities %}"{{ featureview.entities | join('", "') }}",{% endif %} + entity_timestamp, + "{{featureview.name}}__entity_row_unique_id" + FROM entity_dataframe + GROUP BY + {% if featureview.entities %}"{{ featureview.entities | join('", "')}}",{% endif %} + entity_timestamp, + "{{featureview.name}}__entity_row_unique_id" +), + +/* + This query template performs the point-in-time correctness join for a single feature set table + to the provided entity table. + + 1. We first join the current feature_view to the entity dataframe that has been passed. + This JOIN has the following logic: + - For each row of the entity dataframe, only keep the rows where the `timestamp_field` + is less than the one provided in the entity dataframe + - If there a TTL for the current feature_view, also keep the rows where the `timestamp_field` + is higher the the one provided minus the TTL + - For each row, Join on the entity key and retrieve the `entity_row_unique_id` that has been + computed previously + + The output of this CTE will contain all the necessary information and already filtered out most + of the data that is not relevant. +*/ + +"{{ featureview.name }}__subquery" AS ( + SELECT + "{{ featureview.timestamp_field }}" as event_timestamp, + {{ '"' ~ featureview.created_timestamp_column ~ '" as created_timestamp,' if featureview.created_timestamp_column else '' }} + {{ featureview.entity_selections | join(', ')}}{% if featureview.entity_selections %},{% else %}{% endif %} + {% for feature in featureview.features %} + "{{ feature }}" as {% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %}{% if loop.last %}{% else %}, {% endif %} + {% endfor %} + FROM {{ featureview.table_subquery }} AS sub + WHERE "{{ featureview.timestamp_field }}" <= (SELECT MAX(entity_timestamp) FROM entity_dataframe) + {% if featureview.ttl == 0 %}{% else %} + AND "{{ featureview.timestamp_field }}" >= (SELECT MIN(entity_timestamp) FROM entity_dataframe) - interval {{ featureview.ttl }} second + {% endif %} +), + +"{{ featureview.name }}__base" AS ( + SELECT + subquery.*, + entity_dataframe.entity_timestamp, + entity_dataframe."{{featureview.name}}__entity_row_unique_id" + FROM "{{ featureview.name }}__subquery" AS subquery + INNER JOIN "{{ featureview.name }}__entity_dataframe" AS entity_dataframe + ON TRUE + {% for entity in featureview.entities %} + AND subquery."{{ entity }}" = entity_dataframe."{{ entity }}" + {% endfor %} + WHERE TRUE + AND subquery.event_timestamp <= entity_dataframe.entity_timestamp + + {% if featureview.ttl == 0 %}{% else %} + AND subquery.event_timestamp >= entity_dataframe.entity_timestamp - interval {{ featureview.ttl }} second + {% endif %} +), + +/* + 2. If the `created_timestamp_column` has been set, we need to + deduplicate the data first. This is done by calculating the + `MAX(created_at_timestamp)` for each event_timestamp. + We then join the data on the next CTE +*/ +{% if featureview.created_timestamp_column %} +"{{ featureview.name }}__dedup" AS ( + SELECT + "{{featureview.name}}__entity_row_unique_id", + event_timestamp, + MAX(created_timestamp) as created_timestamp + FROM "{{ featureview.name }}__base" + GROUP BY "{{featureview.name}}__entity_row_unique_id", event_timestamp +), +{% endif %} + +/* + 3. The data has been filtered during the first CTE "*__base" + Thus we only need to compute the latest timestamp of each feature. +*/ +"{{ featureview.name }}__latest" AS ( + SELECT + event_timestamp, + {% if featureview.created_timestamp_column %}created_timestamp,{% endif %} + "{{featureview.name}}__entity_row_unique_id" + FROM + ( + SELECT *, + ROW_NUMBER() OVER( + PARTITION BY "{{featureview.name}}__entity_row_unique_id" + ORDER BY event_timestamp DESC{% if featureview.created_timestamp_column %},created_timestamp DESC{% endif %} + ) AS row_number + FROM "{{ featureview.name }}__base" + {% if featureview.created_timestamp_column %} + INNER JOIN "{{ featureview.name }}__dedup" + USING ("{{featureview.name}}__entity_row_unique_id", event_timestamp, created_timestamp) + {% endif %} + ) AS sub + WHERE row_number = 1 +), + +/* + 4. Once we know the latest value of each feature for a given timestamp, + we can join again the data back to the original "base" dataset +*/ +"{{ featureview.name }}__cleaned" AS ( + SELECT base.* + FROM "{{ featureview.name }}__base" as base + INNER JOIN "{{ featureview.name }}__latest" + USING( + "{{featureview.name}}__entity_row_unique_id", + event_timestamp + {% if featureview.created_timestamp_column %} + ,created_timestamp + {% endif %} + ) +){% if loop.last %}{% else %}, {% endif %} + + +{% endfor %} +/* + Joins the outputs of multiple time travel joins to a single table. + The entity_dataframe dataset being our source of truth here. + */ + +SELECT "{{ final_output_feature_names | join('", "')}}" +FROM entity_dataframe +{% for featureview in featureviews %} +LEFT JOIN ( + SELECT + "{{featureview.name}}__entity_row_unique_id" + {% for feature in featureview.features %} + ,"{% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %}" + {% endfor %} + FROM "{{ featureview.name }}__cleaned" +) AS "{{featureview.name}}" USING ("{{featureview.name}}__entity_row_unique_id") +{% endfor %} +""" diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py new file mode 100644 index 00000000000..ab85233c64c --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py @@ -0,0 +1,212 @@ +import json +from typing import Any, Callable, Iterable, Optional, Tuple, Type + +from clickhouse_connect.datatypes.base import ClickHouseType +from clickhouse_connect.datatypes.container import Array +from clickhouse_connect.datatypes.numeric import ( + Boolean, + Decimal, + Float32, + Float64, + Int32, + Int64, +) +from clickhouse_connect.datatypes.registry import get_from_name +from clickhouse_connect.datatypes.string import String +from clickhouse_connect.datatypes.temporal import DateTime, DateTime64 + +from feast import RepoConfig, ValueType +from feast.data_source import DataSource +from feast.errors import DataSourceNoNameException +from feast.infra.utils.clickhouse.connection_utils import get_client +from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from feast.protos.feast.core.SavedDataset_pb2 import ( + SavedDatasetStorage as SavedDatasetStorageProto, +) +from feast.saved_dataset import SavedDatasetStorage + + +class ClickhouseOptions: + def __init__( + self, + name: Optional[str], + query: Optional[str], + table: Optional[str], + ): + self._name = name or "" + self._query = query or "" + self._table = table or "" + + @classmethod + def from_proto(cls, clickhouse_options_proto: DataSourceProto.CustomSourceOptions): + config = json.loads(clickhouse_options_proto.configuration.decode("utf8")) + postgres_options = cls( + name=config["name"], query=config["query"], table=config["table"] + ) + + return postgres_options + + def to_proto(self) -> DataSourceProto.CustomSourceOptions: + clickhouse_options_proto = DataSourceProto.CustomSourceOptions( + configuration=json.dumps( + {"name": self._name, "query": self._query, "table": self._table} + ).encode() + ) + return clickhouse_options_proto + + +class ClickhouseSource(DataSource): + def __init__( + self, + name: str | None = None, + query: str | None = None, + table: str | None = None, + timestamp_field: str | None = "", + created_timestamp_column: str | None = "", + field_mapping: dict[str, str] | None = None, + description: str | None = "", + tags: dict[str, str] | None = None, + owner: str | None = "", + ): + self._clickhouse_options = ClickhouseOptions( + name=name, query=query, table=table + ) + + if name is None and table is None: + raise DataSourceNoNameException() + name = name or table + assert name + + super().__init__( + name=name, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + description=description, + tags=tags, + owner=owner, + ) + + @staticmethod + def from_proto(data_source: DataSourceProto) -> Any: + assert data_source.HasField("custom_options") + + postgres_options = json.loads(data_source.custom_options.configuration) + + return ClickhouseSource( + name=postgres_options["name"], + query=postgres_options["query"], + table=postgres_options["table"], + field_mapping=dict(data_source.field_mapping), + timestamp_field=data_source.timestamp_field, + created_timestamp_column=data_source.created_timestamp_column, + description=data_source.description, + tags=dict(data_source.tags), + owner=data_source.owner, + ) + + def to_proto(self) -> DataSourceProto: + data_source_proto = DataSourceProto( + name=self.name, + type=DataSourceProto.CUSTOM_SOURCE, + data_source_class_type="feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source.ClickhouseSource", + field_mapping=self.field_mapping, + custom_options=self._clickhouse_options.to_proto(), + description=self.description, + tags=self.tags, + owner=self.owner, + ) + + data_source_proto.timestamp_field = self.timestamp_field + data_source_proto.created_timestamp_column = self.created_timestamp_column + + return data_source_proto + + def validate(self, config: RepoConfig): + pass + + @staticmethod + def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: + return ch_type_to_feast_value_type + + def get_table_column_names_and_types( + self, config: RepoConfig + ) -> Iterable[Tuple[str, str]]: + with get_client(config.offline_store) as client: + result = client.query( + f"SELECT * FROM {self.get_table_query_string()} AS sub LIMIT 0" + ) + column_types = list(zip(result.column_names, result.column_types)) + return [ + (name, _ch_type_to_ch_type_str(type_)) for name, type_ in column_types + ] + + def get_table_query_string(self) -> str: + if self._clickhouse_options._table: + return f"{self._clickhouse_options._table}" + else: + return f"({self._clickhouse_options._query})" + + +class SavedDatasetClickhouseStorage(SavedDatasetStorage): + _proto_attr_name = "custom_storage" + + clickhouse_options: ClickhouseOptions + + def __init__(self, table_ref: str): + self.clickhouse_options = ClickhouseOptions( + table=table_ref, name=None, query=None + ) + + @staticmethod + def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage: + return SavedDatasetClickhouseStorage( + table_ref=ClickhouseOptions.from_proto(storage_proto.custom_storage)._table + ) + + def to_proto(self) -> SavedDatasetStorageProto: + return SavedDatasetStorageProto( + custom_storage=self.clickhouse_options.to_proto() + ) + + def to_data_source(self) -> DataSource: + return ClickhouseSource(table=self.clickhouse_options._table) + + +def ch_type_to_feast_value_type(type_: str) -> ValueType: + type_obj = get_from_name(type_) + type_ = type(type_obj) + container_type = None + if isinstance(type_obj, Array): + container_type = Array + type_map: dict[ + tuple[Type[ClickHouseType] | None, Type[ClickHouseType]], ValueType + ] = { + (None, Boolean): ValueType.BOOL, + (None, String): ValueType.STRING, + (None, Float32): ValueType.FLOAT, + (None, Float64): ValueType.DOUBLE, + (None, Decimal): ValueType.DOUBLE, + (None, Int32): ValueType.INT32, + (None, Int64): ValueType.INT64, + (None, DateTime): ValueType.UNIX_TIMESTAMP, + (None, DateTime64): ValueType.UNIX_TIMESTAMP, + (Array, Boolean): ValueType.BOOL_LIST, + (Array, String): ValueType.STRING_LIST, + (Array, Float32): ValueType.FLOAT_LIST, + (Array, Float64): ValueType.DOUBLE_LIST, + (Array, Decimal): ValueType.DOUBLE_LIST, + (Array, Int32): ValueType.INT32_LIST, + (Array, Int64): ValueType.INT64_LIST, + (Array, DateTime): ValueType.UNIX_TIMESTAMP_LIST, + (Array, DateTime64): ValueType.UNIX_TIMESTAMP_LIST, + } + value = type_map.get((container_type, type_), ValueType.UNKNOWN) + if value == ValueType.UNKNOWN: + print("unknown type:", type_) + return value + + +def _ch_type_to_ch_type_str(type_: ClickHouseType) -> str: + return type_.name + # return type_ # type: ignore diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/__init__.py new file mode 100644 index 00000000000..0dd9d78e888 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/__init__.py @@ -0,0 +1 @@ +from .data_source import clickhouse_container # noqa diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py new file mode 100644 index 00000000000..aefb499dd54 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py @@ -0,0 +1,142 @@ +import logging +from typing import Dict, Literal, Optional + +import pandas as pd +import pytest +from testcontainers.clickhouse import ClickHouseContainer +from testcontainers.core.waiting_utils import wait_for_logs + +from feast.data_source import DataSource +from feast.feature_logging import LoggingDestination +from feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse import ( + ClickhouseOfflineStoreConfig, + df_to_clickhouse_table, +) +from feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source import ( + ClickhouseSource, +) +from feast.infra.utils.clickhouse.clickhouse_config import ClickhouseConfig +from feast.infra.utils.clickhouse.connection_utils import get_client +from feast.saved_dataset import SavedDatasetStorage +from tests.integration.feature_repos.universal.data_source_creator import ( + DataSourceCreator, +) +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + +logger = logging.getLogger(__name__) + +CLICKHOUSE_USER = "default" +CLICKHOUSE_PASSWORD = "password" +CLICKHOUSE_OFFLINE_DB = "default" +CLICKHOUSE_ONLINE_DB = "default_online" + + +class ClickhouseOnlineStoreConfig(ClickhouseConfig): + type: Literal[ + "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" + ] = "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" + + +@pytest.fixture(scope="session") +def clickhouse_container(): + container = ClickHouseContainer( + username=CLICKHOUSE_USER, + password=CLICKHOUSE_PASSWORD, + dbname=CLICKHOUSE_OFFLINE_DB, + ) + container.start() + + log_string_to_wait_for = "Logging errors to" + waited = wait_for_logs( + container=container, + predicate=log_string_to_wait_for, + timeout=30, + interval=10, + ) + logger.info("Waited for %s seconds until clickhouse container was up", waited) + + yield container + container.stop() + + +class ClickhouseDataSourceCreator(DataSourceCreator, OnlineStoreCreator): + def create_logged_features_destination(self) -> LoggingDestination: + return None # type: ignore + + def __init__( + self, project_name: str, fixture_request: pytest.FixtureRequest, **kwargs + ): + super().__init__( + project_name, + ) + + self.project_name = project_name + self.container = fixture_request.getfixturevalue("clickhouse_container") + if not self.container: + raise RuntimeError( + "In order to use this data source " + "'feast.infra.offline_stores.contrib.clickhouse_offline_store.tests' " + "must be include into pytest plugins" + ) + + self.offline_store_config = ClickhouseOfflineStoreConfig( + type="feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore", + host="localhost", + port=self.container.get_exposed_port(8123), + database=CLICKHOUSE_OFFLINE_DB, + user=CLICKHOUSE_USER, + password=CLICKHOUSE_PASSWORD, + ) + + def create_data_source( + self, + df: pd.DataFrame, + destination_name: str, + created_timestamp_column="created_ts", + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", + ) -> DataSource: + destination_name = self.get_prefixed_table_name(destination_name) + + if self.offline_store_config: + df_to_clickhouse_table( + self.offline_store_config, df, destination_name, timestamp_field + ) + return ClickhouseSource( + name=destination_name, + query=f"SELECT * FROM {destination_name}", + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping or {"ts_1": "ts"}, + ) + + def create_offline_store_config(self) -> ClickhouseOfflineStoreConfig: + assert self.offline_store_config + return self.offline_store_config + + def get_prefixed_table_name(self, suffix: str) -> str: + return f"{self.project_name}_{suffix}" + + def create_online_store(self) -> ClickhouseOnlineStoreConfig: + assert self.container + + get_client(self.offline_store_config).command( + f"CREATE DATABASE {CLICKHOUSE_ONLINE_DB}" + ) + + return ClickhouseOnlineStoreConfig( + type="feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore", + host="localhost", + port=self.container.get_exposed_port(8123), + database=CLICKHOUSE_ONLINE_DB, + user=CLICKHOUSE_USER, + password=CLICKHOUSE_PASSWORD, + ) + + def create_saved_dataset_destination(self) -> SavedDatasetStorage: + return None + + def teardown(self): + pass diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_repo_configuration.py new file mode 100644 index 00000000000..5c9d4461b16 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_repo_configuration.py @@ -0,0 +1,11 @@ +from feast.infra.offline_stores.contrib.clickhouse_offline_store.tests.data_source import ( + ClickhouseDataSourceCreator, +) +from tests.integration.feature_repos.repo_configuration import REDIS_CONFIG +from tests.integration.feature_repos.universal.online_store.redis import ( + RedisOnlineStoreCreator, +) + +AVAILABLE_OFFLINE_STORES = [("local", ClickhouseDataSourceCreator)] + +AVAILABLE_ONLINE_STORES = {"redis": (REDIS_CONFIG, RedisOnlineStoreCreator)} diff --git a/sdk/python/feast/infra/utils/clickhouse/__init__.py b/sdk/python/feast/infra/utils/clickhouse/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py b/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py new file mode 100644 index 00000000000..a9facc93c0d --- /dev/null +++ b/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py @@ -0,0 +1,12 @@ +from pydantic import StrictStr + +from feast.repo_config import FeastConfigBaseModel + + +class ClickhouseConfig(FeastConfigBaseModel, frozen=True): + host: StrictStr + port: int = 8123 + database: StrictStr + user: StrictStr + password: StrictStr + use_temporary_tables_for_entity_df: bool = True diff --git a/sdk/python/feast/infra/utils/clickhouse/connection_utils.py b/sdk/python/feast/infra/utils/clickhouse/connection_utils.py new file mode 100644 index 00000000000..e60922e478d --- /dev/null +++ b/sdk/python/feast/infra/utils/clickhouse/connection_utils.py @@ -0,0 +1,18 @@ +from functools import cache + +import clickhouse_connect +from clickhouse_connect.driver import Client + +from feast.infra.utils.clickhouse.clickhouse_config import ClickhouseConfig + + +@cache +def get_client(config: ClickhouseConfig) -> Client: + client = clickhouse_connect.get_client( + host=config.host, + port=config.port, + user=config.user, + password=config.password, + database=config.database, + ) + return client diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 4d5d8a71885..2ff616443ac 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.15.2 # via feast (setup.py) aiohappyeyeballs==2.4.3 @@ -40,8 +40,6 @@ async-lru==2.0.4 # via jupyterlab async-property==0.2.2 # via python-keycloak -async-timeout==4.0.3 - # via redis atpublic==5.0 # via ibis-framework attrs==24.2.0 @@ -88,6 +86,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2024.8.30 # via + # clickhouse-connect # elastic-transport # httpcore # httpx @@ -114,6 +113,8 @@ click==8.1.7 # great-expectations # pip-tools # uvicorn +clickhouse-connect==0.8.5 + # via feast (setup.py) cloudpickle==3.1.0 # via dask colorama==0.4.6 @@ -422,6 +423,8 @@ kubernetes==20.13.0 # via feast (setup.py) locket==1.0.0 # via partd +lz4==4.3.3 + # via clickhouse-connect makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -732,6 +735,7 @@ python-keycloak==4.2.2 # via feast (setup.py) pytz==2024.2 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -997,6 +1001,7 @@ urllib3==2.2.3 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1052,3 +1057,5 @@ yarl==1.16.0 # via aiohttp zipp==3.20.2 # via importlib-metadata +zstandard==0.23.0 + # via clickhouse-connect diff --git a/setup.py b/setup.py index b335d39c2b3..f410bf4ef11 100644 --- a/setup.py +++ b/setup.py @@ -148,6 +148,8 @@ QDRANT_REQUIRED = ["qdrant-client>=1.12.0"] +CLICKHOUSE_REQUIRED = ["clickhouse-connect>=0.7.19"] + CI_REQUIRED = ( [ "build", @@ -217,6 +219,7 @@ + OPENTELEMETRY + FAISS_REQUIRED + QDRANT_REQUIRED + + CLICKHOUSE_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -287,7 +290,8 @@ "singlestore": SINGLESTORE_REQUIRED, "opentelemetry": OPENTELEMETRY, "faiss": FAISS_REQUIRED, - "qdrant": QDRANT_REQUIRED + "qdrant": QDRANT_REQUIRED, + "clickhouse": CLICKHOUSE_REQUIRED, }, include_package_data=True, license="Apache", From 1d0417eed44bd316e0d9a79e6efd4ceb4046494d Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 11:33:19 +0100 Subject: [PATCH 02/22] Remove untested `pull_all_from_table_or_query` Signed-off-by: Tomasz Wrona --- .../clickhouse_offline_store/clickhouse.py | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index 3ec09601852..36d3928196c 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -193,40 +193,6 @@ def pull_latest_from_table_or_query( on_demand_feature_views=None, ) - @staticmethod - def pull_all_from_table_or_query( - config: RepoConfig, - data_source: DataSource, - join_key_columns: List[str], - feature_name_columns: List[str], - timestamp_field: str, - start_date: datetime, - end_date: datetime, - ) -> RetrievalJob: - raise NotImplementedError("Not tested (not needed?)") - assert isinstance(config.offline_store, ClickhouseOfflineStoreConfig) - assert isinstance(data_source, ClickhouseSource) - from_expression = data_source.get_table_query_string() - - field_string = ", ".join( - join_key_columns + feature_name_columns + [timestamp_field] - ) - - query = f""" - SELECT {field_string} - FROM {from_expression} AS paftoq_alias - WHERE "{timestamp_field}" - BETWEEN toDateTime64('{start_date.replace(tzinfo=None)!s}', 6, '{start_date.tzinfo!s}') - AND toDateTime64('{end_date.replace(tzinfo=None)!s}', 6, '{end_date.tzinfo!s}') - """ - - return ClickhouseRetrievalJob( - query=query, - config=config, - full_feature_names=False, - on_demand_feature_views=None, - ) - class ClickhouseRetrievalJob(PostgreSQLRetrievalJob): def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: From 3af18897397c50f1b4f74fc018e3d51e19c4c114 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 11:39:24 +0100 Subject: [PATCH 03/22] Reorder functions Signed-off-by: Tomasz Wrona --- .../clickhouse_offline_store/clickhouse.py | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index 36d3928196c..67a85573fd4 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -338,6 +338,37 @@ def _df_to_create_table_schema(entity_df: pd.DataFrame) -> str: return ", ".join(columns) +def arrow_to_ch_type(t_str: str, nullable: bool) -> str: + list_pattern = r"list" + list_res = re.search(list_pattern, t_str) + if list_res is not None: + item_type_str = list_res.group(1) + return f"Array({arrow_to_ch_type(item_type_str, nullable)})" + + if nullable: + return f"Nullable({arrow_to_ch_type(t_str, nullable=False)})" + + try: + if t_str.startswith("timestamp"): + return _arrow_to_ch_timestamp_type(t_str) + return { + "bool": "Boolean", + "int8": "Int8", + "int16": "Int16", + "int32": "Int32", + "int64": "Int64", + "uint8": "UInt8", + "uint16": "UInt16", + "uint32": "UInt32", + "uint64": "Uint64", + "float": "Float32", + "double": "Float64", + "string": "String", + }[t_str] + except KeyError: + raise ValueError(f"Unsupported type: {t_str}") + + def _arrow_to_ch_timestamp_type(t_str: str) -> str: _ARROW_PRECISION_TO_CH_PRECISION = { "s": 0, @@ -379,37 +410,6 @@ def _arrow_to_ch_timestamp_type(t_str: str) -> str: return "DateTime" -def arrow_to_ch_type(t_str: str, nullable: bool) -> str: - list_pattern = r"list" - list_res = re.search(list_pattern, t_str) - if list_res is not None: - item_type_str = list_res.group(1) - return f"Array({arrow_to_ch_type(item_type_str, nullable)})" - - if nullable: - return f"Nullable({arrow_to_ch_type(t_str, nullable=False)})" - - try: - if t_str.startswith("timestamp"): - return _arrow_to_ch_timestamp_type(t_str) - return { - "bool": "Boolean", - "int8": "Int8", - "int16": "Int16", - "int32": "Int32", - "int64": "Int64", - "uint8": "UInt8", - "uint16": "UInt16", - "uint32": "UInt32", - "uint64": "Uint64", - "float": "Float32", - "double": "Float64", - "string": "String", - }[t_str] - except KeyError: - raise ValueError(f"Unsupported type: {t_str}") - - def _append_alias(field_names: List[str], alias: str) -> List[str]: return [f'{alias}."{field_name}"' for field_name in field_names] From 5d7f173d33c91c21c9c0409b97fe480fdca394b9 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 11:39:33 +0100 Subject: [PATCH 04/22] Remove commented line Signed-off-by: Tomasz Wrona --- .../contrib/clickhouse_offline_store/clickhouse_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py index ab85233c64c..dde03eabc4d 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py @@ -209,4 +209,3 @@ def ch_type_to_feast_value_type(type_: str) -> ValueType: def _ch_type_to_ch_type_str(type_: ClickHouseType) -> str: return type_.name - # return type_ # type: ignore From 9d9d24a90ee5e153739a7a4df3e937442e21cb9d Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 11:46:55 +0100 Subject: [PATCH 05/22] Fix frozen mypy errors Signed-off-by: Tomasz Wrona --- .../feast/infra/utils/clickhouse/clickhouse_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py b/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py index a9facc93c0d..c6b5142b7a4 100644 --- a/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py +++ b/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py @@ -1,12 +1,14 @@ -from pydantic import StrictStr +from pydantic import StrictStr, ConfigDict from feast.repo_config import FeastConfigBaseModel -class ClickhouseConfig(FeastConfigBaseModel, frozen=True): +class ClickhouseConfig(FeastConfigBaseModel): host: StrictStr port: int = 8123 database: StrictStr user: StrictStr password: StrictStr use_temporary_tables_for_entity_df: bool = True + + model_config = ConfigDict(frozen=True) From 8207c44cd379eed772382b2d02601366fa7f86a0 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 12:12:34 +0100 Subject: [PATCH 06/22] mypy fixes; remove online source creator Signed-off-by: Tomasz Wrona --- .../clickhouse_offline_store/clickhouse.py | 7 +++-- .../clickhouse_source.py | 10 +++---- .../tests/data_source.py | 29 ++++--------------- .../utils/clickhouse/clickhouse_config.py | 2 +- 4 files changed, 15 insertions(+), 33 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index 67a85573fd4..e06b75f881f 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -382,21 +382,22 @@ def _arrow_to_ch_timestamp_type(t_str: str) -> str: unit = unit.strip() precision = _ARROW_PRECISION_TO_CH_PRECISION[unit] - tz: Optional[str] = None if len(rest): - tz: str = rest[0] + tz = rest[0] tz = ( tz.strip() .removeprefix("tz=") .translate( str.maketrans( - { + { # type: ignore[arg-type] "'": None, '"': None, } ) ) ) + else: + tz = None if precision > 0: if tz is not None: diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py index dde03eabc4d..89d2f96222e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py @@ -173,9 +173,9 @@ def to_data_source(self) -> DataSource: return ClickhouseSource(table=self.clickhouse_options._table) -def ch_type_to_feast_value_type(type_: str) -> ValueType: - type_obj = get_from_name(type_) - type_ = type(type_obj) +def ch_type_to_feast_value_type(type_str: str) -> ValueType: + type_obj = get_from_name(type_str) + type_cls = type(type_obj) container_type = None if isinstance(type_obj, Array): container_type = Array @@ -201,9 +201,9 @@ def ch_type_to_feast_value_type(type_: str) -> ValueType: (Array, DateTime): ValueType.UNIX_TIMESTAMP_LIST, (Array, DateTime64): ValueType.UNIX_TIMESTAMP_LIST, } - value = type_map.get((container_type, type_), ValueType.UNKNOWN) + value = type_map.get((container_type, type_cls), ValueType.UNKNOWN) if value == ValueType.UNKNOWN: - print("unknown type:", type_) + print("unknown type:", type_str) return value diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py index aefb499dd54..d84617bfefa 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py @@ -16,14 +16,9 @@ ClickhouseSource, ) from feast.infra.utils.clickhouse.clickhouse_config import ClickhouseConfig -from feast.infra.utils.clickhouse.connection_utils import get_client -from feast.saved_dataset import SavedDatasetStorage from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) -from tests.integration.feature_repos.universal.online_store_creator import ( - OnlineStoreCreator, -) logger = logging.getLogger(__name__) @@ -61,7 +56,7 @@ def clickhouse_container(): container.stop() -class ClickhouseDataSourceCreator(DataSourceCreator, OnlineStoreCreator): +class ClickhouseDataSourceCreator(DataSourceCreator): def create_logged_features_destination(self) -> LoggingDestination: return None # type: ignore @@ -101,6 +96,8 @@ def create_data_source( destination_name = self.get_prefixed_table_name(destination_name) if self.offline_store_config: + if timestamp_field is None: + timestamp_field = "ts" df_to_clickhouse_table( self.offline_store_config, df, destination_name, timestamp_field ) @@ -119,24 +116,8 @@ def create_offline_store_config(self) -> ClickhouseOfflineStoreConfig: def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}_{suffix}" - def create_online_store(self) -> ClickhouseOnlineStoreConfig: - assert self.container - - get_client(self.offline_store_config).command( - f"CREATE DATABASE {CLICKHOUSE_ONLINE_DB}" - ) - - return ClickhouseOnlineStoreConfig( - type="feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore", - host="localhost", - port=self.container.get_exposed_port(8123), - database=CLICKHOUSE_ONLINE_DB, - user=CLICKHOUSE_USER, - password=CLICKHOUSE_PASSWORD, - ) - - def create_saved_dataset_destination(self) -> SavedDatasetStorage: - return None + def create_saved_dataset_destination(self): + pass def teardown(self): pass diff --git a/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py b/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py index c6b5142b7a4..1f163e0a81b 100644 --- a/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py +++ b/sdk/python/feast/infra/utils/clickhouse/clickhouse_config.py @@ -1,4 +1,4 @@ -from pydantic import StrictStr, ConfigDict +from pydantic import ConfigDict, StrictStr from feast.repo_config import FeastConfigBaseModel From 7aa45d6d64597a652f1f0a2a957b49bbe7696dcc Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 12:12:52 +0100 Subject: [PATCH 07/22] Remove commented code Signed-off-by: Tomasz Wrona --- .../contrib/clickhouse_offline_store/clickhouse.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index e06b75f881f..e9b27899686 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -306,11 +306,6 @@ def df_to_clickhouse_table( table_name: str, entity_timestamp_col: str, ) -> None: - # pd_dtypes = df.dtypes.to_dict() - # ch_dtypes = convert_pd_dtypes_to_ch(pd_dtypes) - # table_schema = ", ".join( - # [f'"{col}" {ch_dtype}' for col, ch_dtype in ch_dtypes.items()] - # ) table_schema = _df_to_create_table_schema(df) if config.use_temporary_tables_for_entity_df: query = f"CREATE TEMPORARY TABLE {table_name} ({table_schema})" @@ -328,13 +323,10 @@ def df_to_clickhouse_table( def _df_to_create_table_schema(entity_df: pd.DataFrame) -> str: pa_table = pa.Table.from_pandas(entity_df) - # print(f"{pa_table.schema=}") - # print(f"{entity_df=}") columns = [ f""""{f.name}" {arrow_to_ch_type(str(f.type), f.nullable)}""" for f in pa_table.schema ] - # print(f"{columns=}") return ", ".join(columns) From 803752f1566476c07cb862eb7aa3524e12e06c62 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 14:13:12 +0100 Subject: [PATCH 08/22] Added docs Signed-off-by: Tomasz Wrona --- docs/SUMMARY.md | 1 + docs/reference/data-sources/README.md | 4 ++ docs/reference/data-sources/clickhouse.md | 36 ++++++++++ docs/reference/offline-stores/clickhouse.md | 69 +++++++++++++++++++ ...tores.contrib.clickhouse_offline_store.rst | 37 ++++++++++ ...contrib.clickhouse_offline_store.tests.rst | 21 ++++++ .../feast.infra.offline_stores.contrib.rst | 9 +++ .../feast.infra.online_stores.contrib.rst | 30 +++++--- .../docs/source/feast.infra.online_stores.rst | 8 +++ sdk/python/docs/source/feast.infra.rst | 8 +++ .../source/feast.infra.utils.clickhouse.rst | 29 ++++++++ sdk/python/docs/source/feast.infra.utils.rst | 1 + .../docs/source/feast.permissions.client.rst | 14 +++- .../docs/source/feast.protos.feast.core.rst | 16 +++++ sdk/python/docs/source/feast.rst | 32 +++++++++ 15 files changed, 301 insertions(+), 14 deletions(-) create mode 100644 docs/reference/data-sources/clickhouse.md create mode 100644 docs/reference/offline-stores/clickhouse.md create mode 100644 sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.rst create mode 100644 sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.tests.rst create mode 100644 sdk/python/docs/source/feast.infra.utils.clickhouse.rst diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index b7faf526c28..9d062bfd971 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -93,6 +93,7 @@ * [PostgreSQL (contrib)](reference/offline-stores/postgres.md) * [Trino (contrib)](reference/offline-stores/trino.md) * [Azure Synapse + Azure SQL (contrib)](reference/offline-stores/mssql.md) + * [Clickhouse (contrib)](reference/offline-stores/clickhouse.md) * [Remote Offline](reference/offline-stores/remote-offline-store.md) * [Online stores](reference/online-stores/README.md) * [Overview](reference/online-stores/overview.md) diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index e69fbab8e36..d0031e84c90 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -49,3 +49,7 @@ Please see [Data Source](../../getting-started/concepts/data-ingestion.md) for a {% content-ref url="mssql.md" %} [mssql.md](mssql.md) {% endcontent-ref %} + +{% content-ref url="clickhouse.md" %} +[clickhouse.md](clickhouse.md) +{% endcontent-ref %} diff --git a/docs/reference/data-sources/clickhouse.md b/docs/reference/data-sources/clickhouse.md new file mode 100644 index 00000000000..7630d5dd14a --- /dev/null +++ b/docs/reference/data-sources/clickhouse.md @@ -0,0 +1,36 @@ +# Clickhouse source (contrib) + +## Description + +Clickhouse data sources are Clickhouse tables or views. +These can be specified either by a table reference or a SQL query. + +## Disclaimer + +The Clickhouse data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a Clickhouse source: + +```python +from feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source import ( + ClickhouseSource, +) + +driver_stats_source = ClickhouseSource( + name="feast_driver_hourly_stats", + query="SELECT * FROM feast_driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source.ClickhouseSource). + +## Supported Types + +Clickhouse data sources support all eight primitive types and their corresponding array types. +The support for Clickhouse Decimal type is achieved by converting it to double. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/clickhouse.md b/docs/reference/offline-stores/clickhouse.md new file mode 100644 index 00000000000..317d6e23e1e --- /dev/null +++ b/docs/reference/offline-stores/clickhouse.md @@ -0,0 +1,69 @@ +# Clickhouse offline store (contrib) + +## Description + +The Clickhouse offline store provides support for reading [ClickhouseSource](../data-sources/clickhouse.md). +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Clickhouse as a table (temporary table by default) in order to complete join operations. + +## Disclaimer + +The Clickhouse offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[clickhouse]'`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD + use_temporary_tables_for_entity_df: true +online_store: + path: data/online_store.db +``` +{% endcode %} + +Note that `use_temporary_tables_for_entity_df` is an optional parameter. +The full set of configuration options is available in [ClickhouseOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Clickhouse offline store. + +| | Clickhouse | +| :----------------------------------------------------------------- |:-----------| +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | no | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `ClickhouseRetrievalJob`. + +| | Clickhouse | +| ----------------------------------------------------- |------------| +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.rst new file mode 100644 index 00000000000..b593da3115b --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.rst @@ -0,0 +1,37 @@ +feast.infra.offline\_stores.contrib.clickhouse\_offline\_store package +====================================================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.clickhouse_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.clickhouse\_offline\_store.clickhouse module +-------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.clickhouse\_offline\_store.clickhouse\_source module +---------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.clickhouse_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.tests.rst new file mode 100644 index 00000000000..0b1265e438c --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.clickhouse_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.clickhouse\_offline\_store.tests package +============================================================================ + +Submodules +---------- + +feast.infra.offline\_stores.contrib.clickhouse\_offline\_store.tests.data\_source module +---------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.clickhouse_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.clickhouse_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst index ec74ddab05c..63a8f65ea61 100644 --- a/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst @@ -8,6 +8,7 @@ Subpackages :maxdepth: 4 feast.infra.offline_stores.contrib.athena_offline_store + feast.infra.offline_stores.contrib.clickhouse_offline_store feast.infra.offline_stores.contrib.mssql_offline_store feast.infra.offline_stores.contrib.postgres_offline_store feast.infra.offline_stores.contrib.spark_offline_store @@ -24,6 +25,14 @@ feast.infra.offline\_stores.contrib.athena\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.offline\_stores.contrib.clickhouse\_repo\_configuration module +-------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.clickhouse_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.offline\_stores.contrib.mssql\_repo\_configuration module --------------------------------------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index 2403b5b8d48..c89bec323d6 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -40,18 +40,10 @@ feast.infra.online\_stores.contrib.elasticsearch\_repo\_configuration module :undoc-members: :show-inheritance: -feast.infra.online\_stores.contrib.qdrant module -------------------------------------------------------- +feast.infra.online\_stores.contrib.faiss\_online\_store module +-------------------------------------------------------------- -.. automodule:: feast.infra.online_stores.contrib.qdrant - :members: - :undoc-members: - :show-inheritance: - -feast.infra.online\_stores.contrib.qdrant\_repo\_configuration module ----------------------------------------------------------------------------- - -.. automodule:: feast.infra.online_stores.contrib.qdrant_repo_configuration +.. automodule:: feast.infra.online_stores.contrib.faiss_online_store :members: :undoc-members: :show-inheritance: @@ -104,6 +96,22 @@ feast.infra.online\_stores.contrib.postgres\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.qdrant module +------------------------------------------------ + +.. automodule:: feast.infra.online_stores.contrib.qdrant + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.qdrant\_repo\_configuration module +--------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.qdrant_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.singlestore\_repo\_configuration module -------------------------------------------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.online_stores.rst b/sdk/python/docs/source/feast.infra.online_stores.rst index 801d187a7c8..608e9fef6ae 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.rst @@ -84,6 +84,14 @@ feast.infra.online\_stores.sqlite module :undoc-members: :show-inheritance: +feast.infra.online\_stores.vector\_store module +----------------------------------------------- + +.. automodule:: feast.infra.online_stores.vector_store + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index b0046a2719e..791a4ace832 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -51,6 +51,14 @@ feast.infra.provider module :undoc-members: :show-inheritance: +feast.infra.supported\_async\_methods module +-------------------------------------------- + +.. automodule:: feast.infra.supported_async_methods + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/sdk/python/docs/source/feast.infra.utils.clickhouse.rst b/sdk/python/docs/source/feast.infra.utils.clickhouse.rst new file mode 100644 index 00000000000..4fa3e2a0cbd --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.clickhouse.rst @@ -0,0 +1,29 @@ +feast.infra.utils.clickhouse package +==================================== + +Submodules +---------- + +feast.infra.utils.clickhouse.clickhouse\_config module +------------------------------------------------------ + +.. automodule:: feast.infra.utils.clickhouse.clickhouse_config + :members: + :undoc-members: + :show-inheritance: + +feast.infra.utils.clickhouse.connection\_utils module +----------------------------------------------------- + +.. automodule:: feast.infra.utils.clickhouse.connection_utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.utils.clickhouse + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.utils.rst b/sdk/python/docs/source/feast.infra.utils.rst index 083259bfaae..6c23f320b8c 100644 --- a/sdk/python/docs/source/feast.infra.utils.rst +++ b/sdk/python/docs/source/feast.infra.utils.rst @@ -7,6 +7,7 @@ Subpackages .. toctree:: :maxdepth: 4 + feast.infra.utils.clickhouse feast.infra.utils.postgres feast.infra.utils.snowflake diff --git a/sdk/python/docs/source/feast.permissions.client.rst b/sdk/python/docs/source/feast.permissions.client.rst index f3468012106..84e58bdc2d9 100644 --- a/sdk/python/docs/source/feast.permissions.client.rst +++ b/sdk/python/docs/source/feast.permissions.client.rst @@ -20,10 +20,10 @@ feast.permissions.client.auth\_client\_manager module :undoc-members: :show-inheritance: -feast.permissions.client.auth\_client\_manager\_factory module --------------------------------------------------------------- +feast.permissions.client.client\_auth\_token module +--------------------------------------------------- -.. automodule:: feast.permissions.client.auth_client_manager_factory +.. automodule:: feast.permissions.client.client_auth_token :members: :undoc-members: :show-inheritance: @@ -44,6 +44,14 @@ feast.permissions.client.http\_auth\_requests\_wrapper module :undoc-members: :show-inheritance: +feast.permissions.client.intra\_comm\_authentication\_client\_manager module +---------------------------------------------------------------------------- + +.. automodule:: feast.permissions.client.intra_comm_authentication_client_manager + :members: + :undoc-members: + :show-inheritance: + feast.permissions.client.kubernetes\_auth\_client\_manager module ----------------------------------------------------------------- diff --git a/sdk/python/docs/source/feast.protos.feast.core.rst b/sdk/python/docs/source/feast.protos.feast.core.rst index 9d079953c1e..78398e54dcb 100644 --- a/sdk/python/docs/source/feast.protos.feast.core.rst +++ b/sdk/python/docs/source/feast.protos.feast.core.rst @@ -244,6 +244,22 @@ feast.protos.feast.core.Policy\_pb2\_grpc module :undoc-members: :show-inheritance: +feast.protos.feast.core.Project\_pb2 module +------------------------------------------- + +.. automodule:: feast.protos.feast.core.Project_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Project\_pb2\_grpc module +------------------------------------------------- + +.. automodule:: feast.protos.feast.core.Project_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + feast.protos.feast.core.Registry\_pb2 module -------------------------------------------- diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst index b8c04ebde6b..ea34c3d8dd9 100644 --- a/sdk/python/docs/source/feast.rst +++ b/sdk/python/docs/source/feast.rst @@ -28,6 +28,14 @@ feast.aggregation module :undoc-members: :show-inheritance: +feast.arrow\_error\_handler module +---------------------------------- + +.. automodule:: feast.arrow_error_handler + :members: + :undoc-members: + :show-inheritance: + feast.base\_feature\_view module -------------------------------- @@ -196,6 +204,14 @@ feast.flags\_helper module :undoc-members: :show-inheritance: +feast.grpc\_error\_interceptor module +------------------------------------- + +.. automodule:: feast.grpc_error_interceptor + :members: + :undoc-members: + :show-inheritance: + feast.importer module --------------------- @@ -244,6 +260,14 @@ feast.online\_response module :undoc-members: :show-inheritance: +feast.project module +-------------------- + +.. automodule:: feast.project + :members: + :undoc-members: + :show-inheritance: + feast.project\_metadata module ------------------------------ @@ -292,6 +316,14 @@ feast.repo\_operations module :undoc-members: :show-inheritance: +feast.rest\_error\_handler module +--------------------------------- + +.. automodule:: feast.rest_error_handler + :members: + :undoc-members: + :show-inheritance: + feast.saved\_dataset module --------------------------- From 7fffc153b4ed069e0035727b20a6b3518dc17a56 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 14:21:30 +0100 Subject: [PATCH 09/22] Python 3.9 deps Signed-off-by: Tomasz Wrona --- .../requirements/py3.9-ci-requirements.txt | 50 ++++--------------- .../requirements/py3.9-requirements.txt | 13 +---- 2 files changed, 13 insertions(+), 50 deletions(-) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 2ba384e205e..5e349c74356 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.15.2 # via feast (setup.py) aiohappyeyeballs==2.4.3 @@ -40,10 +40,6 @@ async-lru==2.0.4 # via jupyterlab async-property==0.2.2 # via python-keycloak -async-timeout==4.0.3 - # via - # aiohttp - # redis atpublic==4.1.0 # via ibis-framework attrs==24.2.0 @@ -92,6 +88,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2024.8.30 # via + # clickhouse-connect # elastic-transport # httpcore # httpx @@ -118,6 +115,8 @@ click==8.1.7 # great-expectations # pip-tools # uvicorn +clickhouse-connect==0.8.5 + # via feast (setup.py) cloudpickle==3.1.0 # via dask colorama==0.4.6 @@ -180,11 +179,6 @@ elasticsearch==8.15.1 # via feast (setup.py) entrypoints==0.4 # via altair -exceptiongroup==1.2.2 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.1.0 @@ -337,16 +331,7 @@ idna==3.10 imagesize==1.4.1 # via sphinx importlib-metadata==8.5.0 - # via - # build - # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx - # typeguard + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -440,6 +425,8 @@ kubernetes==20.13.0 # via feast (setup.py) locket==1.0.0 # via partd +lz4==4.3.3 + # via clickhouse-connect makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -749,6 +736,7 @@ python-keycloak==4.2.2 # via feast (setup.py) pytz==2024.2 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -914,16 +902,6 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env - # singlestoredb tomlkit==0.13.2 # via snowflake-connector-python toolz==0.12.1 @@ -995,30 +973,22 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via - # aioitertools - # anyio - # async-lru # azure-core # azure-identity # azure-storage-blob # fastapi # great-expectations # ibis-framework - # ipython # jwcrypto - # multidict # mypy # psycopg # psycopg-pool # pydantic # pydantic-core - # rich # snowflake-connector-python # sqlalchemy - # starlette # testcontainers # typeguard - # uvicorn tzdata==2024.2 # via pandas tzlocal==5.2 @@ -1031,6 +1001,7 @@ urllib3==1.26.20 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1039,7 +1010,6 @@ urllib3==1.26.20 # qdrant-client # requests # responses - # snowflake-connector-python # testcontainers uvicorn[standard]==0.32.0 # via @@ -1087,3 +1057,5 @@ yarl==1.16.0 # via aiohttp zipp==3.20.2 # via importlib-metadata +zstandard==0.23.0 + # via clickhouse-connect diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 7f8eecd6f84..7426d83443b 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt annotated-types==0.7.0 # via pydantic anyio==4.6.2.post1 @@ -33,8 +33,6 @@ dask-expr==1.1.10 # via dask dill==0.3.9 # via feast (setup.py) -exceptiongroup==1.2.2 - # via anyio fastapi==0.115.3 # via feast (setup.py) fsspec==2024.10.0 @@ -52,9 +50,7 @@ idna==3.10 # anyio # requests importlib-metadata==8.5.0 - # via - # dask - # typeguard + # via dask jinja2==3.1.4 # via feast (setup.py) jsonschema==4.23.0 @@ -143,8 +139,6 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 - # via mypy toolz==1.0.0 # via # dask @@ -155,15 +149,12 @@ typeguard==4.3.0 # via feast (setup.py) typing-extensions==4.12.2 # via - # anyio # fastapi # mypy # pydantic # pydantic-core # sqlalchemy - # starlette # typeguard - # uvicorn tzdata==2024.2 # via pandas urllib3==2.2.3 From 6e35db8646d2ea5fda2e61ea647153e4ba4a3d4d Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 14:22:39 +0100 Subject: [PATCH 10/22] Python 3.10 deps Signed-off-by: Tomasz Wrona --- .../requirements/py3.10-ci-requirements.txt | 39 ++++++------------- .../requirements/py3.10-requirements.txt | 8 +--- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 8c940ba84e2..32367ee3ee0 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.15.2 # via feast (setup.py) aiohappyeyeballs==2.4.3 @@ -40,10 +40,6 @@ async-lru==2.0.4 # via jupyterlab async-property==0.2.2 # via python-keycloak -async-timeout==4.0.3 - # via - # aiohttp - # redis atpublic==5.0 # via ibis-framework attrs==24.2.0 @@ -90,6 +86,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2024.8.30 # via + # clickhouse-connect # elastic-transport # httpcore # httpx @@ -116,6 +113,8 @@ click==8.1.7 # great-expectations # pip-tools # uvicorn +clickhouse-connect==0.8.5 + # via feast (setup.py) cloudpickle==3.1.0 # via dask colorama==0.4.6 @@ -178,11 +177,6 @@ elasticsearch==8.15.1 # via feast (setup.py) entrypoints==0.4 # via altair -exceptiongroup==1.2.2 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.1.0 @@ -335,9 +329,7 @@ idna==3.10 imagesize==1.4.1 # via sphinx importlib-metadata==8.5.0 - # via - # build - # dask + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -431,6 +423,8 @@ kubernetes==20.13.0 # via feast (setup.py) locket==1.0.0 # via partd +lz4==4.3.3 + # via clickhouse-connect makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -741,6 +735,7 @@ python-keycloak==4.2.2 # via feast (setup.py) pytz==2024.2 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -906,16 +901,6 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env - # singlestoredb tomlkit==0.13.2 # via snowflake-connector-python toolz==0.12.1 @@ -987,8 +972,6 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via - # anyio - # async-lru # azure-core # azure-identity # azure-storage-blob @@ -997,18 +980,15 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto - # multidict # mypy # psycopg # psycopg-pool # pydantic # pydantic-core - # rich # snowflake-connector-python # sqlalchemy # testcontainers # typeguard - # uvicorn tzdata==2024.2 # via pandas tzlocal==5.2 @@ -1021,6 +1001,7 @@ urllib3==2.2.3 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1076,3 +1057,5 @@ yarl==1.16.0 # via aiohttp zipp==3.20.2 # via importlib-metadata +zstandard==0.23.0 + # via clickhouse-connect diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 94c5d3945a9..2b1dce3dde7 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt annotated-types==0.7.0 # via pydantic anyio==4.6.2.post1 @@ -33,8 +33,6 @@ dask-expr==1.1.16 # via dask dill==0.3.9 # via feast (setup.py) -exceptiongroup==1.2.2 - # via anyio fastapi==0.115.3 # via feast (setup.py) fsspec==2024.10.0 @@ -141,8 +139,6 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 - # via mypy toolz==1.0.0 # via # dask @@ -153,14 +149,12 @@ typeguard==4.3.0 # via feast (setup.py) typing-extensions==4.12.2 # via - # anyio # fastapi # mypy # pydantic # pydantic-core # sqlalchemy # typeguard - # uvicorn tzdata==2024.2 # via pandas urllib3==2.2.3 From ca9d795fe6eb16fde78b1af7d87e464c66d504ca Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 14:23:21 +0100 Subject: [PATCH 11/22] Python 3.11 deps (updated) Signed-off-by: Tomasz Wrona --- sdk/python/requirements/py3.11-ci-requirements.txt | 5 ----- sdk/python/requirements/py3.11-requirements.txt | 6 +----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 2ff616443ac..8c36e72d226 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -328,8 +328,6 @@ idna==3.10 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==8.5.0 - # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -978,7 +976,6 @@ typing-extensions==4.12.2 # fastapi # great-expectations # ibis-framework - # ipython # jwcrypto # mypy # psycopg @@ -1055,7 +1052,5 @@ xmltodict==0.14.2 # via moto yarl==1.16.0 # via aiohttp -zipp==3.20.2 - # via importlib-metadata zstandard==0.23.0 # via clickhouse-connect diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index e2a8589e77d..a3c932cc20b 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt annotated-types==0.7.0 # via pydantic anyio==4.6.2.post1 @@ -49,8 +49,6 @@ idna==3.10 # via # anyio # requests -importlib-metadata==8.5.0 - # via dask jinja2==3.1.4 # via feast (setup.py) jsonschema==4.23.0 @@ -171,5 +169,3 @@ watchfiles==0.24.0 # via uvicorn websockets==13.1 # via uvicorn -zipp==3.20.2 - # via importlib-metadata From f8a835608824265c4f6f0496c83cd2e6fa60f148 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 31 Oct 2024 14:34:52 +0100 Subject: [PATCH 12/22] Remove unused ClickhouseOnlineStoreConfig Signed-off-by: Tomasz Wrona --- .../clickhouse_offline_store/tests/data_source.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py index d84617bfefa..8ab4fdb5408 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Literal, Optional +from typing import Dict, Optional import pandas as pd import pytest @@ -15,7 +15,6 @@ from feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source import ( ClickhouseSource, ) -from feast.infra.utils.clickhouse.clickhouse_config import ClickhouseConfig from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) @@ -28,12 +27,6 @@ CLICKHOUSE_ONLINE_DB = "default_online" -class ClickhouseOnlineStoreConfig(ClickhouseConfig): - type: Literal[ - "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" - ] = "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" - - @pytest.fixture(scope="session") def clickhouse_container(): container = ClickHouseContainer( From 7ce7db130df55ad32f30085e5cf76d73431be20c Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Tue, 11 Feb 2025 17:51:01 +0100 Subject: [PATCH 13/22] Regenerate requirements.txt files Signed-off-by: Tomasz Wrona --- .../requirements/py3.10-ci-requirements.txt | 29 ++++++++++++------- .../requirements/py3.10-requirements.txt | 2 +- .../requirements/py3.11-ci-requirements.txt | 29 ++++++++++++------- .../requirements/py3.11-requirements.txt | 2 +- .../requirements/py3.9-ci-requirements.txt | 29 ++++++++++++------- .../requirements/py3.9-requirements.txt | 2 +- 6 files changed, 60 insertions(+), 33 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 96948e78e23..76a32b0e36b 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -2,7 +2,7 @@ # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.19.0 # via feast (setup.py) -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp aiohttp==3.11.12 # via aiobotocore @@ -67,7 +67,7 @@ babel==2.17.0 # sphinx beautifulsoup4==4.13.3 # via nbconvert -bigtree==0.23.1 +bigtree==0.24.0 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert @@ -92,6 +92,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # elastic-transport # httpcore # httpx @@ -118,6 +119,8 @@ click==8.1.8 # great-expectations # pip-tools # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -130,7 +133,7 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.10 +coverage[toml]==7.6.12 # via pytest-cov cryptography==43.0.3 # via @@ -146,7 +149,7 @@ cryptography==43.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -cython==3.0.11 +cython==3.0.12 # via thriftpy2 dask[dataframe]==2025.1.0 # via feast (setup.py) @@ -325,7 +328,7 @@ ibis-framework[duckdb]==9.5.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.6 +identify==2.6.7 # via pre-commit idna==3.10 # via @@ -436,7 +439,9 @@ kubernetes==20.13.0 locket==1.0.0 # via partd lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -573,7 +578,7 @@ pbr==6.1.1 # via mock pexpect==4.9.0 # via ipython -pip==25.0 +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via feast (setup.py) @@ -763,6 +768,7 @@ python-keycloak==4.2.2 # via feast (setup.py) pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -843,7 +849,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.5 +ruff==0.9.6 # via feast (setup.py) s3transfer==0.11.2 # via boto3 @@ -1000,7 +1006,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250110 +types-setuptools==75.8.0.20250210 # via # feast (setup.py) # types-cffi @@ -1051,6 +1057,7 @@ urllib3==2.3.0 # feast (setup.py) # aiobotocore # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1107,4 +1114,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index c25bda58b7e..3f80e0dfd90 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.23.1 +bigtree==0.24.0 # via feast (setup.py) certifi==2025.1.31 # via requests diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 20976b02045..209c474a57d 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -2,7 +2,7 @@ # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.19.0 # via feast (setup.py) -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp aiohttp==3.11.12 # via aiobotocore @@ -65,7 +65,7 @@ babel==2.17.0 # sphinx beautifulsoup4==4.13.3 # via nbconvert -bigtree==0.23.1 +bigtree==0.24.0 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert @@ -90,6 +90,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # elastic-transport # httpcore # httpx @@ -116,6 +117,8 @@ click==8.1.8 # great-expectations # pip-tools # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -128,7 +131,7 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.10 +coverage[toml]==7.6.12 # via pytest-cov cryptography==43.0.3 # via @@ -144,7 +147,7 @@ cryptography==43.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -cython==3.0.11 +cython==3.0.12 # via thriftpy2 dask[dataframe]==2025.1.0 # via feast (setup.py) @@ -318,7 +321,7 @@ ibis-framework[duckdb]==9.5.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.6 +identify==2.6.7 # via pre-commit idna==3.10 # via @@ -427,7 +430,9 @@ kubernetes==20.13.0 locket==1.0.0 # via partd lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -564,7 +569,7 @@ pbr==6.1.1 # via mock pexpect==4.9.0 # via ipython -pip==25.0 +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via feast (setup.py) @@ -754,6 +759,7 @@ python-keycloak==4.2.2 # via feast (setup.py) pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -834,7 +840,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.5 +ruff==0.9.6 # via feast (setup.py) s3transfer==0.11.2 # via boto3 @@ -981,7 +987,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250110 +types-setuptools==75.8.0.20250210 # via # feast (setup.py) # types-cffi @@ -1027,6 +1033,7 @@ urllib3==2.3.0 # feast (setup.py) # aiobotocore # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1083,4 +1090,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index f79776a9147..885ca2f9178 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.23.1 +bigtree==0.24.0 # via feast (setup.py) certifi==2025.1.31 # via requests diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 00eb59c93d2..98179d6a072 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -2,7 +2,7 @@ # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.19.0 # via feast (setup.py) -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp aiohttp==3.11.12 # via aiobotocore @@ -69,7 +69,7 @@ beautifulsoup4==4.13.3 # via nbconvert bidict==0.23.1 # via ibis-framework -bigtree==0.23.1 +bigtree==0.24.0 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert @@ -94,6 +94,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # elastic-transport # httpcore # httpx @@ -120,6 +121,8 @@ click==8.1.8 # great-expectations # pip-tools # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -132,7 +135,7 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.10 +coverage[toml]==7.6.12 # via pytest-cov cryptography==43.0.3 # via @@ -148,7 +151,7 @@ cryptography==43.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -cython==3.0.11 +cython==3.0.12 # via thriftpy2 dask[dataframe]==2024.8.0 # via @@ -331,7 +334,7 @@ ibis-framework[duckdb]==9.0.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.6 +identify==2.6.7 # via pre-commit idna==3.10 # via @@ -449,7 +452,9 @@ kubernetes==20.13.0 locket==1.0.0 # via partd lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -586,7 +591,7 @@ pbr==6.1.1 # via mock pexpect==4.9.0 # via ipython -pip==25.0 +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via feast (setup.py) @@ -776,6 +781,7 @@ python-keycloak==4.2.2 # via feast (setup.py) pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -856,7 +862,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.5 +ruff==0.9.6 # via feast (setup.py) s3transfer==0.11.2 # via boto3 @@ -1013,7 +1019,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250110 +types-setuptools==75.8.0.20250210 # via # feast (setup.py) # types-cffi @@ -1067,6 +1073,7 @@ urllib3==1.26.20 # feast (setup.py) # aiobotocore # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1124,4 +1131,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 1b82c994a1c..8f4a3893b8e 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.23.1 +bigtree==0.24.0 # via feast (setup.py) certifi==2025.1.31 # via requests From bcb90cab4d50bc9a518bfd4738e3a4291139cc4e Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Tue, 11 Feb 2025 17:55:08 +0100 Subject: [PATCH 14/22] Lint & format fixes Signed-off-by: Tomasz Wrona --- .../clickhouse_offline_store/clickhouse.py | 2 +- .../clickhouse_source.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index e9b27899686..3fb559b611c 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -102,7 +102,7 @@ def query_generator() -> Iterator[str]: # Hack for query_context.entity_selections to support uppercase in columns for context in query_context_dict: context["entity_selections"] = [ - f""""{entity_selection.replace(' AS ', '" AS "')}\"""" + f""""{entity_selection.replace(" AS ", '" AS "')}\"""" for entity_selection in context["entity_selections"] ] diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py index 89d2f96222e..6ed86e1c37c 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py @@ -58,15 +58,15 @@ def to_proto(self) -> DataSourceProto.CustomSourceOptions: class ClickhouseSource(DataSource): def __init__( self, - name: str | None = None, - query: str | None = None, - table: str | None = None, - timestamp_field: str | None = "", - created_timestamp_column: str | None = "", - field_mapping: dict[str, str] | None = None, - description: str | None = "", - tags: dict[str, str] | None = None, - owner: str | None = "", + name: Optional[str] = None, + query: Optional[str] = None, + table: Optional[str] = None, + timestamp_field: Optional[str] = "", + created_timestamp_column: Optional[str] = "", + field_mapping: Optional[dict[str, str]] = None, + description: Optional[str] = "", + tags: Optional[dict[str, str]] = None, + owner: Optional[str] = "", ): self._clickhouse_options = ClickhouseOptions( name=name, query=query, table=table @@ -180,7 +180,7 @@ def ch_type_to_feast_value_type(type_str: str) -> ValueType: if isinstance(type_obj, Array): container_type = Array type_map: dict[ - tuple[Type[ClickHouseType] | None, Type[ClickHouseType]], ValueType + tuple[Optional[Type[ClickHouseType]], Type[ClickHouseType]], ValueType ] = { (None, Boolean): ValueType.BOOL, (None, String): ValueType.STRING, From 80bf5d7d55e635b1d9f320cf369c3737cecd69cc Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Thu, 6 Mar 2025 11:11:58 +0100 Subject: [PATCH 15/22] Regenerate requirements.txt files Signed-off-by: Tomasz Wrona --- .../requirements/py3.10-ci-requirements.txt | 117 +++++++++------ .../requirements/py3.10-requirements.txt | 14 +- .../requirements/py3.11-ci-requirements.txt | 119 +++++++++------ .../requirements/py3.11-requirements.txt | 14 +- .../requirements/py3.9-ci-requirements.txt | 137 +++++++++++------- .../requirements/py3.9-requirements.txt | 14 +- 6 files changed, 245 insertions(+), 170 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 239d6b4de30..28eed7f0a3f 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,8 +1,8 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -aiobotocore==2.20.0 +aiobotocore==2.21.1 # via feast (setup.py) -aiohappyeyeballs==2.4.6 +aiohappyeyeballs==2.5.0 # via aiohttp aiohttp==3.11.13 # via aiobotocore @@ -70,16 +70,16 @@ beautifulsoup4==4.13.3 # via # docling # nbconvert -bigtree==0.25.0 +bigtree==0.25.1 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert -boto3==1.36.23 +boto3==1.37.1 # via # feast (setup.py) # ikvpy # moto -botocore==1.36.23 +botocore==1.37.1 # via # aiobotocore # boto3 @@ -160,15 +160,17 @@ cython==3.0.12 # via thriftpy2 dask[dataframe]==2025.2.0 # via feast (setup.py) -db-dtypes==1.4.1 - # via google-cloud-bigquery -debugpy==1.8.12 +db-dtypes==1.4.2 + # via + # google-cloud-bigquery + # pandas-gbq +debugpy==1.8.13 # via ipykernel decorator==5.2.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.25.1 +deltalake==0.25.4 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -180,14 +182,14 @@ distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers -docling==2.24.0 +docling==2.25.2 # via feast (setup.py) -docling-core[chunking]==2.20.0 +docling-core[chunking]==2.21.2 # via # docling # docling-ibm-models # docling-parse -docling-ibm-models==3.4.0 +docling-ibm-models==3.4.1 # via docling docling-parse==3.4.0 # via docling @@ -199,7 +201,7 @@ easyocr==1.7.2 # via docling elastic-transport==8.17.0 # via elasticsearch -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via feast (setup.py) entrypoints==0.4 # via altair @@ -218,7 +220,7 @@ executing==2.2.0 # via stack-data faiss-cpu==1.10.0 # via feast (setup.py) -fastapi==0.115.8 +fastapi==0.115.11 # via feast (setup.py) fastjsonschema==2.21.1 # via nbformat @@ -254,9 +256,11 @@ google-api-core[grpc]==2.24.1 # google-cloud-core # google-cloud-datastore # google-cloud-storage + # pandas-gbq google-auth==2.38.0 # via # google-api-core + # google-auth-oauthlib # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -264,11 +268,19 @@ google-auth==2.38.0 # google-cloud-datastore # google-cloud-storage # kubernetes -google-cloud-bigquery[pandas]==3.29.0 - # via feast (setup.py) + # pandas-gbq + # pydata-google-auth +google-auth-oauthlib==1.2.1 + # via + # pandas-gbq + # pydata-google-auth +google-cloud-bigquery[pandas]==3.30.0 + # via + # feast (setup.py) + # pandas-gbq google-cloud-bigquery-storage==2.28.0 # via feast (setup.py) -google-cloud-bigtable==2.28.1 +google-cloud-bigtable==2.29.0 # via feast (setup.py) google-cloud-core==2.4.2 # via @@ -288,7 +300,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.68.0 +googleapis-common-protos[grpc]==1.69.0 # via # feast (setup.py) # google-api-core @@ -296,14 +308,13 @@ googleapis-common-protos[grpc]==1.68.0 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) -greenlet==3.1.1 - # via sqlalchemy -grpc-google-iam-v1==0.14.0 +grpc-google-iam-v1==0.14.1 # via google-cloud-bigtable grpcio==1.70.0 # via # feast (setup.py) # google-api-core + # google-cloud-bigquery # googleapis-common-protos # grpc-google-iam-v1 # grpcio-health-checking @@ -356,7 +367,7 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client -huggingface-hub==0.29.1 +huggingface-hub==0.29.2 # via # docling # docling-ibm-models @@ -394,7 +405,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.32.0 +ipython==8.33.0 # via # great-expectations # ipykernel @@ -407,7 +418,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.5 +jinja2==3.1.6 # via # feast (setup.py) # altair @@ -601,6 +612,7 @@ numpy==1.26.4 # imageio # opencv-python-headless # pandas + # pandas-gbq # pyarrow # qdrant-client # safetensors @@ -641,6 +653,7 @@ packaging==24.2 # lazy-loader # marshmallow # nbconvert + # pandas-gbq # pytest # scikit-image # snowflake-connector-python @@ -657,8 +670,11 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pandas-gbq # pymilvus # snowflake-connector-python +pandas-gbq==0.28.0 + # via google-cloud-bigquery pandocfilters==1.5.1 # via nbconvert parsimonious==0.10.0 @@ -746,7 +762,7 @@ psycopg[binary, pool]==3.2.5 # via feast (setup.py) psycopg-binary==3.2.5 # via psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via psycopg ptyprocess==0.7.0 # via @@ -768,6 +784,7 @@ pyarrow==17.0.0 # deltalake # google-cloud-bigquery # ibis-framework + # pandas-gbq # snowflake-connector-python pyarrow-hotfix==0.6 # via ibis-framework @@ -798,8 +815,10 @@ pydantic==2.10.6 # qdrant-client pydantic-core==2.27.2 # via pydantic -pydantic-settings==2.8.0 +pydantic-settings==2.8.1 # via docling +pydata-google-auth==1.9.1 + # via pandas-gbq pygments==2.19.1 # via # feast (setup.py) @@ -824,7 +843,7 @@ pyodbc==5.2.0 # via # feast (setup.py) # ibis-framework -pyopenssl==24.3.0 +pyopenssl==25.0.0 # via snowflake-connector-python pyparsing==3.2.1 # via great-expectations @@ -834,7 +853,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pyspark==3.5.4 +pyspark==3.5.5 # via feast (setup.py) pytest==7.4.4 # via @@ -920,7 +939,7 @@ pyzmq==26.2.1 # ipykernel # jupyter-client # jupyter-server -qdrant-client==1.13.2 +qdrant-client==1.13.3 # via feast (setup.py) redis==4.6.0 # via feast (setup.py) @@ -959,7 +978,9 @@ requests==2.32.3 # transformers # trino requests-oauthlib==2.0.0 - # via kubernetes + # via + # google-auth-oauthlib + # kubernetes requests-toolbelt==1.0.0 # via python-keycloak responses==0.25.6 @@ -982,17 +1003,17 @@ rpds-py==0.23.1 # referencing rsa==4.9 # via google-auth -rtree==1.3.0 +rtree==1.4.0 # via docling ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.7 +ruff==0.9.9 # via feast (setup.py) -s3transfer==0.11.2 +s3transfer==0.11.3 # via boto3 -safetensors[torch]==0.5.2 +safetensors[torch]==0.5.3 # via # docling-ibm-models # transformers @@ -1008,15 +1029,18 @@ semchunk==2.2.2 # via docling-core send2trash==1.8.3 # via jupyter-server -setuptools==75.8.0 +setuptools==75.8.2 # via # grpcio-tools # jupyterlab # kubernetes + # pandas-gbq # pbr # pip-tools + # pydata-google-auth # pymilvus # singlestoredb + # types-setuptools shapely==2.0.7 # via easyocr shellingham==1.5.4 @@ -1039,7 +1063,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.13.2 +snowflake-connector-python[pandas]==3.14.0 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -1069,11 +1093,11 @@ sqlparams==6.2.0 # via singlestoredb stack-data==0.6.3 # via ipython -starlette==0.45.3 +starlette==0.46.0 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.3 +sympy==1.13.1 # via torch tabulate==0.9.0 # via @@ -1094,7 +1118,7 @@ tifffile==2025.2.18 # via scikit-image tinycss2==1.4.0 # via bleach -tokenizers==0.19.1 +tokenizers==0.21.0 # via transformers toml==0.10.2 # via feast (setup.py) @@ -1116,14 +1140,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.2.2 +torch==2.6.0 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.17.2 +torchvision==0.21.0 # via # feast (setup.py) # docling-ibm-models @@ -1162,7 +1186,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -transformers==4.42.4 +transformers==4.49.0 # via # docling-core # docling-ibm-models @@ -1196,7 +1220,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250225 +types-setuptools==75.8.2.20250305 # via # feast (setup.py) # types-cffi @@ -1212,6 +1236,7 @@ typing-extensions==4.12.2 # azure-identity # azure-storage-blob # beautifulsoup4 + # couchbase-columnar # docling-core # fastapi # great-expectations @@ -1227,6 +1252,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pyopenssl # python-docx # python-pptx # referencing @@ -1240,7 +1266,7 @@ typing-extensions==4.12.2 # uvicorn tzdata==2025.1 # via pandas -tzlocal==5.3 +tzlocal==5.3.1 # via # great-expectations # trino @@ -1251,7 +1277,6 @@ uri-template==1.3.0 urllib3==2.3.0 # via # feast (setup.py) - # aiobotocore # botocore # docker # elastic-transport @@ -1288,7 +1313,7 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==15.0 +websockets==15.0.1 # via uvicorn werkzeug==3.1.3 # via moto diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index ea4baadec05..a7943207378 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.25.0 +bigtree==0.25.1 # via feast (setup.py) certifi==2025.1.31 # via requests @@ -31,12 +31,10 @@ dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.8 +fastapi==0.115.11 # via feast (setup.py) fsspec==2025.2.0 # via dask -greenlet==3.1.1 - # via sqlalchemy gunicorn==23.0.0 # via # feast (setup.py) @@ -51,7 +49,7 @@ idna==3.10 # requests importlib-metadata==8.6.1 # via dask -jinja2==3.1.5 +jinja2==3.1.6 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) @@ -84,7 +82,7 @@ partd==1.4.2 # via dask prometheus-client==0.21.1 # via feast (setup.py) -protobuf==5.29.3 +protobuf==6.30.0 # via feast (setup.py) psutil==7.0.0 # via feast (setup.py) @@ -129,7 +127,7 @@ sniffio==1.3.1 # via anyio sqlalchemy[mypy]==2.0.38 # via feast (setup.py) -starlette==0.45.3 +starlette==0.46.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -172,7 +170,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==1.0.4 # via uvicorn -websockets==15.0 +websockets==15.0.1 # via uvicorn zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index af6b5b469c9..c2f9749e978 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,8 +1,8 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -aiobotocore==2.20.0 +aiobotocore==2.21.1 # via feast (setup.py) -aiohappyeyeballs==2.4.6 +aiohappyeyeballs==2.5.0 # via aiohttp aiohttp==3.11.13 # via aiobotocore @@ -68,16 +68,16 @@ beautifulsoup4==4.13.3 # via # docling # nbconvert -bigtree==0.25.0 +bigtree==0.25.1 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert -boto3==1.36.23 +boto3==1.37.1 # via # feast (setup.py) # ikvpy # moto -botocore==1.36.23 +botocore==1.37.1 # via # aiobotocore # boto3 @@ -158,15 +158,17 @@ cython==3.0.12 # via thriftpy2 dask[dataframe]==2025.2.0 # via feast (setup.py) -db-dtypes==1.4.1 - # via google-cloud-bigquery -debugpy==1.8.12 +db-dtypes==1.4.2 + # via + # google-cloud-bigquery + # pandas-gbq +debugpy==1.8.13 # via ipykernel decorator==5.2.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.25.1 +deltalake==0.25.4 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -178,14 +180,14 @@ distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers -docling==2.24.0 +docling==2.25.2 # via feast (setup.py) -docling-core[chunking]==2.20.0 +docling-core[chunking]==2.21.2 # via # docling # docling-ibm-models # docling-parse -docling-ibm-models==3.4.0 +docling-ibm-models==3.4.1 # via docling docling-parse==3.4.0 # via docling @@ -197,7 +199,7 @@ easyocr==1.7.2 # via docling elastic-transport==8.17.0 # via elasticsearch -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via feast (setup.py) entrypoints==0.4 # via altair @@ -211,7 +213,7 @@ executing==2.2.0 # via stack-data faiss-cpu==1.10.0 # via feast (setup.py) -fastapi==0.115.8 +fastapi==0.115.11 # via feast (setup.py) fastjsonschema==2.21.1 # via nbformat @@ -247,9 +249,11 @@ google-api-core[grpc]==2.24.1 # google-cloud-core # google-cloud-datastore # google-cloud-storage + # pandas-gbq google-auth==2.38.0 # via # google-api-core + # google-auth-oauthlib # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -257,11 +261,19 @@ google-auth==2.38.0 # google-cloud-datastore # google-cloud-storage # kubernetes -google-cloud-bigquery[pandas]==3.29.0 - # via feast (setup.py) + # pandas-gbq + # pydata-google-auth +google-auth-oauthlib==1.2.1 + # via + # pandas-gbq + # pydata-google-auth +google-cloud-bigquery[pandas]==3.30.0 + # via + # feast (setup.py) + # pandas-gbq google-cloud-bigquery-storage==2.28.0 # via feast (setup.py) -google-cloud-bigtable==2.28.1 +google-cloud-bigtable==2.29.0 # via feast (setup.py) google-cloud-core==2.4.2 # via @@ -281,7 +293,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.68.0 +googleapis-common-protos[grpc]==1.69.0 # via # feast (setup.py) # google-api-core @@ -289,14 +301,13 @@ googleapis-common-protos[grpc]==1.68.0 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) -greenlet==3.1.1 - # via sqlalchemy -grpc-google-iam-v1==0.14.0 +grpc-google-iam-v1==0.14.1 # via google-cloud-bigtable grpcio==1.70.0 # via # feast (setup.py) # google-api-core + # google-cloud-bigquery # googleapis-common-protos # grpc-google-iam-v1 # grpcio-health-checking @@ -349,7 +360,7 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client -huggingface-hub==0.29.1 +huggingface-hub==0.29.2 # via # docling # docling-ibm-models @@ -385,11 +396,13 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.32.0 +ipython==9.0.1 # via # great-expectations # ipykernel # ipywidgets +ipython-pygments-lexers==1.1.1 + # via ipython ipywidgets==8.1.5 # via great-expectations isodate==0.7.2 @@ -398,7 +411,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.5 +jinja2==3.1.6 # via # feast (setup.py) # altair @@ -592,6 +605,7 @@ numpy==1.26.4 # imageio # opencv-python-headless # pandas + # pandas-gbq # pyarrow # qdrant-client # safetensors @@ -632,6 +646,7 @@ packaging==24.2 # lazy-loader # marshmallow # nbconvert + # pandas-gbq # pytest # scikit-image # snowflake-connector-python @@ -648,8 +663,11 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pandas-gbq # pymilvus # snowflake-connector-python +pandas-gbq==0.28.0 + # via google-cloud-bigquery pandocfilters==1.5.1 # via nbconvert parsimonious==0.10.0 @@ -737,7 +755,7 @@ psycopg[binary, pool]==3.2.5 # via feast (setup.py) psycopg-binary==3.2.5 # via psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via psycopg ptyprocess==0.7.0 # via @@ -759,6 +777,7 @@ pyarrow==17.0.0 # deltalake # google-cloud-bigquery # ibis-framework + # pandas-gbq # snowflake-connector-python pyarrow-hotfix==0.6 # via ibis-framework @@ -789,12 +808,15 @@ pydantic==2.10.6 # qdrant-client pydantic-core==2.27.2 # via pydantic -pydantic-settings==2.8.0 +pydantic-settings==2.8.1 # via docling +pydata-google-auth==1.9.1 + # via pandas-gbq pygments==2.19.1 # via # feast (setup.py) # ipython + # ipython-pygments-lexers # mpire # nbconvert # rich @@ -815,7 +837,7 @@ pyodbc==5.2.0 # via # feast (setup.py) # ibis-framework -pyopenssl==24.3.0 +pyopenssl==25.0.0 # via snowflake-connector-python pyparsing==3.2.1 # via great-expectations @@ -825,7 +847,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pyspark==3.5.4 +pyspark==3.5.5 # via feast (setup.py) pytest==7.4.4 # via @@ -911,7 +933,7 @@ pyzmq==26.2.1 # ipykernel # jupyter-client # jupyter-server -qdrant-client==1.13.2 +qdrant-client==1.13.3 # via feast (setup.py) redis==4.6.0 # via feast (setup.py) @@ -950,7 +972,9 @@ requests==2.32.3 # transformers # trino requests-oauthlib==2.0.0 - # via kubernetes + # via + # google-auth-oauthlib + # kubernetes requests-toolbelt==1.0.0 # via python-keycloak responses==0.25.6 @@ -973,17 +997,17 @@ rpds-py==0.23.1 # referencing rsa==4.9 # via google-auth -rtree==1.3.0 +rtree==1.4.0 # via docling ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.7 +ruff==0.9.9 # via feast (setup.py) -s3transfer==0.11.2 +s3transfer==0.11.3 # via boto3 -safetensors[torch]==0.5.2 +safetensors[torch]==0.5.3 # via # docling-ibm-models # transformers @@ -999,15 +1023,18 @@ semchunk==2.2.2 # via docling-core send2trash==1.8.3 # via jupyter-server -setuptools==75.8.0 +setuptools==75.8.2 # via # grpcio-tools # jupyterlab # kubernetes + # pandas-gbq # pbr # pip-tools + # pydata-google-auth # pymilvus # singlestoredb + # types-setuptools shapely==2.0.7 # via easyocr shellingham==1.5.4 @@ -1030,7 +1057,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.13.2 +snowflake-connector-python[pandas]==3.14.0 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -1060,11 +1087,11 @@ sqlparams==6.2.0 # via singlestoredb stack-data==0.6.3 # via ipython -starlette==0.45.3 +starlette==0.46.0 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.3 +sympy==1.13.1 # via torch tabulate==0.9.0 # via @@ -1085,7 +1112,7 @@ tifffile==2025.2.18 # via scikit-image tinycss2==1.4.0 # via bleach -tokenizers==0.19.1 +tokenizers==0.21.0 # via transformers toml==0.10.2 # via feast (setup.py) @@ -1099,14 +1126,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.2.2 +torch==2.6.0 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.17.2 +torchvision==0.21.0 # via # feast (setup.py) # docling-ibm-models @@ -1145,7 +1172,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -transformers==4.42.4 +transformers==4.49.0 # via # docling-core # docling-ibm-models @@ -1179,7 +1206,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250225 +types-setuptools==75.8.2.20250305 # via # feast (setup.py) # types-cffi @@ -1207,6 +1234,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pyopenssl # python-docx # python-pptx # referencing @@ -1218,7 +1246,7 @@ typing-extensions==4.12.2 # typer tzdata==2025.1 # via pandas -tzlocal==5.3 +tzlocal==5.3.1 # via # great-expectations # trino @@ -1229,7 +1257,6 @@ uri-template==1.3.0 urllib3==2.3.0 # via # feast (setup.py) - # aiobotocore # botocore # docker # elastic-transport @@ -1266,7 +1293,7 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==15.0 +websockets==15.0.1 # via uvicorn werkzeug==3.1.3 # via moto diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index d33da6d75c2..df1d5104ef0 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.25.0 +bigtree==0.25.1 # via feast (setup.py) certifi==2025.1.31 # via requests @@ -29,12 +29,10 @@ dask[dataframe]==2025.2.0 # via feast (setup.py) dill==0.3.9 # via feast (setup.py) -fastapi==0.115.8 +fastapi==0.115.11 # via feast (setup.py) fsspec==2025.2.0 # via dask -greenlet==3.1.1 - # via sqlalchemy gunicorn==23.0.0 # via # feast (setup.py) @@ -49,7 +47,7 @@ idna==3.10 # requests importlib-metadata==8.6.1 # via dask -jinja2==3.1.5 +jinja2==3.1.6 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) @@ -82,7 +80,7 @@ partd==1.4.2 # via dask prometheus-client==0.21.1 # via feast (setup.py) -protobuf==5.29.3 +protobuf==6.30.0 # via feast (setup.py) psutil==7.0.0 # via feast (setup.py) @@ -127,7 +125,7 @@ sniffio==1.3.1 # via anyio sqlalchemy[mypy]==2.0.38 # via feast (setup.py) -starlette==0.45.3 +starlette==0.46.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -167,7 +165,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==1.0.4 # via uvicorn -websockets==15.0 +websockets==15.0.1 # via uvicorn zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 4473d933258..0a9bdb881b7 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,8 +1,8 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -aiobotocore==2.20.0 +aiobotocore==2.21.1 # via feast (setup.py) -aiohappyeyeballs==2.4.6 +aiohappyeyeballs==2.5.0 # via aiohttp aiohttp==3.11.13 # via aiobotocore @@ -72,16 +72,16 @@ beautifulsoup4==4.13.3 # nbconvert bidict==0.23.1 # via ibis-framework -bigtree==0.25.0 +bigtree==0.25.1 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert -boto3==1.36.23 +boto3==1.37.1 # via # feast (setup.py) # ikvpy # moto -botocore==1.36.23 +botocore==1.37.1 # via # aiobotocore # boto3 @@ -166,15 +166,17 @@ dask[dataframe]==2024.8.0 # dask-expr dask-expr==1.1.10 # via dask -db-dtypes==1.4.1 - # via google-cloud-bigquery -debugpy==1.8.12 +db-dtypes==1.4.2 + # via + # google-cloud-bigquery + # pandas-gbq +debugpy==1.8.13 # via ipykernel decorator==5.2.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.25.1 +deltalake==0.25.4 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -186,14 +188,14 @@ distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers -docling==2.24.0 +docling==2.25.2 # via feast (setup.py) -docling-core[chunking]==2.20.0 +docling-core[chunking]==2.21.2 # via # docling # docling-ibm-models # docling-parse -docling-ibm-models==3.4.0 +docling-ibm-models==3.4.1 # via docling docling-parse==3.4.0 # via docling @@ -205,10 +207,12 @@ easyocr==1.7.2 # via docling elastic-transport==8.17.0 # via elasticsearch -elasticsearch==8.17.1 +elasticsearch==8.17.2 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus et-xmlfile==2.0.0 # via openpyxl exceptiongroup==1.2.2 @@ -222,7 +226,7 @@ executing==2.2.0 # via stack-data faiss-cpu==1.10.0 # via feast (setup.py) -fastapi==0.115.8 +fastapi==0.115.11 # via feast (setup.py) fastjsonschema==2.21.1 # via nbformat @@ -258,9 +262,11 @@ google-api-core[grpc]==2.24.1 # google-cloud-core # google-cloud-datastore # google-cloud-storage + # pandas-gbq google-auth==2.38.0 # via # google-api-core + # google-auth-oauthlib # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -268,11 +274,19 @@ google-auth==2.38.0 # google-cloud-datastore # google-cloud-storage # kubernetes -google-cloud-bigquery[pandas]==3.29.0 - # via feast (setup.py) + # pandas-gbq + # pydata-google-auth +google-auth-oauthlib==1.2.1 + # via + # pandas-gbq + # pydata-google-auth +google-cloud-bigquery[pandas]==3.30.0 + # via + # feast (setup.py) + # pandas-gbq google-cloud-bigquery-storage==2.28.0 # via feast (setup.py) -google-cloud-bigtable==2.28.1 +google-cloud-bigtable==2.29.0 # via feast (setup.py) google-cloud-core==2.4.2 # via @@ -292,7 +306,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.68.0 +googleapis-common-protos[grpc]==1.69.0 # via # feast (setup.py) # google-api-core @@ -300,14 +314,13 @@ googleapis-common-protos[grpc]==1.68.0 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) -greenlet==3.1.1 - # via sqlalchemy -grpc-google-iam-v1==0.14.0 +grpc-google-iam-v1==0.14.1 # via google-cloud-bigtable -grpcio==1.67.1 +grpcio==1.70.0 # via # feast (setup.py) # google-api-core + # google-cloud-bigquery # googleapis-common-protos # grpc-google-iam-v1 # grpcio-health-checking @@ -318,17 +331,17 @@ grpcio==1.67.1 # ikvpy # pymilvus # qdrant-client -grpcio-health-checking==1.67.1 +grpcio-health-checking==1.70.0 # via feast (setup.py) -grpcio-reflection==1.67.1 +grpcio-reflection==1.70.0 # via feast (setup.py) -grpcio-status==1.67.1 +grpcio-status==1.70.0 # via # google-api-core # ikvpy -grpcio-testing==1.67.1 +grpcio-testing==1.70.0 # via feast (setup.py) -grpcio-tools==1.67.1 +grpcio-tools==1.70.0 # via # feast (setup.py) # qdrant-client @@ -360,7 +373,7 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client -huggingface-hub==0.29.1 +huggingface-hub==0.29.2 # via # docling # docling-ibm-models @@ -418,7 +431,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.5 +jinja2==3.1.6 # via # feast (setup.py) # altair @@ -524,7 +537,9 @@ markupsafe==3.0.2 # nbconvert # werkzeug marshmallow==3.26.1 - # via great-expectations + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel @@ -610,6 +625,7 @@ numpy==1.26.4 # imageio # opencv-python-headless # pandas + # pandas-gbq # pyarrow # qdrant-client # safetensors @@ -649,6 +665,7 @@ packaging==24.2 # lazy-loader # marshmallow # nbconvert + # pandas-gbq # pytest # scikit-image # snowflake-connector-python @@ -666,8 +683,11 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pandas-gbq # pymilvus # snowflake-connector-python +pandas-gbq==0.28.0 + # via google-cloud-bigquery pandocfilters==1.5.1 # via nbconvert parsimonious==0.10.0 @@ -755,7 +775,7 @@ psycopg[binary, pool]==3.2.5 # via feast (setup.py) psycopg-binary==3.2.5 # via psycopg -psycopg-pool==3.2.5 +psycopg-pool==3.2.6 # via psycopg ptyprocess==0.7.0 # via @@ -777,6 +797,7 @@ pyarrow==16.1.0 # deltalake # google-cloud-bigquery # ibis-framework + # pandas-gbq # snowflake-connector-python pyarrow-hotfix==0.6 # via ibis-framework @@ -807,8 +828,10 @@ pydantic==2.10.6 # qdrant-client pydantic-core==2.27.2 # via pydantic -pydantic-settings==2.8.0 +pydantic-settings==2.8.1 # via docling +pydata-google-auth==1.9.1 + # via pandas-gbq pygments==2.19.1 # via # feast (setup.py) @@ -823,7 +846,7 @@ pyjwt[crypto]==2.10.1 # msal # singlestoredb # snowflake-connector-python -pymilvus==2.5.4 +pymilvus==2.4.9 # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) @@ -833,7 +856,7 @@ pyodbc==5.2.0 # via # feast (setup.py) # ibis-framework -pyopenssl==24.3.0 +pyopenssl==25.0.0 # via snowflake-connector-python pyparsing==3.2.1 # via great-expectations @@ -843,7 +866,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pyspark==3.5.4 +pyspark==3.5.5 # via feast (setup.py) pytest==7.4.4 # via @@ -894,8 +917,8 @@ python-docx==1.1.2 # via docling python-dotenv==1.0.1 # via + # environs # pydantic-settings - # pymilvus # uvicorn python-json-logger==3.2.1 # via jupyter-events @@ -929,7 +952,7 @@ pyzmq==26.2.1 # ipykernel # jupyter-client # jupyter-server -qdrant-client==1.13.2 +qdrant-client==1.13.3 # via feast (setup.py) redis==4.6.0 # via feast (setup.py) @@ -968,7 +991,9 @@ requests==2.32.3 # transformers # trino requests-oauthlib==2.0.0 - # via kubernetes + # via + # google-auth-oauthlib + # kubernetes requests-toolbelt==1.0.0 # via python-keycloak responses==0.25.6 @@ -991,17 +1016,17 @@ rpds-py==0.23.1 # referencing rsa==4.9 # via google-auth -rtree==1.3.0 +rtree==1.4.0 # via docling ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.7 +ruff==0.9.9 # via feast (setup.py) -s3transfer==0.11.2 +s3transfer==0.11.3 # via boto3 -safetensors[torch]==0.5.2 +safetensors[torch]==0.5.3 # via # docling-ibm-models # transformers @@ -1017,15 +1042,18 @@ semchunk==2.2.2 # via docling-core send2trash==1.8.3 # via jupyter-server -setuptools==75.8.0 +setuptools==75.8.2 # via # grpcio-tools # jupyterlab # kubernetes + # pandas-gbq # pbr # pip-tools + # pydata-google-auth # pymilvus # singlestoredb + # types-setuptools shapely==2.0.7 # via easyocr shellingham==1.5.4 @@ -1048,7 +1076,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.13.2 +snowflake-connector-python[pandas]==3.14.0 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -1078,11 +1106,11 @@ sqlparams==6.2.0 # via singlestoredb stack-data==0.6.3 # via ipython -starlette==0.45.3 +starlette==0.46.0 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.3 +sympy==1.13.1 # via torch tabulate==0.9.0 # via @@ -1103,7 +1131,7 @@ tifffile==2024.8.30 # via scikit-image tinycss2==1.4.0 # via bleach -tokenizers==0.19.1 +tokenizers==0.21.0 # via transformers toml==0.10.2 # via feast (setup.py) @@ -1125,14 +1153,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.2.2 +torch==2.6.0 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.17.2 +torchvision==0.21.0 # via # feast (setup.py) # docling-ibm-models @@ -1171,7 +1199,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -transformers==4.42.4 +transformers==4.49.0 # via # docling-core # docling-ibm-models @@ -1205,7 +1233,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250225 +types-setuptools==75.8.2.20250305 # via # feast (setup.py) # types-cffi @@ -1222,6 +1250,7 @@ typing-extensions==4.12.2 # azure-identity # azure-storage-blob # beautifulsoup4 + # couchbase-columnar # docling-core # fastapi # great-expectations @@ -1237,6 +1266,7 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # pyopenssl # python-docx # python-json-logger # python-pptx @@ -1252,7 +1282,7 @@ typing-extensions==4.12.2 # uvicorn tzdata==2025.1 # via pandas -tzlocal==5.3 +tzlocal==5.3.1 # via # great-expectations # trino @@ -1263,7 +1293,6 @@ uri-template==1.3.0 urllib3==1.26.20 # via # feast (setup.py) - # aiobotocore # botocore # docker # elastic-transport @@ -1301,7 +1330,7 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==15.0 +websockets==15.0.1 # via uvicorn werkzeug==3.1.3 # via moto diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index e7aa5a42409..3b6fddd2cb2 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.25.0 +bigtree==0.25.1 # via feast (setup.py) certifi==2025.1.31 # via requests @@ -35,12 +35,10 @@ dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.8 +fastapi==0.115.11 # via feast (setup.py) fsspec==2025.2.0 # via dask -greenlet==3.1.1 - # via sqlalchemy gunicorn==23.0.0 # via # feast (setup.py) @@ -57,7 +55,7 @@ importlib-metadata==8.6.1 # via # dask # typeguard -jinja2==3.1.5 +jinja2==3.1.6 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) @@ -91,7 +89,7 @@ partd==1.4.2 # via dask prometheus-client==0.21.1 # via feast (setup.py) -protobuf==5.29.3 +protobuf==6.30.0 # via feast (setup.py) psutil==7.0.0 # via feast (setup.py) @@ -136,7 +134,7 @@ sniffio==1.3.1 # via anyio sqlalchemy[mypy]==2.0.38 # via feast (setup.py) -starlette==0.45.3 +starlette==0.46.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -180,7 +178,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==1.0.4 # via uvicorn -websockets==15.0 +websockets==15.0.1 # via uvicorn zipp==3.21.0 # via importlib-metadata From 53839a83b79371cacba885b3a771fb0c2b69cabf Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Fri, 7 Mar 2025 11:46:11 +0100 Subject: [PATCH 16/22] Add clickhouse to pyproject.toml Signed-off-by: Tomasz Wrona --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9eec118099a..5295647fb24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ azure = [ "pymssql" ] cassandra = ["cassandra-driver>=3.24.0,<4"] +clickhouse = ["clickhouse-connect>=0.7.19"] couchbase = ["couchbase==4.3.2", "couchbase-columnar==1.0.0"] delta = ["deltalake"] docling = ["docling>=2.23.0"] From 987294453560b8a879556644def6c9daf34031f1 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Fri, 7 Mar 2025 13:32:43 +0100 Subject: [PATCH 17/22] Fix dependencies Signed-off-by: Tomasz Wrona --- pyproject.toml | 2 +- .../requirements/py3.10-ci-requirements.txt | 19 ++++++++++++++----- .../requirements/py3.11-ci-requirements.txt | 19 ++++++++++++++----- .../requirements/py3.9-ci-requirements.txt | 19 ++++++++++++++----- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5295647fb24..3b2e6223bec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,7 @@ ci = [ "types-setuptools", "types-tabulate", "virtualenv<20.24.2", - "feast[aws, azure, cassandra, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, go, grpcio, hazelcast, hbase, ibis, ikv, k8s, milvus, mssql, mysql, opentelemetry, spark, trino, postgres, pytorch, qdrant, redis, singlestore, snowflake, sqlite_vec]" + "feast[aws, azure, cassandra, clickhouse, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, go, grpcio, hazelcast, hbase, ibis, ikv, k8s, milvus, mssql, mysql, opentelemetry, spark, trino, postgres, pytorch, qdrant, redis, singlestore, snowflake, sqlite_vec]" ] nlp = ["feast[docling, milvus, pytorch]"] dev = ["feast[ci]"] diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 28eed7f0a3f..58796030194 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -96,6 +96,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # docling # elastic-transport # httpcore @@ -126,6 +127,8 @@ click==8.1.8 # pip-tools # typer # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -300,7 +303,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.69.0 +googleapis-common-protos[grpc]==1.69.1 # via # feast (setup.py) # google-api-core @@ -511,7 +514,9 @@ lxml==5.3.1 # python-docx # python-pptx lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -907,7 +912,7 @@ python-dotenv==1.0.1 # environs # pydantic-settings # uvicorn -python-json-logger==3.2.1 +python-json-logger==3.3.0 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -915,6 +920,7 @@ python-pptx==1.0.2 # via docling pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -1198,7 +1204,7 @@ typer==0.12.5 # via # docling # docling-core -types-cffi==1.16.0.20241221 +types-cffi==1.16.0.20250307 # via types-pyopenssl types-protobuf==3.19.22 # via @@ -1278,6 +1284,7 @@ urllib3==2.3.0 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1336,4 +1343,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index c2f9749e978..59912cebbbf 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -94,6 +94,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # docling # elastic-transport # httpcore @@ -124,6 +125,8 @@ click==8.1.8 # pip-tools # typer # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -293,7 +296,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.69.0 +googleapis-common-protos[grpc]==1.69.1 # via # feast (setup.py) # google-api-core @@ -504,7 +507,9 @@ lxml==5.3.1 # python-docx # python-pptx lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -901,7 +906,7 @@ python-dotenv==1.0.1 # environs # pydantic-settings # uvicorn -python-json-logger==3.2.1 +python-json-logger==3.3.0 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -909,6 +914,7 @@ python-pptx==1.0.2 # via docling pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -1184,7 +1190,7 @@ typer==0.12.5 # via # docling # docling-core -types-cffi==1.16.0.20241221 +types-cffi==1.16.0.20250307 # via types-pyopenssl types-protobuf==3.19.22 # via @@ -1258,6 +1264,7 @@ urllib3==2.3.0 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1316,4 +1323,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 0a9bdb881b7..0e7bd1d41b0 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -98,6 +98,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # docling # elastic-transport # httpcore @@ -128,6 +129,8 @@ click==8.1.8 # pip-tools # typer # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -306,7 +309,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.69.0 +googleapis-common-protos[grpc]==1.69.1 # via # feast (setup.py) # google-api-core @@ -524,7 +527,9 @@ lxml==5.3.1 # python-docx # python-pptx lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -920,7 +925,7 @@ python-dotenv==1.0.1 # environs # pydantic-settings # uvicorn -python-json-logger==3.2.1 +python-json-logger==3.3.0 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -928,6 +933,7 @@ python-pptx==1.0.2 # via docling pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -1211,7 +1217,7 @@ typer==0.12.5 # via # docling # docling-core -types-cffi==1.16.0.20241221 +types-cffi==1.16.0.20250307 # via types-pyopenssl types-protobuf==3.19.22 # via @@ -1294,6 +1300,7 @@ urllib3==1.26.20 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1353,4 +1360,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino From e8e65b12918f01352cb8b170e904248d452608c5 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Fri, 7 Mar 2025 13:33:05 +0100 Subject: [PATCH 18/22] Simplify names Signed-off-by: Tomasz Wrona --- .../contrib/clickhouse_offline_store/clickhouse.py | 4 ++-- .../contrib/clickhouse_offline_store/tests/data_source.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index 3fb559b611c..f1272e7f8af 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -35,8 +35,8 @@ class ClickhouseOfflineStoreConfig(ClickhouseConfig): type: Literal[ - "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" - ] = "feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore" + "clickhouse" + ] = "clickhouse" class ClickhouseOfflineStore(OfflineStore): diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py index 8ab4fdb5408..80fd1751dc5 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/tests/data_source.py @@ -70,7 +70,7 @@ def __init__( ) self.offline_store_config = ClickhouseOfflineStoreConfig( - type="feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore", + type="clickhouse", host="localhost", port=self.container.get_exposed_port(8123), database=CLICKHOUSE_OFFLINE_DB, From c0f19b2a8a49d1cba268eddaf23457ab96eb7fb2 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Fri, 7 Mar 2025 13:33:22 +0100 Subject: [PATCH 19/22] Skip problematic Clickhouse tests Signed-off-by: Tomasz Wrona --- .../tests/integration/registration/test_universal_types.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index 9d0b620c083..0625ec05901 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -340,6 +340,11 @@ def offline_types_test_fixtures(request, environment): config: TypeTestConfig = request.param if environment.provider == "aws" and config.feature_is_list is True: pytest.skip("Redshift doesn't support list features") + if (environment.data_source_creator.__class__.__name__ == "ClickhouseDataSourceCreator" + and config.feature_dtype in {"float", "datetime", "bool"} + and config.feature_is_list + and not config.has_empty_list): + pytest.skip("Clickhouse doesn't support Nullable(Array) type features") return get_fixtures(request, environment) From 2d694275dec1f981fe5127778d4ec413e94f575e Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Fri, 7 Mar 2025 13:34:37 +0100 Subject: [PATCH 20/22] format & lint Signed-off-by: Tomasz Wrona --- .../contrib/clickhouse_offline_store/clickhouse.py | 4 +--- .../integration/registration/test_universal_types.py | 11 +++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index f1272e7f8af..dad5b470cd8 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -34,9 +34,7 @@ class ClickhouseOfflineStoreConfig(ClickhouseConfig): - type: Literal[ - "clickhouse" - ] = "clickhouse" + type: Literal["clickhouse"] = "clickhouse" class ClickhouseOfflineStore(OfflineStore): diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index 0625ec05901..5ba99b9d7f1 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -340,10 +340,13 @@ def offline_types_test_fixtures(request, environment): config: TypeTestConfig = request.param if environment.provider == "aws" and config.feature_is_list is True: pytest.skip("Redshift doesn't support list features") - if (environment.data_source_creator.__class__.__name__ == "ClickhouseDataSourceCreator" - and config.feature_dtype in {"float", "datetime", "bool"} - and config.feature_is_list - and not config.has_empty_list): + if ( + environment.data_source_creator.__class__.__name__ + == "ClickhouseDataSourceCreator" + and config.feature_dtype in {"float", "datetime", "bool"} + and config.feature_is_list + and not config.has_empty_list + ): pytest.skip("Clickhouse doesn't support Nullable(Array) type features") return get_fixtures(request, environment) From 325890376f0636e06b87208d43e62f528e2d1011 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Wed, 12 Mar 2025 11:38:54 +0100 Subject: [PATCH 21/22] Post-merge `make lock-python-dependencies-all` Signed-off-by: Tomasz Wrona --- .../requirements/py3.10-ci-requirements.txt | 73 ++++++++++--------- .../requirements/py3.10-requirements.txt | 8 +- .../requirements/py3.11-ci-requirements.txt | 73 ++++++++++--------- .../requirements/py3.11-requirements.txt | 8 +- .../requirements/py3.9-ci-requirements.txt | 71 ++++++++++-------- .../requirements/py3.9-requirements.txt | 8 +- 6 files changed, 131 insertions(+), 110 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index fec49b8cb0d..3347e22ed5a 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -2,7 +2,7 @@ # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.21.1 # via feast (setup.py) -aiohappyeyeballs==2.5.0 +aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.11.13 # via aiobotocore @@ -48,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==5.1 # via ibis-framework -attrs==25.1.0 +attrs==25.2.0 # via # aiohttp # jsonlines @@ -58,9 +58,9 @@ azure-core==1.32.0 # via # azure-identity # azure-storage-blob -azure-identity==1.20.0 +azure-identity==1.21.0 # via feast (setup.py) -azure-storage-blob==12.24.1 +azure-storage-blob==12.25.0 # via feast (setup.py) babel==2.17.0 # via @@ -96,6 +96,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # docling # elastic-transport # httpcore @@ -126,6 +127,8 @@ click==8.1.8 # pip-tools # typer # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -182,7 +185,7 @@ distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers -docling==2.25.2 +docling==2.26.0 # via feast (setup.py) docling-core[chunking]==2.21.2 # via @@ -247,7 +250,7 @@ fsspec==2024.9.0 # torch geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.24.1 +google-api-core[grpc]==2.24.2 # via # feast (setup.py) # google-cloud-bigquery @@ -278,11 +281,11 @@ google-cloud-bigquery[pandas]==3.30.0 # via # feast (setup.py) # pandas-gbq -google-cloud-bigquery-storage==2.28.0 +google-cloud-bigquery-storage==2.29.0 # via feast (setup.py) google-cloud-bigtable==2.29.0 # via feast (setup.py) -google-cloud-core==2.4.2 +google-cloud-core==2.4.3 # via # google-cloud-bigquery # google-cloud-bigtable @@ -308,11 +311,9 @@ googleapis-common-protos[grpc]==1.69.1 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) -greenlet==3.1.1 - # via sqlalchemy grpc-google-iam-v1==0.14.1 # via google-cloud-bigtable -grpcio==1.70.0 +grpcio==1.71.0 # via # feast (setup.py) # google-api-core @@ -327,17 +328,17 @@ grpcio==1.70.0 # ikvpy # pymilvus # qdrant-client -grpcio-health-checking==1.70.0 +grpcio-health-checking==1.71.0 # via feast (setup.py) -grpcio-reflection==1.70.0 +grpcio-reflection==1.71.0 # via feast (setup.py) -grpcio-status==1.70.0 +grpcio-status==1.71.0 # via # google-api-core # ikvpy -grpcio-testing==1.70.0 +grpcio-testing==1.71.0 # via feast (setup.py) -grpcio-tools==1.70.0 +grpcio-tools==1.71.0 # via # feast (setup.py) # qdrant-client @@ -369,7 +370,7 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client -huggingface-hub==0.29.2 +huggingface-hub==0.29.3 # via # docling # docling-ibm-models @@ -383,7 +384,7 @@ ibis-framework[duckdb, mssql]==9.5.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.8 +identify==2.6.9 # via pre-commit idna==3.10 # via @@ -407,7 +408,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.33.0 +ipython==8.34.0 # via # great-expectations # ipykernel @@ -513,7 +514,9 @@ lxml==5.3.1 # python-docx # python-pptx lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -731,7 +734,7 @@ propcache==0.3.0 # via # aiohttp # yarl -proto-plus==1.26.0 +proto-plus==1.26.1 # via # google-api-core # google-cloud-bigquery-storage @@ -918,6 +921,7 @@ python-pptx==1.0.2 # via docling pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -937,7 +941,7 @@ pyyaml==6.0.2 # responses # transformers # uvicorn -pyzmq==26.2.1 +pyzmq==26.3.0 # via # ipykernel # jupyter-client @@ -986,7 +990,7 @@ requests-oauthlib==2.0.0 # kubernetes requests-toolbelt==1.0.0 # via python-keycloak -responses==0.25.6 +responses==0.25.7 # via moto rfc3339-validator==0.1.4 # via @@ -1012,7 +1016,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.9 +ruff==0.9.10 # via feast (setup.py) s3transfer==0.11.3 # via boto3 @@ -1032,7 +1036,7 @@ semchunk==2.2.2 # via docling-core send2trash==1.8.3 # via jupyter-server -setuptools==75.8.2 +setuptools==76.0.0 # via # grpcio-tools # jupyterlab @@ -1086,7 +1090,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sqlalchemy[mypy]==2.0.38 +sqlalchemy[mypy]==2.0.39 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework @@ -1096,11 +1100,11 @@ sqlparams==6.2.0 # via singlestoredb stack-data==0.6.3 # via ipython -starlette==0.46.0 +starlette==0.46.1 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.3 +sympy==1.13.1 # via torch tabulate==0.9.0 # via @@ -1121,7 +1125,7 @@ tifffile==2025.2.18 # via scikit-image tinycss2==1.4.0 # via bleach -tokenizers==0.19.1 +tokenizers==0.21.0 # via transformers toml==0.10.2 # via feast (setup.py) @@ -1143,14 +1147,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.2.2 +torch==2.6.0 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.17.2 +torchvision==0.21.0 # via # feast (setup.py) # docling-ibm-models @@ -1189,7 +1193,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -transformers==4.42.4 +transformers==4.49.0 # via # docling-core # docling-ibm-models @@ -1281,6 +1285,7 @@ urllib3==2.3.0 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1339,4 +1344,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index a7943207378..dbcf2648cfb 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -6,7 +6,7 @@ anyio==4.8.0 # via # starlette # watchfiles -attrs==25.1.0 +attrs==25.2.0 # via # jsonschema # referencing @@ -33,7 +33,7 @@ exceptiongroup==1.2.2 # via anyio fastapi==0.115.11 # via feast (setup.py) -fsspec==2025.2.0 +fsspec==2025.3.0 # via dask gunicorn==23.0.0 # via @@ -125,9 +125,9 @@ six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.38 +sqlalchemy[mypy]==2.0.39 # via feast (setup.py) -starlette==0.46.0 +starlette==0.46.1 # via fastapi tabulate==0.9.0 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 7f03e1d9ccd..c5c3d22fceb 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -2,7 +2,7 @@ # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.21.1 # via feast (setup.py) -aiohappyeyeballs==2.5.0 +aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.11.13 # via aiobotocore @@ -46,7 +46,7 @@ async-timeout==5.0.1 # via redis atpublic==5.1 # via ibis-framework -attrs==25.1.0 +attrs==25.2.0 # via # aiohttp # jsonlines @@ -56,9 +56,9 @@ azure-core==1.32.0 # via # azure-identity # azure-storage-blob -azure-identity==1.20.0 +azure-identity==1.21.0 # via feast (setup.py) -azure-storage-blob==12.24.1 +azure-storage-blob==12.25.0 # via feast (setup.py) babel==2.17.0 # via @@ -94,6 +94,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # docling # elastic-transport # httpcore @@ -124,6 +125,8 @@ click==8.1.8 # pip-tools # typer # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -180,7 +183,7 @@ distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers -docling==2.25.2 +docling==2.26.0 # via feast (setup.py) docling-core[chunking]==2.21.2 # via @@ -240,7 +243,7 @@ fsspec==2024.9.0 # torch geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.24.1 +google-api-core[grpc]==2.24.2 # via # feast (setup.py) # google-cloud-bigquery @@ -271,11 +274,11 @@ google-cloud-bigquery[pandas]==3.30.0 # via # feast (setup.py) # pandas-gbq -google-cloud-bigquery-storage==2.28.0 +google-cloud-bigquery-storage==2.29.0 # via feast (setup.py) google-cloud-bigtable==2.29.0 # via feast (setup.py) -google-cloud-core==2.4.2 +google-cloud-core==2.4.3 # via # google-cloud-bigquery # google-cloud-bigtable @@ -301,11 +304,9 @@ googleapis-common-protos[grpc]==1.69.1 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) -greenlet==3.1.1 - # via sqlalchemy grpc-google-iam-v1==0.14.1 # via google-cloud-bigtable -grpcio==1.70.0 +grpcio==1.71.0 # via # feast (setup.py) # google-api-core @@ -320,17 +321,17 @@ grpcio==1.70.0 # ikvpy # pymilvus # qdrant-client -grpcio-health-checking==1.70.0 +grpcio-health-checking==1.71.0 # via feast (setup.py) -grpcio-reflection==1.70.0 +grpcio-reflection==1.71.0 # via feast (setup.py) -grpcio-status==1.70.0 +grpcio-status==1.71.0 # via # google-api-core # ikvpy -grpcio-testing==1.70.0 +grpcio-testing==1.71.0 # via feast (setup.py) -grpcio-tools==1.70.0 +grpcio-tools==1.71.0 # via # feast (setup.py) # qdrant-client @@ -362,7 +363,7 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client -huggingface-hub==0.29.2 +huggingface-hub==0.29.3 # via # docling # docling-ibm-models @@ -376,7 +377,7 @@ ibis-framework[duckdb, mssql]==9.5.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.8 +identify==2.6.9 # via pre-commit idna==3.10 # via @@ -398,7 +399,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==9.0.1 +ipython==9.0.2 # via # great-expectations # ipykernel @@ -506,7 +507,9 @@ lxml==5.3.1 # python-docx # python-pptx lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -724,7 +727,7 @@ propcache==0.3.0 # via # aiohttp # yarl -proto-plus==1.26.0 +proto-plus==1.26.1 # via # google-api-core # google-cloud-bigquery-storage @@ -912,6 +915,7 @@ python-pptx==1.0.2 # via docling pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -931,7 +935,7 @@ pyyaml==6.0.2 # responses # transformers # uvicorn -pyzmq==26.2.1 +pyzmq==26.3.0 # via # ipykernel # jupyter-client @@ -980,7 +984,7 @@ requests-oauthlib==2.0.0 # kubernetes requests-toolbelt==1.0.0 # via python-keycloak -responses==0.25.6 +responses==0.25.7 # via moto rfc3339-validator==0.1.4 # via @@ -1006,7 +1010,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.9 +ruff==0.9.10 # via feast (setup.py) s3transfer==0.11.3 # via boto3 @@ -1026,7 +1030,7 @@ semchunk==2.2.2 # via docling-core send2trash==1.8.3 # via jupyter-server -setuptools==75.8.2 +setuptools==76.0.0 # via # grpcio-tools # jupyterlab @@ -1080,7 +1084,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sqlalchemy[mypy]==2.0.38 +sqlalchemy[mypy]==2.0.39 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework @@ -1090,11 +1094,11 @@ sqlparams==6.2.0 # via singlestoredb stack-data==0.6.3 # via ipython -starlette==0.46.0 +starlette==0.46.1 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.3 +sympy==1.13.1 # via torch tabulate==0.9.0 # via @@ -1115,7 +1119,7 @@ tifffile==2025.2.18 # via scikit-image tinycss2==1.4.0 # via bleach -tokenizers==0.19.1 +tokenizers==0.21.0 # via transformers toml==0.10.2 # via feast (setup.py) @@ -1129,14 +1133,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.2.2 +torch==2.6.0 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.17.2 +torchvision==0.21.0 # via # feast (setup.py) # docling-ibm-models @@ -1175,7 +1179,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -transformers==4.42.4 +transformers==4.49.0 # via # docling-core # docling-ibm-models @@ -1261,6 +1265,7 @@ urllib3==2.3.0 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1319,4 +1324,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index df1d5104ef0..5d182e2fc09 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -6,7 +6,7 @@ anyio==4.8.0 # via # starlette # watchfiles -attrs==25.1.0 +attrs==25.2.0 # via # jsonschema # referencing @@ -31,7 +31,7 @@ dill==0.3.9 # via feast (setup.py) fastapi==0.115.11 # via feast (setup.py) -fsspec==2025.2.0 +fsspec==2025.3.0 # via dask gunicorn==23.0.0 # via @@ -123,9 +123,9 @@ six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.38 +sqlalchemy[mypy]==2.0.39 # via feast (setup.py) -starlette==0.46.0 +starlette==0.46.1 # via fastapi tabulate==0.9.0 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 3a564962973..ef2604f14be 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -2,7 +2,7 @@ # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.21.1 # via feast (setup.py) -aiohappyeyeballs==2.5.0 +aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.11.13 # via aiobotocore @@ -48,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==4.1.0 # via ibis-framework -attrs==25.1.0 +attrs==25.2.0 # via # aiohttp # jsonlines @@ -58,9 +58,9 @@ azure-core==1.32.0 # via # azure-identity # azure-storage-blob -azure-identity==1.20.0 +azure-identity==1.21.0 # via feast (setup.py) -azure-storage-blob==12.24.1 +azure-storage-blob==12.25.0 # via feast (setup.py) babel==2.17.0 # via @@ -98,6 +98,7 @@ cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # clickhouse-connect # docling # elastic-transport # httpcore @@ -128,6 +129,8 @@ click==8.1.8 # pip-tools # typer # uvicorn +clickhouse-connect==0.8.15 + # via feast (setup.py) cloudpickle==3.1.1 # via dask colorama==0.4.6 @@ -188,7 +191,7 @@ distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers -docling==2.25.2 +docling==2.26.0 # via feast (setup.py) docling-core[chunking]==2.21.2 # via @@ -253,7 +256,7 @@ fsspec==2024.9.0 # torch geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.24.1 +google-api-core[grpc]==2.24.2 # via # feast (setup.py) # google-cloud-bigquery @@ -284,11 +287,11 @@ google-cloud-bigquery[pandas]==3.30.0 # via # feast (setup.py) # pandas-gbq -google-cloud-bigquery-storage==2.28.0 +google-cloud-bigquery-storage==2.29.0 # via feast (setup.py) google-cloud-bigtable==2.29.0 # via feast (setup.py) -google-cloud-core==2.4.2 +google-cloud-core==2.4.3 # via # google-cloud-bigquery # google-cloud-bigtable @@ -314,11 +317,9 @@ googleapis-common-protos[grpc]==1.69.1 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) -greenlet==3.1.1 - # via sqlalchemy grpc-google-iam-v1==0.14.1 # via google-cloud-bigtable -grpcio==1.70.0 +grpcio==1.71.0 # via # feast (setup.py) # google-api-core @@ -333,17 +334,17 @@ grpcio==1.70.0 # ikvpy # pymilvus # qdrant-client -grpcio-health-checking==1.70.0 +grpcio-health-checking==1.71.0 # via feast (setup.py) -grpcio-reflection==1.70.0 +grpcio-reflection==1.71.0 # via feast (setup.py) -grpcio-status==1.70.0 +grpcio-status==1.71.0 # via # google-api-core # ikvpy -grpcio-testing==1.70.0 +grpcio-testing==1.71.0 # via feast (setup.py) -grpcio-tools==1.70.0 +grpcio-tools==1.71.0 # via # feast (setup.py) # qdrant-client @@ -375,7 +376,7 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client -huggingface-hub==0.29.2 +huggingface-hub==0.29.3 # via # docling # docling-ibm-models @@ -389,7 +390,7 @@ ibis-framework[duckdb, mssql]==9.0.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.8 +identify==2.6.9 # via pre-commit idna==3.10 # via @@ -526,7 +527,9 @@ lxml==5.3.1 # python-docx # python-pptx lz4==4.4.3 - # via trino + # via + # clickhouse-connect + # trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 @@ -744,7 +747,7 @@ propcache==0.3.0 # via # aiohttp # yarl -proto-plus==1.26.0 +proto-plus==1.26.1 # via # google-api-core # google-cloud-bigquery-storage @@ -931,6 +934,7 @@ python-pptx==1.0.2 # via docling pytz==2025.1 # via + # clickhouse-connect # great-expectations # ibis-framework # pandas @@ -950,7 +954,7 @@ pyyaml==6.0.2 # responses # transformers # uvicorn -pyzmq==26.2.1 +pyzmq==26.3.0 # via # ipykernel # jupyter-client @@ -999,7 +1003,7 @@ requests-oauthlib==2.0.0 # kubernetes requests-toolbelt==1.0.0 # via python-keycloak -responses==0.25.6 +responses==0.25.7 # via moto rfc3339-validator==0.1.4 # via @@ -1025,7 +1029,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.9 +ruff==0.9.10 # via feast (setup.py) s3transfer==0.11.3 # via boto3 @@ -1045,7 +1049,7 @@ semchunk==2.2.2 # via docling-core send2trash==1.8.3 # via jupyter-server -setuptools==75.8.2 +setuptools==76.0.0 # via # grpcio-tools # jupyterlab @@ -1099,7 +1103,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sqlalchemy[mypy]==2.0.38 +sqlalchemy[mypy]==2.0.39 # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework @@ -1109,11 +1113,11 @@ sqlparams==6.2.0 # via singlestoredb stack-data==0.6.3 # via ipython -starlette==0.46.0 +starlette==0.46.1 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.3 +sympy==1.13.1 # via torch tabulate==0.9.0 # via @@ -1134,7 +1138,7 @@ tifffile==2024.8.30 # via scikit-image tinycss2==1.4.0 # via bleach -tokenizers==0.19.1 +tokenizers==0.21.0 # via transformers toml==0.10.2 # via feast (setup.py) @@ -1156,14 +1160,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.2.2 +torch==2.6.0 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.17.2 +torchvision==0.21.0 # via # feast (setup.py) # docling-ibm-models @@ -1202,7 +1206,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -transformers==4.42.4 +transformers==4.49.0 # via # docling-core # docling-ibm-models @@ -1297,6 +1301,7 @@ urllib3==1.26.20 # via # feast (setup.py) # botocore + # clickhouse-connect # docker # elastic-transport # great-expectations @@ -1356,4 +1361,6 @@ yarl==1.18.3 zipp==3.21.0 # via importlib-metadata zstandard==0.23.0 - # via trino + # via + # clickhouse-connect + # trino diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 3b6fddd2cb2..8daedb75c88 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -6,7 +6,7 @@ anyio==4.8.0 # via # starlette # watchfiles -attrs==25.1.0 +attrs==25.2.0 # via # jsonschema # referencing @@ -37,7 +37,7 @@ exceptiongroup==1.2.2 # via anyio fastapi==0.115.11 # via feast (setup.py) -fsspec==2025.2.0 +fsspec==2025.3.0 # via dask gunicorn==23.0.0 # via @@ -132,9 +132,9 @@ six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio -sqlalchemy[mypy]==2.0.38 +sqlalchemy[mypy]==2.0.39 # via feast (setup.py) -starlette==0.46.0 +starlette==0.46.1 # via fastapi tabulate==0.9.0 # via feast (setup.py) From df6ad54322403ae55a6336154a2e954133e83161 Mon Sep 17 00:00:00 2001 From: Tomasz Wrona Date: Wed, 12 Mar 2025 16:26:38 +0100 Subject: [PATCH 22/22] Pin torch to 2.2.2 Signed-off-by: Tomasz Wrona --- pyproject.toml | 2 +- sdk/python/requirements/py3.10-ci-requirements.txt | 6 +++--- sdk/python/requirements/py3.11-ci-requirements.txt | 6 +++--- sdk/python/requirements/py3.9-ci-requirements.txt | 6 +++--- setup.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 00a5d8e81d9..f24235176b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,7 @@ opentelemetry = ["prometheus_client", "psutil"] spark = ["pyspark>=3.0.0,<4"] trino = ["trino>=0.305.0,<0.400.0", "regex"] postgres = ["psycopg[binary,pool]>=3.0.0,<4"] -pytorch = ["torch>=2.2.2", "torchvision>=0.17.2"] +pytorch = ["torch==2.2.2", "torchvision>=0.17.2"] qdrant = ["qdrant-client>=1.12.0"] redis = [ "redis>=4.2.2,<5", diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 3347e22ed5a..51e68920f08 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1104,7 +1104,7 @@ starlette==0.46.1 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.1 +sympy==1.13.3 # via torch tabulate==0.9.0 # via @@ -1147,14 +1147,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.6.0 +torch==2.2.2 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.21.0 +torchvision==0.17.2 # via # feast (setup.py) # docling-ibm-models diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index c5c3d22fceb..da6cb402ebc 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1098,7 +1098,7 @@ starlette==0.46.1 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.1 +sympy==1.13.3 # via torch tabulate==0.9.0 # via @@ -1133,14 +1133,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.6.0 +torch==2.2.2 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.21.0 +torchvision==0.17.2 # via # feast (setup.py) # docling-ibm-models diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index ef2604f14be..c2d8966c5ea 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1117,7 +1117,7 @@ starlette==0.46.1 # via fastapi substrait==0.23.0 # via ibis-substrait -sympy==1.13.1 +sympy==1.13.3 # via torch tabulate==0.9.0 # via @@ -1160,14 +1160,14 @@ toolz==0.12.1 # dask # ibis-framework # partd -torch==2.6.0 +torch==2.2.2 # via # feast (setup.py) # docling-ibm-models # easyocr # safetensors # torchvision -torchvision==0.21.0 +torchvision==0.17.2 # via # feast (setup.py) # docling-ibm-models diff --git a/setup.py b/setup.py index f7273365fc3..503329aa6d8 100644 --- a/setup.py +++ b/setup.py @@ -164,7 +164,7 @@ MILVUS_REQUIRED = ["pymilvus"] TORCH_REQUIRED = [ - "torch>=2.2.2", + "torch==2.2.2", "torchvision>=0.17.2", ]