From a9be27d34f398248cc3e01be23fe4bb9182d1209 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 4 May 2024 22:04:23 -0400 Subject: [PATCH 01/58] feat: Adding vector search for sqlite Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 5 ++ .../example_repos/example_feature_repo_1.py | 22 ++++++- .../online_store/test_online_retrieval.py | 60 ++++++++++++++++++- setup.py | 1 + 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 63d3ef03f51..3034e82e8c7 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -14,6 +14,7 @@ import itertools import os import sqlite3 +import sqlite_vss from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple @@ -73,6 +74,10 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) + db = sqlite3.connect(':memory:') + db.enable_load_extension(True) + sqlite_vss.load(db) + return self._conn def online_write_batch( diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index fbf1fbb9b07..579e7634f17 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -4,7 +4,7 @@ from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Int64, String +from feast.types import Float32, Int64, String, Array # Note that file source paths are not validated, so there doesn't actually need to be any data # at the paths for these file sources. Since these paths are effectively fake, this example @@ -32,6 +32,12 @@ batch_source=driver_locations_source, ) +rag_documents_source = FileSource( + name="rag_documents_source", + path="data/rag_documents.parquet", + timestamp_field="event_timestamp", +) + driver = Entity( name="driver", # The name is derived from this argument, not object name. join_keys=["driver_id"], @@ -43,6 +49,10 @@ join_keys=["customer_id"], ) +item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], +) driver_locations = FeatureView( name="driver_locations", @@ -101,6 +111,16 @@ tags={}, ) +document_embeddings = FeatureView( + name="document_embeddings", + entities=[item], + schema=[ + Field(name="Embeddings", dtype=Array(Float32)), + Field(name="item_id", dtype=String), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), +) @on_demand_feature_view( sources=[customer_profile], diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 5368b1e11cd..cf3457653fc 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -3,6 +3,7 @@ from datetime import datetime import pandas as pd +import numpy as np import pytest from pandas.testing import assert_frame_equal @@ -14,13 +15,13 @@ from tests.utils.cli_repo_creator import CliRunner, get_example_repo -def test_online() -> None: +def test_get_online_features() -> None: """ Test reading from the online store in local mode. """ runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write some data to two tables driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -277,7 +278,7 @@ def test_online_to_df(): runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write three tables to online store driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -415,3 +416,56 @@ def test_online_to_df(): ] expected_df = pd.DataFrame({k: reversed(v) for (k, v) in df_dict.items()}) assert_frame_equal(result_df[ordered_column], expected_df) + + +def test_get_online_Documents() -> None: + """ + Test retrieving documents from the online store in local mode. + """ + runner = CliRunner() + with runner.local_repo( + get_example_repo("example_feature_repo_1.py"), "file" + ) as store: + # Write some data to two tables + document_embeddings_fv = store.get_feature_view(name="document_embeddings") + + provider = store._get_provider() + + item_key = EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=0)] + ) + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=[ + ( + item_key, + { + "Embeddings": [ + np.array([0.17517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408 , 0.02616226]), + np.array([0.18517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408, 0.02616226]), + np.array( + [0.19517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, + 0.01173803, -0.0573408, 0.02616226]), + np.array( + [0.20517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, + 0.01173803, -0.0573408, 0.02616226]), + ] + }, + datetime.utcnow(), + datetime.utcnow(), + ) + ], + progress=None, + ) + + query = np.array([ 0.17517076, -0.1259909 , 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408 , 0.02616226]) + # Retrieve two features using two keys, one valid one non-existing + result = store.retrieve_online_documents( + feature="document_embeddings:Embeddings", + query=query, + top_k=3 + ).to_dict() + + assert "Embeddings" in result + assert result["driver_id"] == [0] diff --git a/setup.py b/setup.py index cdab69b6848..236affeaa5c 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", + "sqlite-vss", "SQLAlchemy[mypy]>1", "tabulate>=0.8.0,<1", "tenacity>=7,<9", From 2addd450fd7cc17b279516ed3c379ae7df6dce3e Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 4 May 2024 22:25:02 -0400 Subject: [PATCH 02/58] adding the sqlite_vss dependency Signed-off-by: Francisco Javier Arceo --- .../requirements/py-ci-requirements.txt | 1049 +++++++++++++++++ sdk/python/requirements/py-requirements.txt | 229 ++++ 2 files changed, 1278 insertions(+) create mode 100644 sdk/python/requirements/py-ci-requirements.txt create mode 100644 sdk/python/requirements/py-requirements.txt diff --git a/sdk/python/requirements/py-ci-requirements.txt b/sdk/python/requirements/py-ci-requirements.txt new file mode 100644 index 00000000000..9e6a0e1a031 --- /dev/null +++ b/sdk/python/requirements/py-ci-requirements.txt @@ -0,0 +1,1049 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=ci --output-file=sdk/python/requirements/py-ci-requirements.txt +# +alabaster==0.7.16 + # via sphinx +altair==4.2.2 + # via great-expectations +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # httpx + # jupyter-server + # starlette + # watchfiles +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asn1crypto==1.5.1 + # via snowflake-connector-python +assertpy==1.1 + # via feast (setup.py) +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 + # via redis +atpublic==4.1.0 + # via ibis-framework +attrs==23.2.0 + # via + # jsonschema + # referencing +azure-core==1.30.1 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.16.0 + # via feast (setup.py) +azure-storage-blob==12.19.1 + # via feast (setup.py) +babel==2.14.0 + # via + # jupyterlab-server + # sphinx +beautifulsoup4==4.12.3 + # via nbconvert +bidict==0.23.1 + # via ibis-framework +bleach==6.1.0 + # via nbconvert +boto3==1.34.98 + # via + # feast (setup.py) + # moto +botocore==1.34.98 + # via + # boto3 + # moto + # s3transfer +build==1.2.1 + # via + # feast (setup.py) + # pip-tools +cachecontrol==0.14.0 + # via firebase-admin +cachetools==5.3.3 + # via google-auth +cassandra-driver==3.29.1 + # via feast (setup.py) +certifi==2024.2.2 + # via + # httpcore + # httpx + # kubernetes + # minio + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # argon2-cffi-bindings + # cryptography + # snowflake-connector-python +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # dask + # feast (setup.py) + # geomet + # great-expectations + # pip-tools + # typer + # uvicorn +cloudpickle==3.0.0 + # via dask +colorama==0.4.6 + # via + # feast (setup.py) + # great-expectations +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.5.1 + # via pytest-cov +cryptography==42.0.6 + # via + # azure-identity + # azure-storage-blob + # feast (setup.py) + # great-expectations + # moto + # msal + # pyjwt + # pyopenssl + # snowflake-connector-python + # types-pyopenssl + # types-redis +dask[array,dataframe]==2024.5.0 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.1.0 + # via dask +db-dtypes==1.2.0 + # via google-cloud-bigquery +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +deltalake==0.17.3 + # via feast (setup.py) +dill==0.3.8 + # via feast (setup.py) +distlib==0.3.8 + # via virtualenv +dnspython==2.6.1 + # via email-validator +docker==7.0.0 + # via + # feast (setup.py) + # testcontainers +docutils==0.19 + # via sphinx +duckdb==0.10.2 + # via ibis-framework +email-validator==2.1.1 + # via fastapi +entrypoints==0.4 + # via altair +exceptiongroup==1.2.1 + # via + # anyio + # ipython + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.0.1 + # via stack-data +fastapi==0.111.0 + # via + # fastapi-cli + # feast (setup.py) +fastapi-cli==0.0.2 + # via fastapi +fastjsonschema==2.19.1 + # via nbformat +filelock==3.14.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 + # via feast (setup.py) +fqdn==1.5.1 + # via jsonschema +fsspec==2023.12.2 + # via + # dask + # feast (setup.py) +geojson==2.5.0 + # via rockset +geomet==0.2.1.post1 + # via cassandra-driver +google-api-core[grpc]==2.19.0 + # via + # feast (setup.py) + # firebase-admin + # google-api-python-client + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-api-python-client==2.127.0 + # via firebase-admin +google-auth==2.29.0 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-bigquery-storage + # google-cloud-core + # google-cloud-firestore + # google-cloud-storage + # kubernetes +google-auth-httplib2==0.2.0 + # via google-api-python-client +google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-core==2.4.1 + # via + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-cloud-datastore==2.19.0 + # via feast (setup.py) +google-cloud-firestore==2.16.0 + # via firebase-admin +google-cloud-storage==2.16.0 + # via + # feast (setup.py) + # firebase-admin +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.63.0 + # via + # feast (setup.py) + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +great-expectations==0.18.13 + # via feast (setup.py) +grpc-google-iam-v1==0.13.0 + # via google-cloud-bigtable +grpcio==1.63.0 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools +grpcio-health-checking==1.62.2 + # via feast (setup.py) +grpcio-reflection==1.62.2 + # via feast (setup.py) +grpcio-status==1.62.2 + # via google-api-core +grpcio-testing==1.62.2 + # via feast (setup.py) +grpcio-tools==1.62.2 + # via feast (setup.py) +gunicorn==22.0.0 ; platform_system != "Windows" + # via feast (setup.py) +h11==0.14.0 + # via + # httpcore + # uvicorn +happybase==1.2.0 + # via feast (setup.py) +hazelcast-python-client==5.3.0 + # via feast (setup.py) +hiredis==2.3.2 + # via feast (setup.py) +httpcore==1.0.5 + # via httpx +httplib2==0.22.0 + # via + # google-api-python-client + # google-auth-httplib2 +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via + # fastapi + # feast (setup.py) + # jupyterlab +ibis-framework[duckdb]==9.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.36 + # via pre-commit +idna==3.7 + # via + # anyio + # email-validator + # httpx + # jsonschema + # requests + # snowflake-connector-python +imagesize==1.4.1 + # via sphinx +importlib-metadata==7.1.0 + # via + # build + # dask + # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # nbconvert + # sphinx + # typeguard +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.4 + # via jupyterlab +ipython==8.18.1 + # via + # great-expectations + # ipykernel + # ipywidgets +ipywidgets==8.1.2 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython +jinja2==3.1.3 + # via + # altair + # fastapi + # feast (setup.py) + # great-expectations + # jupyter-server + # jupyterlab + # jupyterlab-server + # moto + # nbconvert + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +json5==0.9.25 + # via jupyterlab-server +jsonpatch==1.33 + # via great-expectations +jsonpointer==2.4 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.22.0 + # via + # altair + # feast (setup.py) + # great-expectations + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter-client==8.6.1 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.0 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.1.8 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.1 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.10 + # via ipywidgets +kubernetes==20.13.0 + # via feast (setup.py) +locket==1.0.0 + # via partd +makefun==1.15.2 + # via great-expectations +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.21.2 + # via great-expectations +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdurl==0.1.2 + # via markdown-it-py +minio==7.1.0 + # via feast (setup.py) +mistune==3.0.2 + # via + # great-expectations + # nbconvert +mmh3==4.1.0 + # via feast (setup.py) +mock==2.0.0 + # via feast (setup.py) +moto==4.2.14 + # via feast (setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +msgpack==1.0.8 + # via cachecontrol +mypy==1.10.0 + # via + # feast (setup.py) + # sqlalchemy +mypy-extensions==1.0.0 + # via mypy +mypy-protobuf==3.3.0 + # via feast (setup.py) +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # great-expectations + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.8.0 + # via pre-commit +notebook==7.1.3 + # via great-expectations +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook +numpy==1.26.4 + # via + # altair + # dask + # db-dtypes + # feast (setup.py) + # great-expectations + # ibis-framework + # pandas + # pyarrow + # scipy +oauthlib==3.2.2 + # via requests-oauthlib +orjson==3.10.3 + # via fastapi +overrides==7.7.0 + # via jupyter-server +packaging==24.0 + # via + # build + # dask + # db-dtypes + # docker + # google-cloud-bigquery + # great-expectations + # gunicorn + # ibis-substrait + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # msal-extensions + # nbconvert + # pytest + # snowflake-connector-python + # sphinx +pandas==2.2.2 + # via + # altair + # dask + # dask-expr + # db-dtypes + # feast (setup.py) + # google-cloud-bigquery + # great-expectations + # ibis-framework + # snowflake-connector-python +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +parsy==2.1 + # via ibis-framework +partd==1.4.1 + # via dask +pbr==6.0.0 + # via mock +pexpect==4.9.0 + # via ipython +pip-tools==7.4.1 + # via feast (setup.py) +platformdirs==3.11.0 + # via + # jupyter-core + # snowflake-connector-python + # virtualenv +pluggy==1.5.0 + # via pytest +ply==3.11 + # via thriftpy2 +portalocker==2.8.2 + # via msal-extensions +pre-commit==3.3.1 + # via feast (setup.py) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.43 + # via ipython +proto-plus==1.23.0 + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore +protobuf==4.25.3 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools + # mypy-protobuf + # proto-plus + # substrait +psutil==5.9.0 + # via + # feast (setup.py) + # ipykernel +psycopg2-binary==2.9.9 + # via feast (setup.py) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via feast (setup.py) +py-cpuinfo==9.0.0 + # via pytest-benchmark +py4j==0.10.9.7 + # via pyspark +pyarrow==16.0.0 + # via + # dask-expr + # db-dtypes + # deltalake + # feast (setup.py) + # google-cloud-bigquery + # ibis-framework + # snowflake-connector-python +pyarrow-hotfix==0.6 + # via + # deltalake + # ibis-framework +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pybindgen==0.22.1 + # via feast (setup.py) +pycparser==2.22 + # via cffi +pydantic==2.7.1 + # via + # fastapi + # feast (setup.py) + # great-expectations +pydantic-core==2.18.2 + # via pydantic +pygments==2.18.0 + # via + # feast (setup.py) + # ipython + # nbconvert + # rich + # sphinx +pyjwt[crypto]==2.8.0 + # via + # msal + # snowflake-connector-python +pymssql==2.3.0 + # via feast (setup.py) +pymysql==1.1.0 + # via feast (setup.py) +pyodbc==5.1.0 + # via feast (setup.py) +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via + # great-expectations + # httplib2 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pyspark==3.5.1 + # via feast (setup.py) +pytest==7.4.4 + # via + # feast (setup.py) + # pytest-benchmark + # pytest-cov + # pytest-env + # pytest-lazy-fixture + # pytest-mock + # pytest-ordering + # pytest-timeout + # pytest-xdist +pytest-benchmark==3.4.1 + # via feast (setup.py) +pytest-cov==5.0.0 + # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) +pytest-lazy-fixture==0.6.3 + # via feast (setup.py) +pytest-mock==1.10.4 + # via feast (setup.py) +pytest-ordering==0.6 + # via feast (setup.py) +pytest-timeout==1.4.2 + # via feast (setup.py) +pytest-xdist==3.6.1 + # via feast (setup.py) +python-dateutil==2.9.0.post0 + # via + # arrow + # botocore + # google-cloud-bigquery + # great-expectations + # ibis-framework + # jupyter-client + # kubernetes + # moto + # pandas + # rockset + # trino +python-dotenv==1.0.1 + # via uvicorn +python-json-logger==2.0.7 + # via jupyter-events +python-multipart==0.0.9 + # via fastapi +pytz==2024.1 + # via + # great-expectations + # ibis-framework + # pandas + # snowflake-connector-python + # trino +pyyaml==6.0.1 + # via + # dask + # feast (setup.py) + # ibis-substrait + # jupyter-events + # kubernetes + # pre-commit + # responses + # uvicorn +pyzmq==26.0.3 + # via + # ipykernel + # jupyter-client + # jupyter-server +redis==4.6.0 + # via feast (setup.py) +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2024.4.28 + # via feast (setup.py) +requests==2.31.0 + # via + # azure-core + # cachecontrol + # docker + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # great-expectations + # jupyterlab-server + # kubernetes + # moto + # msal + # requests-oauthlib + # responses + # snowflake-connector-python + # sphinx + # trino +requests-oauthlib==2.0.0 + # via kubernetes +responses==0.25.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rich==13.7.1 + # via + # ibis-framework + # typer +rockset==2.1.2 + # via feast (setup.py) +rpds-py==0.18.0 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +ruamel-yaml==0.17.17 + # via great-expectations +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +ruff==0.4.3 + # via feast (setup.py) +s3transfer==0.10.1 + # via boto3 +scipy==1.13.0 + # via great-expectations +send2trash==1.8.3 + # via jupyter-server +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # asttokens + # azure-core + # bleach + # geomet + # happybase + # isodate + # kubernetes + # mock + # python-dateutil + # rfc3339-validator + # thriftpy2 +sniffio==1.3.1 + # via + # anyio + # httpx +snowballstemmer==2.2.0 + # via sphinx +snowflake-connector-python[pandas]==3.10.0 + # via feast (setup.py) +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sphinx==6.2.1 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy[mypy]==2.0.29 + # via feast (setup.py) +sqlglot==23.12.2 + # via ibis-framework +sqlite-vss==0.1.2 + # via feast (setup.py) +stack-data==0.6.3 + # via ipython +starlette==0.37.2 + # via fastapi +substrait==0.17.0 + # via ibis-substrait +tabulate==0.9.0 + # via feast (setup.py) +tenacity==8.2.3 + # via feast (setup.py) +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +testcontainers==4.4.0 + # via feast (setup.py) +thriftpy2==0.4.20 + # via happybase +tinycss2==1.3.0 + # via nbconvert +toml==0.10.2 + # via feast (setup.py) +tomli==2.0.1 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pytest + # pytest-env +tomlkit==0.12.4 + # via snowflake-connector-python +toolz==0.12.1 + # via + # altair + # dask + # ibis-framework + # partd +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +tqdm==4.66.4 + # via + # feast (setup.py) + # great-expectations +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +trino==0.328.0 + # via feast (setup.py) +typeguard==4.2.1 + # via feast (setup.py) +typer==0.12.3 + # via fastapi-cli +types-cffi==1.16.0.20240331 + # via types-pyopenssl +types-protobuf==3.19.22 + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240425 + # via feast (setup.py) +types-pyopenssl==24.1.0.20240425 + # via types-redis +types-python-dateutil==2.9.0.20240316 + # via + # arrow + # feast (setup.py) +types-pytz==2024.1.0.20240417 + # via feast (setup.py) +types-pyyaml==6.0.12.20240311 + # via feast (setup.py) +types-redis==4.6.0.20240425 + # via feast (setup.py) +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==69.5.0.20240423 + # via + # feast (setup.py) + # types-cffi +types-tabulate==0.9.0.20240106 + # via feast (setup.py) +types-urllib3==1.26.25.14 + # via types-requests +typing-extensions==4.11.0 + # via + # anyio + # async-lru + # azure-core + # azure-storage-blob + # fastapi + # great-expectations + # ibis-framework + # ipython + # mypy + # pydantic + # pydantic-core + # snowflake-connector-python + # sqlalchemy + # starlette + # testcontainers + # typeguard + # typer + # uvicorn +tzdata==2024.1 + # via pandas +tzlocal==5.2 + # via + # great-expectations + # trino +ujson==5.9.0 + # via fastapi +uri-template==1.3.0 + # via jsonschema +uritemplate==4.1.1 + # via google-api-python-client +urllib3==1.26.18 + # via + # botocore + # docker + # feast (setup.py) + # great-expectations + # kubernetes + # minio + # requests + # responses + # rockset + # snowflake-connector-python + # testcontainers +uvicorn[standard]==0.29.0 + # via + # fastapi + # fastapi-cli + # feast (setup.py) +uvloop==0.19.0 + # via uvicorn +virtualenv==20.23.0 + # via + # feast (setup.py) + # pre-commit +watchfiles==0.21.0 + # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via + # jupyter-server + # kubernetes +websockets==12.0 + # via uvicorn +werkzeug==3.0.2 + # via moto +wheel==0.43.0 + # via pip-tools +widgetsnbextension==4.0.10 + # via ipywidgets +wrapt==1.16.0 + # via testcontainers +xmltodict==0.13.0 + # via moto +zipp==3.18.1 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/sdk/python/requirements/py-requirements.txt b/sdk/python/requirements/py-requirements.txt new file mode 100644 index 00000000000..c738559e551 --- /dev/null +++ b/sdk/python/requirements/py-requirements.txt @@ -0,0 +1,229 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --output-file=sdk/python/requirements/py-requirements.txt +# +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # httpx + # starlette + # watchfiles +attrs==23.2.0 + # via + # jsonschema + # referencing +certifi==2024.2.2 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dask + # feast (setup.py) + # typer + # uvicorn +cloudpickle==3.0.0 + # via dask +colorama==0.4.6 + # via feast (setup.py) +dask[array,dataframe]==2024.5.0 + # via + # dask-expr + # feast (setup.py) +dask-expr==1.1.0 + # via dask +dill==0.3.8 + # via feast (setup.py) +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via fastapi +exceptiongroup==1.2.1 + # via anyio +fastapi==0.111.0 + # via + # fastapi-cli + # feast (setup.py) +fastapi-cli==0.0.2 + # via fastapi +fsspec==2024.3.1 + # via dask +gunicorn==22.0.0 ; platform_system != "Windows" + # via feast (setup.py) +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via fastapi +idna==3.7 + # via + # anyio + # email-validator + # httpx + # requests +importlib-metadata==7.1.0 + # via + # dask + # typeguard +jinja2==3.1.3 + # via + # fastapi + # feast (setup.py) +jsonschema==4.22.0 + # via feast (setup.py) +jsonschema-specifications==2023.12.1 + # via jsonschema +locket==1.0.0 + # via partd +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +mmh3==4.1.0 + # via feast (setup.py) +mypy==1.10.0 + # via sqlalchemy +mypy-extensions==1.0.0 + # via mypy +mypy-protobuf==3.6.0 + # via feast (setup.py) +numpy==1.26.4 + # via + # dask + # feast (setup.py) + # pandas + # pyarrow +orjson==3.10.3 + # via fastapi +packaging==24.0 + # via + # dask + # gunicorn +pandas==2.2.2 + # via + # dask + # dask-expr + # feast (setup.py) +partd==1.4.1 + # via dask +protobuf==4.25.3 + # via + # feast (setup.py) + # mypy-protobuf +pyarrow==16.0.0 + # via + # dask-expr + # feast (setup.py) +pydantic==2.7.1 + # via + # fastapi + # feast (setup.py) +pydantic-core==2.18.2 + # via pydantic +pygments==2.18.0 + # via + # feast (setup.py) + # rich +python-dateutil==2.9.0.post0 + # via pandas +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.9 + # via fastapi +pytz==2024.1 + # via pandas +pyyaml==6.0.1 + # via + # dask + # feast (setup.py) + # uvicorn +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via feast (setup.py) +rich==13.7.1 + # via typer +rpds-py==0.18.0 + # via + # jsonschema + # referencing +shellingham==1.5.4 + # via typer +six==1.16.0 + # via python-dateutil +sniffio==1.3.1 + # via + # anyio + # httpx +sqlalchemy[mypy]==2.0.29 + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) +starlette==0.37.2 + # via fastapi +tabulate==0.9.0 + # via feast (setup.py) +tenacity==8.2.3 + # via feast (setup.py) +toml==0.10.2 + # via feast (setup.py) +tomli==2.0.1 + # via mypy +toolz==0.12.1 + # via + # dask + # partd +tqdm==4.66.4 + # via feast (setup.py) +typeguard==4.2.1 + # via feast (setup.py) +typer==0.12.3 + # via fastapi-cli +types-protobuf==5.26.0.20240422 + # via mypy-protobuf +typing-extensions==4.11.0 + # via + # anyio + # fastapi + # mypy + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typeguard + # typer + # uvicorn +tzdata==2024.1 + # via pandas +ujson==5.9.0 + # via fastapi +urllib3==2.2.1 + # via requests +uvicorn[standard]==0.29.0 + # via + # fastapi + # fastapi-cli + # feast (setup.py) +uvloop==0.19.0 + # via uvicorn +watchfiles==0.21.0 + # via uvicorn +websockets==12.0 + # via uvicorn +zipp==3.18.1 + # via importlib-metadata From 4f9070baf3010037aeb08e5e7e3aec12fc85518c Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 5 May 2024 22:03:38 -0400 Subject: [PATCH 03/58] linter Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 4 +- .../example_repos/example_feature_repo_1.py | 5 +- .../online_store/test_online_retrieval.py | 87 ++++++++++++++++--- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 3034e82e8c7..0e9015af9c8 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -14,11 +14,11 @@ import itertools import os import sqlite3 -import sqlite_vss from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +import sqlite_vss from pydantic import StrictStr from feast import Entity @@ -74,7 +74,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - db = sqlite3.connect(':memory:') + db = sqlite3.connect(":memory:") db.enable_load_extension(True) sqlite_vss.load(db) diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index 579e7634f17..20a8ad7bd86 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -4,7 +4,7 @@ from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Int64, String, Array +from feast.types import Array, Float32, Int64, String # Note that file source paths are not validated, so there doesn't actually need to be any data # at the paths for these file sources. Since these paths are effectively fake, this example @@ -50,7 +50,7 @@ ) item = Entity( - name="item_id", # The name is derived from this argument, not object name. + name="item_id", # The name is derived from this argument, not object name. join_keys=["item_id"], ) @@ -122,6 +122,7 @@ ttl=timedelta(hours=24), ) + @on_demand_feature_view( sources=[customer_profile], schema=[Field(name="on_demand_age", dtype=Int64)], diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index cf3457653fc..c656acd7acc 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -2,8 +2,8 @@ import time from datetime import datetime -import pandas as pd import numpy as np +import pandas as pd import pytest from pandas.testing import assert_frame_equal @@ -21,7 +21,7 @@ def test_get_online_features() -> None: """ runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write some data to two tables driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -278,7 +278,7 @@ def test_online_to_df(): runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write three tables to online store driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -424,7 +424,7 @@ def test_get_online_Documents() -> None: """ runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write some data to two tables document_embeddings_fv = store.get_feature_view(name="document_embeddings") @@ -442,14 +442,62 @@ def test_get_online_Documents() -> None: item_key, { "Embeddings": [ - np.array([0.17517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408 , 0.02616226]), - np.array([0.18517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408, 0.02616226]), np.array( - [0.19517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, - 0.01173803, -0.0573408, 0.02616226]), + [ + 0.17517076, + -0.1259909, + 0.01954236, + 0.03045186, + -0.00074535, + -0.02715777, + -0.04582673, + 0.01173803, + -0.0573408, + 0.02616226, + ] + ), + np.array( + [ + 0.18517076, + -0.1259909, + 0.01954236, + 0.03045186, + -0.00074535, + -0.02715777, + -0.04582673, + 0.01173803, + -0.0573408, + 0.02616226, + ] + ), np.array( - [0.20517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, - 0.01173803, -0.0573408, 0.02616226]), + [ + 0.19517076, + -0.1259909, + 0.01954236, + 0.03045186, + -0.00074535, + -0.02715777, + -0.04582673, + 0.01173803, + -0.0573408, + 0.02616226, + ] + ), + np.array( + [ + 0.20517076, + -0.1259909, + 0.01954236, + 0.03045186, + -0.00074535, + -0.02715777, + -0.04582673, + 0.01173803, + -0.0573408, + 0.02616226, + ] + ), ] }, datetime.utcnow(), @@ -459,12 +507,23 @@ def test_get_online_Documents() -> None: progress=None, ) - query = np.array([ 0.17517076, -0.1259909 , 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408 , 0.02616226]) + query = np.array( + [ + 0.17517076, + -0.1259909, + 0.01954236, + 0.03045186, + -0.00074535, + -0.02715777, + -0.04582673, + 0.01173803, + -0.0573408, + 0.02616226, + ] + ) # Retrieve two features using two keys, one valid one non-existing result = store.retrieve_online_documents( - feature="document_embeddings:Embeddings", - query=query, - top_k=3 + feature="document_embeddings:Embeddings", query=query, top_k=3 ).to_dict() assert "Embeddings" in result From 7599770538d4830eed8d5872f559f9cd00236fe3 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 8 May 2024 22:33:29 -0400 Subject: [PATCH 04/58] latest progress Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 48 +++++++++++++++++++ .../online_store/test_online_retrieval.py | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 0e9015af9c8..6550216154f 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -46,6 +46,9 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ + faiss_enabled: Optional[bool] = False + """ (optional) Enable or disable faiss indexing for online store (vector search)""" + class SqliteOnlineStore(OnlineStore): """ @@ -238,6 +241,51 @@ def teardown( pass + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + """ + + Args: + config: Feast configuration object + table: FeatureView object as the table to search + requested_feature: The requested feature as the column to search + embedding: The query embedding to search for + top_k: The number of items to return + Returns: + List of tuples containing the event timestamp and the document feature + """ + project = config.project + if not config.online_store.faiss_enabled: + raise ValueError("Faiss is not enabled in the online store config") + + # Convert the embedding to a string to be used in postgres vector search + query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" + + result: List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ] = [] + + raise NotImplementedError("SQLiteOnlineStore does not support retrieval") + + def _initialize_conn(db_path: str): Path(db_path).parent.mkdir(exist_ok=True) return sqlite3.connect( diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index c656acd7acc..3e5b87961d3 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -418,7 +418,7 @@ def test_online_to_df(): assert_frame_equal(result_df[ordered_column], expected_df) -def test_get_online_Documents() -> None: +def test_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ From 3f9c0b5c512fef15f3083e3f94cbfec6690492f9 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 15 May 2024 13:44:29 -0400 Subject: [PATCH 05/58] uploading latest progress Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 139 +++++++++++++----- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 6550216154f..9df8ab89bbf 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -46,8 +46,8 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ - faiss_enabled: Optional[bool] = False - """ (optional) Enable or disable faiss indexing for online store (vector search)""" + vss_enabled: Optional[bool] = False + """ (optional) Enable or disable sqlite-vss for vector search""" class SqliteOnlineStore(OnlineStore): @@ -107,35 +107,70 @@ def online_write_batch( created_ts = to_naive_utc(created_ts) for feature_name, val in values.items(): - conn.execute( - f""" - UPDATE {_table_id(project, table)} - SET value = ?, event_ts = ?, created_ts = ? - WHERE (entity_key = ? AND feature_name = ?) - """, - ( - # SET - val.SerializeToString(), - timestamp, - created_ts, - # WHERE - entity_key_bin, - feature_name, - ), - ) - - conn.execute( - f"""INSERT OR IGNORE INTO {_table_id(project, table)} - (entity_key, feature_name, value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?)""", - ( - entity_key_bin, - feature_name, - val.SerializeToString(), - timestamp, - created_ts, - ), - ) + vector_val = None + if config.online_store.vss_enabled: + vector_val = get_list_val_str(val) + conn.execute( + f""" + UPDATE {_table_id(project, table)} + SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, + ( + # SET + val.SerializeToString(), + vector_val, + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + + conn.execute( + f"""INSERT OR IGNORE INTO {_table_id(project, table)} + (entity_key, feature_name, value, vector_value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + vector_val, + timestamp, + created_ts, + ), + ) + else: + conn.execute( + f""" + UPDATE {_table_id(project, table)} + SET value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, + ( + # SET + val.SerializeToString(), + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + + conn.execute( + f"""INSERT OR IGNORE INTO {_table_id(project, table)} + (entity_key, feature_name, value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + ), + ) if progress: progress(1) @@ -265,15 +300,32 @@ def retrieve_online_documents( embedding: The query embedding to search for top_k: The number of items to return Returns: - List of tuples containing the event timestamp and the document feature + List of tuples containing the event timestamp, the document feature, the vector value, and the distance """ project = config.project - if not config.online_store.faiss_enabled: - raise ValueError("Faiss is not enabled in the online store config") + + if not config.online_store.vss_enabled: + raise ValueError("sqlite-vss is not enabled in the online store config") + + conn = self._get_conn(config) + cur = conn.cursor() # Convert the embedding to a string to be used in postgres vector search query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" + cur.execute( + f""" + SELECT entity_key, feature_name, value, vector_value, vector_value <=> ? AS distance, event_ts + FROM {_table_id(project, table)} + WHERE feature_name = ? + ORDER BY distance + LIMIT ? + """, + (query_embedding_str, requested_feature, top_k), + ) + + rows = cur.fetchall() + result: List[ Tuple[ Optional[datetime], @@ -283,7 +335,24 @@ def retrieve_online_documents( ] ] = [] - raise NotImplementedError("SQLiteOnlineStore does not support retrieval") + for entity_key, feature_name, val_bin, vector_val, distance, event_ts in rows: + feature_value_proto = ValueProto() + feature_value_proto.ParseFromString(val_bin) + + vector_value_proto = ValueProto(string_val=vector_val) + distance_value_proto = ValueProto(float_val=distance) + + result.append( + ( + event_ts, + feature_value_proto, + vector_value_proto, + distance_value_proto, + ) + ) + + return result + def _initialize_conn(db_path: str): From 4f1d01be94f3f992896d875682b499460e52ac0a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 15 May 2024 14:16:05 -0400 Subject: [PATCH 06/58] updated function Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 9df8ab89bbf..668292a6eee 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -109,7 +109,7 @@ def online_write_batch( for feature_name, val in values.items(): vector_val = None if config.online_store.vss_enabled: - vector_val = get_list_val_str(val) + vector_val = self._get_list_val_str(val) conn.execute( f""" UPDATE {_table_id(project, table)} @@ -275,6 +275,19 @@ def teardown( except FileNotFoundError: pass + def _get_list_val_str(self, val: ValueProto) -> str: + if val.HasField("string_list_val"): + return ",".join(val.string_list_val.val) + elif val.HasField("bytes_list_val"): + return ",".join(map(str, val.bytes_list_val.val)) + elif val.HasField("int64_list_val"): + return ",".join(map(str, val.int64_list_val.val)) + elif val.HasField("float_list_val"): + return ",".join(map(str, val.float_list_val.val)) + elif val.HasField("double_list_val"): + return ",".join(map(str, val.double_list_val.val)) + else: + raise ValueError("Unsupported list value type") def retrieve_online_documents( self, @@ -354,7 +367,6 @@ def retrieve_online_documents( return result - def _initialize_conn(db_path: str): Path(db_path).parent.mkdir(exist_ok=True) return sqlite3.connect( From 2a33c693cae922af1994cfddcc7e58afe18d319e Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 15 May 2024 21:13:56 -0400 Subject: [PATCH 07/58] adding configuration Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 668292a6eee..36313beb875 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -77,9 +77,10 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - db = sqlite3.connect(":memory:") - db.enable_load_extension(True) - sqlite_vss.load(db) + if config.online_store.vss_enabled: + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + sqlite_vss.load(db) return self._conn From 81e4a278577839d8e405d53036e05b253c301da5 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 16 May 2024 14:58:38 -0400 Subject: [PATCH 08/58] adding current progress Signed-off-by: Francisco Javier Arceo --- docs/reference/alpha-vector-database.md | 12 +++++++ .../feast/infra/online_stores/sqlite.py | 32 ++++++------------- .../online_store/test_online_retrieval.py | 32 ++++++++----------- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 37d9b9cdf87..cdabf195f16 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -108,4 +108,16 @@ def print_online_features(features): print(key, " : ", value) print_online_features(features) +``` + +### Configuration +We offer two Online Store options for Vector Databases. PGVector and SQLite. + +#### Installation with SQLite +If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command: +```bash +PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ + LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" \ + CPPFLAGS="-I/opt/homebrew/opt/sqlite/include" \ + pyenv install ``` \ No newline at end of file diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 36313beb875..b3c4042b884 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -13,12 +13,13 @@ # limitations under the License. import itertools import os +import json import sqlite3 +import sqlite_vss from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple -import sqlite_vss from pydantic import StrictStr from feast import Entity @@ -108,9 +109,9 @@ def online_write_batch( created_ts = to_naive_utc(created_ts) for feature_name, val in values.items(): - vector_val = None if config.online_store.vss_enabled: - vector_val = self._get_list_val_str(val) + print('using vector search') + vector_val = json.dumps(val) conn.execute( f""" UPDATE {_table_id(project, table)} @@ -119,7 +120,7 @@ def online_write_batch( """, ( # SET - val.SerializeToString(), + str(val), vector_val, timestamp, created_ts, @@ -136,13 +137,14 @@ def online_write_batch( ( entity_key_bin, feature_name, - val.SerializeToString(), + str(val), vector_val, timestamp, created_ts, ), ) else: + print('not using vector search') conn.execute( f""" UPDATE {_table_id(project, table)} @@ -238,8 +240,8 @@ def update( project = config.project for table in tables_to_keep: - conn.execute( - f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + self.conn.execute( + f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) conn.execute( f"CREATE INDEX IF NOT EXISTS {_table_id(project, table)}_ek ON {_table_id(project, table)} (entity_key);" @@ -276,20 +278,6 @@ def teardown( except FileNotFoundError: pass - def _get_list_val_str(self, val: ValueProto) -> str: - if val.HasField("string_list_val"): - return ",".join(val.string_list_val.val) - elif val.HasField("bytes_list_val"): - return ",".join(map(str, val.bytes_list_val.val)) - elif val.HasField("int64_list_val"): - return ",".join(map(str, val.int64_list_val.val)) - elif val.HasField("float_list_val"): - return ",".join(map(str, val.float_list_val.val)) - elif val.HasField("double_list_val"): - return ",".join(map(str, val.double_list_val.val)) - else: - raise ValueError("Unsupported list value type") - def retrieve_online_documents( self, config: RepoConfig, @@ -428,7 +416,7 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: def update(self): self.conn.execute( - f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 3e5b87961d3..b31fc0ce9b7 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -426,6 +426,7 @@ def test_get_online_documents() -> None: with runner.local_repo( get_example_repo("example_feature_repo_1.py"), "file" ) as store: + store.config.online_store.vss_enabled = True # Write some data to two tables document_embeddings_fv = store.get_feature_view(name="document_embeddings") @@ -434,15 +435,11 @@ def test_get_online_documents() -> None: item_key = EntityKeyProto( join_keys=["item_id"], entity_values=[ValueProto(int64_val=0)] ) - provider.online_write_batch( - config=store.config, - table=document_embeddings_fv, - data=[ + data = [ ( item_key, { "Embeddings": [ - np.array( [ 0.17517076, -0.1259909, @@ -454,9 +451,7 @@ def test_get_online_documents() -> None: 0.01173803, -0.0573408, 0.02616226, - ] - ), - np.array( + ], [ 0.18517076, -0.1259909, @@ -468,9 +463,7 @@ def test_get_online_documents() -> None: 0.01173803, -0.0573408, 0.02616226, - ] - ), - np.array( + ], [ 0.19517076, -0.1259909, @@ -482,9 +475,7 @@ def test_get_online_documents() -> None: 0.01173803, -0.0573408, 0.02616226, - ] - ), - np.array( + ], [ 0.20517076, -0.1259909, @@ -497,17 +488,20 @@ def test_get_online_documents() -> None: -0.0573408, 0.02616226, ] - ), - ] + ], }, datetime.utcnow(), datetime.utcnow(), ) - ], + ] + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, progress=None, ) - query = np.array( + query_embedding = np.array( [ 0.17517076, -0.1259909, @@ -523,7 +517,7 @@ def test_get_online_documents() -> None: ) # Retrieve two features using two keys, one valid one non-existing result = store.retrieve_online_documents( - feature="document_embeddings:Embeddings", query=query, top_k=3 + feature="document_embeddings:Embeddings", query=query_embedding, top_k=3 ).to_dict() assert "Embeddings" in result From ee300d9b6eb1ecc73db5cfc48ad0d705e1e7b616 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 17 May 2024 12:43:15 -0400 Subject: [PATCH 09/58] updating requirements files Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 3 +- .../requirements/py-ci-requirements.txt | 884 +----------------- sdk/python/requirements/py-requirements.txt | 32 +- .../requirements/py3.10-ci-requirements.txt | 800 +--------------- .../requirements/py3.10-requirements.txt | 57 +- .../requirements/py3.11-ci-requirements.txt | 789 +--------------- .../requirements/py3.11-requirements.txt | 61 +- .../requirements/py3.9-ci-requirements.txt | 807 +--------------- .../requirements/py3.9-requirements.txt | 52 +- 9 files changed, 324 insertions(+), 3161 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index b3c4042b884..91d3291645b 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -240,7 +240,7 @@ def update( project = config.project for table in tables_to_keep: - self.conn.execute( + conn.execute( f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) conn.execute( @@ -285,6 +285,7 @@ def retrieve_online_documents( requested_feature: str, embedding: List[float], top_k: int, + distance_metric: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], diff --git a/sdk/python/requirements/py-ci-requirements.txt b/sdk/python/requirements/py-ci-requirements.txt index 9e6a0e1a031..f6c6987fd63 100644 --- a/sdk/python/requirements/py-ci-requirements.txt +++ b/sdk/python/requirements/py-ci-requirements.txt @@ -1,1049 +1,225 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=ci --output-file=sdk/python/requirements/py-ci-requirements.txt -# -alabaster==0.7.16 - # via sphinx -altair==4.2.2 - # via great-expectations +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py-ci-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx - # jupyter-server # starlette # watchfiles -appnope==0.1.4 - # via ipykernel -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via isoduration -asn1crypto==1.5.1 - # via snowflake-connector-python -assertpy==1.1 - # via feast (setup.py) -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -async-timeout==4.0.3 - # via redis -atpublic==4.1.0 - # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing -azure-core==1.30.1 - # via - # azure-identity - # azure-storage-blob -azure-identity==1.16.0 - # via feast (setup.py) -azure-storage-blob==12.19.1 - # via feast (setup.py) -babel==2.14.0 - # via - # jupyterlab-server - # sphinx -beautifulsoup4==4.12.3 - # via nbconvert -bidict==0.23.1 - # via ibis-framework -bleach==6.1.0 - # via nbconvert -boto3==1.34.98 - # via - # feast (setup.py) - # moto -botocore==1.34.98 - # via - # boto3 - # moto - # s3transfer -build==1.2.1 - # via - # feast (setup.py) - # pip-tools -cachecontrol==0.14.0 - # via firebase-admin -cachetools==5.3.3 - # via google-auth -cassandra-driver==3.29.1 - # via feast (setup.py) certifi==2024.2.2 # via # httpcore # httpx - # kubernetes - # minio # requests - # snowflake-connector-python -cffi==1.16.0 - # via - # argon2-cffi-bindings - # cryptography - # snowflake-connector-python -cfgv==3.4.0 - # via pre-commit charset-normalizer==3.3.2 - # via - # requests - # snowflake-connector-python + # via requests click==8.1.7 # via - # dask # feast (setup.py) - # geomet - # great-expectations - # pip-tools + # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) +dask[dataframe]==2024.5.0 # via # feast (setup.py) - # great-expectations -comm==0.2.2 - # via - # ipykernel - # ipywidgets -coverage[toml]==7.5.1 - # via pytest-cov -cryptography==42.0.6 - # via - # azure-identity - # azure-storage-blob - # feast (setup.py) - # great-expectations - # moto - # msal - # pyjwt - # pyopenssl - # snowflake-connector-python - # types-pyopenssl - # types-redis -dask[array,dataframe]==2024.5.0 - # via # dask-expr - # feast (setup.py) dask-expr==1.1.0 # via dask -db-dtypes==1.2.0 - # via google-cloud-bigquery -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -deltalake==0.17.3 - # via feast (setup.py) dill==0.3.8 # via feast (setup.py) -distlib==0.3.8 - # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 - # via - # feast (setup.py) - # testcontainers -docutils==0.19 - # via sphinx -duckdb==0.10.2 - # via ibis-framework email-validator==2.1.1 # via fastapi -entrypoints==0.4 - # via altair exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest -execnet==2.1.1 - # via pytest-xdist -executing==2.0.1 - # via stack-data + # via anyio fastapi==0.111.0 # via - # fastapi-cli # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via - # snowflake-connector-python - # virtualenv -firebase-admin==5.4.0 - # via feast (setup.py) -fqdn==1.5.1 - # via jsonschema fsspec==2023.12.2 - # via - # dask - # feast (setup.py) -geojson==2.5.0 - # via rockset -geomet==0.2.1.post1 - # via cassandra-driver -google-api-core[grpc]==2.19.0 - # via - # feast (setup.py) - # firebase-admin - # google-api-python-client - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-core - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-api-python-client==2.127.0 - # via firebase-admin -google-auth==2.29.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 - # google-cloud-bigquery-storage - # google-cloud-core - # google-cloud-firestore - # google-cloud-storage - # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.25.0 - # via feast (setup.py) -google-cloud-bigtable==2.23.1 - # via feast (setup.py) -google-cloud-core==2.4.1 - # via - # google-cloud-bigquery - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-cloud-datastore==2.19.0 - # via feast (setup.py) -google-cloud-firestore==2.16.0 - # via firebase-admin -google-cloud-storage==2.16.0 - # via - # feast (setup.py) - # firebase-admin -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 - # via - # feast (setup.py) - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -great-expectations==0.18.13 - # via feast (setup.py) -grpc-google-iam-v1==0.13.0 - # via google-cloud-bigtable -grpcio==1.63.0 - # via - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools -grpcio-health-checking==1.62.2 - # via feast (setup.py) -grpcio-reflection==1.62.2 - # via feast (setup.py) -grpcio-status==1.62.2 - # via google-api-core -grpcio-testing==1.62.2 - # via feast (setup.py) -grpcio-tools==1.62.2 - # via feast (setup.py) -gunicorn==22.0.0 ; platform_system != "Windows" + # via dask +gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -happybase==1.2.0 - # via feast (setup.py) -hazelcast-python-client==5.3.0 - # via feast (setup.py) -hiredis==2.3.2 - # via feast (setup.py) httpcore==1.0.5 # via httpx -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via - # fastapi - # feast (setup.py) - # jupyterlab -ibis-framework[duckdb]==9.0.0 - # via - # feast (setup.py) - # ibis-substrait -ibis-substrait==3.2.0 - # via feast (setup.py) -identify==2.5.36 - # via pre-commit + # via fastapi idna==3.7 # via # anyio # email-validator # httpx - # jsonschema # requests - # snowflake-connector-python -imagesize==1.4.1 - # via sphinx importlib-metadata==7.1.0 # via - # build # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx # typeguard -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.4 - # via jupyterlab -ipython==8.18.1 - # via - # great-expectations - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via great-expectations -isodate==0.6.1 - # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -jedi==0.19.1 - # via ipython jinja2==3.1.3 # via - # altair - # fastapi - # feast (setup.py) - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # moto - # nbconvert - # sphinx -jmespath==1.0.1 - # via - # boto3 - # botocore -json5==0.9.25 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.22.0 - # via - # altair # feast (setup.py) - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat + # fastapi +jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # jupyter-server - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat -jupyter-events==0.10.0 - # via jupyter-server -jupyter-lsp==2.2.5 - # via jupyterlab -jupyter-server==2.14.0 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.5.3 - # via jupyter-server -jupyterlab==4.1.8 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.27.1 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.10 - # via ipywidgets -kubernetes==20.13.0 - # via feast (setup.py) locket==1.0.0 # via partd -makefun==1.15.2 - # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.21.2 - # via great-expectations -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython + # via jinja2 mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 - # via feast (setup.py) -mistune==3.0.2 - # via - # great-expectations - # nbconvert mmh3==4.1.0 # via feast (setup.py) -mock==2.0.0 - # via feast (setup.py) -moto==4.2.14 - # via feast (setup.py) -msal==1.28.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.8 - # via cachecontrol mypy==1.10.0 - # via - # feast (setup.py) - # sqlalchemy + # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 - # via nbconvert -nbconvert==7.16.4 - # via jupyter-server -nbformat==5.10.4 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -notebook==7.1.3 - # via great-expectations -notebook-shim==0.2.4 - # via - # jupyterlab - # notebook numpy==1.26.4 # via - # altair - # dask - # db-dtypes # feast (setup.py) - # great-expectations - # ibis-framework + # dask # pandas # pyarrow - # scipy -oauthlib==3.2.2 - # via requests-oauthlib orjson==3.10.3 # via fastapi -overrides==7.7.0 - # via jupyter-server packaging==24.0 # via - # build # dask - # db-dtypes - # docker - # google-cloud-bigquery - # great-expectations # gunicorn - # ibis-substrait - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # msal-extensions - # nbconvert - # pytest - # snowflake-connector-python - # sphinx pandas==2.2.2 # via - # altair + # feast (setup.py) # dask # dask-expr - # db-dtypes - # feast (setup.py) - # google-cloud-bigquery - # great-expectations - # ibis-framework - # snowflake-connector-python -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.4 - # via jedi -parsy==2.1 - # via ibis-framework partd==1.4.1 # via dask -pbr==6.0.0 - # via mock -pexpect==4.9.0 - # via ipython -pip-tools==7.4.1 - # via feast (setup.py) -platformdirs==3.11.0 - # via - # jupyter-core - # snowflake-connector-python - # virtualenv -pluggy==1.5.0 - # via pytest -ply==3.11 - # via thriftpy2 -portalocker==2.8.2 - # via msal-extensions -pre-commit==3.3.1 - # via feast (setup.py) -prometheus-client==0.20.0 - # via jupyter-server -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore protobuf==4.25.3 # via # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools # mypy-protobuf - # proto-plus - # substrait -psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel -psycopg2-binary==2.9.9 - # via feast (setup.py) -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via feast (setup.py) -py-cpuinfo==9.0.0 - # via pytest-benchmark -py4j==0.10.9.7 - # via pyspark pyarrow==16.0.0 # via - # dask-expr - # db-dtypes - # deltalake # feast (setup.py) - # google-cloud-bigquery - # ibis-framework - # snowflake-connector-python -pyarrow-hotfix==0.6 - # via - # deltalake - # ibis-framework -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pybindgen==0.22.1 - # via feast (setup.py) -pycparser==2.22 - # via cffi + # dask-expr pydantic==2.7.1 # via - # fastapi # feast (setup.py) - # great-expectations + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # feast (setup.py) - # ipython - # nbconvert # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 - # via feast (setup.py) -pymysql==1.1.0 - # via feast (setup.py) -pyodbc==5.1.0 - # via feast (setup.py) -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 - # via feast (setup.py) -pytest==7.4.4 - # via - # feast (setup.py) - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 - # via feast (setup.py) -pytest-cov==5.0.0 - # via feast (setup.py) -pytest-env==1.1.3 - # via feast (setup.py) -pytest-lazy-fixture==0.6.3 - # via feast (setup.py) -pytest-mock==1.10.4 - # via feast (setup.py) -pytest-ordering==0.6 - # via feast (setup.py) -pytest-timeout==1.4.2 - # via feast (setup.py) -pytest-xdist==3.6.1 - # via feast (setup.py) python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # google-cloud-bigquery - # great-expectations - # ibis-framework - # jupyter-client - # kubernetes - # moto - # pandas - # rockset - # trino + # via pandas python-dotenv==1.0.1 # via uvicorn -python-json-logger==2.0.7 - # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via - # great-expectations - # ibis-framework - # pandas - # snowflake-connector-python - # trino + # via pandas pyyaml==6.0.1 # via - # dask # feast (setup.py) - # ibis-substrait - # jupyter-events - # kubernetes - # pre-commit - # responses + # dask # uvicorn -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client - # jupyter-server -redis==4.6.0 - # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications - # jupyter-events -regex==2024.4.28 - # via feast (setup.py) requests==2.31.0 - # via - # azure-core - # cachecontrol - # docker - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # great-expectations - # jupyterlab-server - # kubernetes - # moto - # msal - # requests-oauthlib - # responses - # snowflake-connector-python - # sphinx - # trino -requests-oauthlib==2.0.0 - # via kubernetes -responses==0.25.0 - # via moto -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rich==13.7.1 - # via - # ibis-framework - # typer -rockset==2.1.2 # via feast (setup.py) +rich==13.7.1 + # via typer rpds-py==0.18.0 # via # jsonschema # referencing -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml -ruff==0.4.3 - # via feast (setup.py) -s3transfer==0.10.1 - # via boto3 -scipy==1.13.0 - # via great-expectations -send2trash==1.8.3 - # via jupyter-server shellingham==1.5.4 # via typer six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # geomet - # happybase - # isodate - # kubernetes - # mock - # python-dateutil - # rfc3339-validator - # thriftpy2 + # via python-dateutil sniffio==1.3.1 # via # anyio # httpx -snowballstemmer==2.2.0 - # via sphinx -snowflake-connector-python[pandas]==3.10.0 - # via feast (setup.py) -sortedcontainers==2.4.0 - # via snowflake-connector-python -soupsieve==2.5 - # via beautifulsoup4 -sphinx==6.2.1 - # via feast (setup.py) -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx sqlalchemy[mypy]==2.0.29 # via feast (setup.py) -sqlglot==23.12.2 - # via ibis-framework sqlite-vss==0.1.2 # via feast (setup.py) -stack-data==0.6.3 - # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 - # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 # via feast (setup.py) -terminado==0.18.1 - # via - # jupyter-server - # jupyter-server-terminals -testcontainers==4.4.0 - # via feast (setup.py) -thriftpy2==0.4.20 - # via happybase -tinycss2==1.3.0 - # via nbconvert toml==0.10.2 # via feast (setup.py) tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env -tomlkit==0.12.4 - # via snowflake-connector-python + # via mypy toolz==0.12.1 # via - # altair # dask - # ibis-framework # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado tqdm==4.66.4 - # via - # feast (setup.py) - # great-expectations -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -trino==0.328.0 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) typer==0.12.3 # via fastapi-cli -types-cffi==1.16.0.20240331 - # via types-pyopenssl types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf -types-pymysql==1.1.0.20240425 - # via feast (setup.py) -types-pyopenssl==24.1.0.20240425 - # via types-redis -types-python-dateutil==2.9.0.20240316 - # via - # arrow - # feast (setup.py) -types-pytz==2024.1.0.20240417 - # via feast (setup.py) -types-pyyaml==6.0.12.20240311 - # via feast (setup.py) -types-redis==4.6.0.20240425 - # via feast (setup.py) -types-requests==2.30.0.0 - # via feast (setup.py) -types-setuptools==69.5.0.20240423 - # via - # feast (setup.py) - # types-cffi -types-tabulate==0.9.0.20240106 - # via feast (setup.py) -types-urllib3==1.26.25.14 - # via types-requests + # via mypy-protobuf typing-extensions==4.11.0 # via # anyio - # async-lru - # azure-core - # azure-storage-blob # fastapi - # great-expectations - # ibis-framework - # ipython # mypy # pydantic # pydantic-core - # snowflake-connector-python # sqlalchemy # starlette - # testcontainers # typeguard # typer # uvicorn tzdata==2024.1 # via pandas -tzlocal==5.2 - # via - # great-expectations - # trino ujson==5.9.0 # via fastapi -uri-template==1.3.0 - # via jsonschema -uritemplate==4.1.1 - # via google-api-python-client urllib3==1.26.18 - # via - # botocore - # docker - # feast (setup.py) - # great-expectations - # kubernetes - # minio - # requests - # responses - # rockset - # snowflake-connector-python - # testcontainers + # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli - # feast (setup.py) uvloop==0.19.0 # via uvicorn -virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.13 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.8.0 - # via - # jupyter-server - # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.2 - # via moto -wheel==0.43.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -wrapt==1.16.0 - # via testcontainers -xmltodict==0.13.0 - # via moto zipp==3.18.1 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/sdk/python/requirements/py-requirements.txt b/sdk/python/requirements/py-requirements.txt index c738559e551..7edb05a30f2 100644 --- a/sdk/python/requirements/py-requirements.txt +++ b/sdk/python/requirements/py-requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --output-file=sdk/python/requirements/py-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -24,18 +20,18 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # dask # feast (setup.py) + # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[array,dataframe]==2024.5.0 +dask[dataframe]==2024.5.0 # via - # dask-expr # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 @@ -48,13 +44,13 @@ exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 # via - # fastapi-cli # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -gunicorn==22.0.0 ; platform_system != "Windows" +gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 # via @@ -78,8 +74,8 @@ importlib-metadata==7.1.0 # typeguard jinja2==3.1.3 # via - # fastapi # feast (setup.py) + # fastapi jsonschema==4.22.0 # via feast (setup.py) jsonschema-specifications==2023.12.1 @@ -102,8 +98,8 @@ mypy-protobuf==3.6.0 # via feast (setup.py) numpy==1.26.4 # via - # dask # feast (setup.py) + # dask # pandas # pyarrow orjson==3.10.3 @@ -114,9 +110,9 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr - # feast (setup.py) partd==1.4.1 # via dask protobuf==4.25.3 @@ -125,12 +121,12 @@ protobuf==4.25.3 # mypy-protobuf pyarrow==16.0.0 # via - # dask-expr # feast (setup.py) + # dask-expr pydantic==2.7.1 # via - # fastapi # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 @@ -147,8 +143,8 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via - # dask # feast (setup.py) + # dask # uvicorn referencing==0.35.1 # via @@ -216,9 +212,9 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli - # feast (setup.py) uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index e7ca9ca35b6..913db69ed5e 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,933 +1,225 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -alabaster==0.7.16 - # via sphinx -altair==4.2.2 - # via great-expectations +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-ci-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx - # jupyter-server # starlette # watchfiles -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via isoduration -asn1crypto==1.5.1 - # via snowflake-connector-python -assertpy==1.1 -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -async-timeout==4.0.3 - # via redis -atpublic==4.1.0 - # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing -azure-core==1.30.1 - # via - # azure-identity - # azure-storage-blob -azure-identity==1.16.0 -azure-storage-blob==12.19.1 -babel==2.15.0 - # via - # jupyterlab-server - # sphinx -beautifulsoup4==4.12.3 - # via nbconvert -bidict==0.23.1 - # via ibis-framework -bleach==6.1.0 - # via nbconvert -boto3==1.34.99 - # via moto -botocore==1.34.99 - # via - # boto3 - # moto - # s3transfer -build==1.2.1 - # via pip-tools -cachecontrol==0.14.0 - # via firebase-admin -cachetools==5.3.3 - # via google-auth -cassandra-driver==3.29.1 certifi==2024.2.2 # via # httpcore # httpx - # kubernetes - # minio # requests - # snowflake-connector-python -cffi==1.16.0 - # via - # argon2-cffi-bindings - # cryptography - # snowflake-connector-python -cfgv==3.4.0 - # via pre-commit charset-normalizer==3.3.2 - # via - # requests - # snowflake-connector-python + # via requests click==8.1.7 # via + # feast (setup.py) # dask - # geomet - # great-expectations - # pip-tools # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations -comm==0.2.2 - # via - # ipykernel - # ipywidgets -coverage[toml]==7.5.1 - # via pytest-cov -cryptography==42.0.7 - # via - # azure-identity - # azure-storage-blob - # great-expectations - # moto - # msal - # pyjwt - # pyopenssl - # snowflake-connector-python - # types-pyopenssl - # types-redis + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask -db-dtypes==1.2.0 - # via google-cloud-bigquery -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -deltalake==0.17.3 dill==0.3.8 -distlib==0.3.8 - # via virtualenv + # via feast (setup.py) dnspython==2.6.1 # via email-validator -docker==7.0.0 - # via testcontainers -docutils==0.19 - # via sphinx -duckdb==0.10.2 - # via - # duckdb-engine - # ibis-framework -duckdb-engine==0.12.0 - # via ibis-framework email-validator==2.1.1 # via fastapi -entrypoints==0.4 - # via altair exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest -execnet==2.1.1 - # via pytest-xdist -executing==2.0.1 - # via stack-data + # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via - # snowflake-connector-python - # virtualenv -firebase-admin==5.4.0 -fqdn==1.5.1 - # via jsonschema fsspec==2023.12.2 # via dask -geojson==2.5.0 - # via rockset -geomet==0.2.1.post1 - # via cassandra-driver -google-api-core[grpc]==2.19.0 - # via - # firebase-admin - # google-api-python-client - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-core - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-api-python-client==2.128.0 - # via firebase-admin -google-auth==2.29.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 - # google-cloud-bigquery-storage - # google-cloud-core - # google-cloud-firestore - # google-cloud-storage - # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 -google-cloud-bigquery-storage==2.25.0 -google-cloud-bigtable==2.23.1 -google-cloud-core==2.4.1 - # via - # google-cloud-bigquery - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-cloud-datastore==2.19.0 -google-cloud-firestore==2.16.0 - # via firebase-admin -google-cloud-storage==2.16.0 - # via firebase-admin -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 - # via - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -great-expectations==0.18.13 -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 - # via google-cloud-bigtable -grpcio==1.63.0 - # via - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools -grpcio-health-checking==1.62.2 -grpcio-reflection==1.62.2 -grpcio-status==1.62.2 - # via google-api-core -grpcio-testing==1.62.2 -grpcio-tools==1.62.2 gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -happybase==1.2.0 -hazelcast-python-client==5.3.0 -hiredis==2.3.2 httpcore==1.0.5 # via httpx -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via - # fastapi - # jupyterlab -ibis-framework[duckdb]==8.0.0 - # via ibis-substrait -ibis-substrait==3.2.0 -identify==2.5.36 - # via pre-commit + # via fastapi idna==3.7 # via # anyio # email-validator # httpx - # jsonschema # requests - # snowflake-connector-python -imagesize==1.4.1 - # via sphinx importlib-metadata==7.1.0 - # via dask -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.4 - # via jupyterlab -ipython==8.24.0 # via - # great-expectations - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via great-expectations -isodate==0.6.1 - # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -jedi==0.19.1 - # via ipython + # dask + # typeguard jinja2==3.1.4 # via - # altair + # feast (setup.py) # fastapi - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # moto - # nbconvert - # sphinx -jmespath==1.0.1 - # via - # boto3 - # botocore -json5==0.9.25 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.22.0 - # via - # altair - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat +jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # jupyter-server - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat -jupyter-events==0.10.0 - # via jupyter-server -jupyter-lsp==2.2.5 - # via jupyterlab -jupyter-server==2.14.0 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.5.3 - # via jupyter-server -jupyterlab==4.1.8 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.27.1 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.10 - # via ipywidgets -kubernetes==20.13.0 locket==1.0.0 # via partd -makefun==1.15.2 - # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.21.2 - # via great-expectations -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython + # via jinja2 mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 -mistune==3.0.2 - # via - # great-expectations - # nbconvert mmh3==4.1.0 -mock==2.0.0 -moto==4.2.14 -msal==1.28.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.8 - # via cachecontrol -multipledispatch==1.0.0 - # via ibis-framework + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 -nbclient==0.10.0 - # via nbconvert -nbconvert==7.16.4 - # via jupyter-server -nbformat==5.10.4 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -notebook==7.1.3 - # via great-expectations -notebook-shim==0.2.4 - # via - # jupyterlab - # notebook + # via feast (setup.py) numpy==1.26.4 # via - # altair + # feast (setup.py) # dask - # db-dtypes - # great-expectations - # ibis-framework # pandas # pyarrow - # scipy -oauthlib==3.2.2 - # via requests-oauthlib orjson==3.10.3 # via fastapi -overrides==7.7.0 - # via jupyter-server packaging==24.0 # via - # build # dask - # db-dtypes - # docker - # duckdb-engine - # google-cloud-bigquery - # great-expectations # gunicorn - # ibis-substrait - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # msal-extensions - # nbconvert - # pytest - # snowflake-connector-python - # sphinx pandas==2.2.2 # via - # altair + # feast (setup.py) # dask # dask-expr - # db-dtypes - # google-cloud-bigquery - # great-expectations - # ibis-framework - # snowflake-connector-python -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.4 - # via jedi -parsy==2.1 - # via ibis-framework partd==1.4.2 # via dask -pbr==6.0.0 - # via mock -pexpect==4.9.0 - # via ipython -pip==24.0 - # via pip-tools -pip-tools==7.4.1 -platformdirs==3.11.0 - # via - # jupyter-core - # snowflake-connector-python - # virtualenv -pluggy==1.5.0 - # via pytest -ply==3.11 - # via thriftpy2 -portalocker==2.8.2 - # via msal-extensions -pre-commit==3.3.1 -prometheus-client==0.20.0 - # via jupyter-server -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore protobuf==4.25.3 # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools + # feast (setup.py) # mypy-protobuf - # proto-plus - # substrait -psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -py==1.11.0 -py-cpuinfo==9.0.0 - # via pytest-benchmark -py4j==0.10.9.7 - # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr - # db-dtypes - # deltalake - # google-cloud-bigquery - # ibis-framework - # snowflake-connector-python -pyarrow-hotfix==0.6 - # via - # deltalake - # ibis-framework -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pybindgen==0.22.1 -pycparser==2.22 - # via cffi pydantic==2.7.1 # via + # feast (setup.py) # fastapi - # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via - # ipython - # nbconvert + # feast (setup.py) # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 -pymysql==1.1.0 -pyodbc==5.1.0 -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 -pytest==7.4.4 - # via - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 -pytest-cov==5.0.0 -pytest-env==1.1.3 -pytest-lazy-fixture==0.6.3 -pytest-mock==1.10.4 -pytest-ordering==0.6 -pytest-timeout==1.4.2 -pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # google-cloud-bigquery - # great-expectations - # ibis-framework - # jupyter-client - # kubernetes - # moto - # pandas - # rockset - # trino + # via pandas python-dotenv==1.0.1 # via uvicorn -python-json-logger==2.0.7 - # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via - # great-expectations - # ibis-framework - # pandas - # snowflake-connector-python - # trino + # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask - # ibis-substrait - # jupyter-events - # kubernetes - # pre-commit - # responses # uvicorn -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client - # jupyter-server -redis==4.6.0 referencing==0.35.1 # via # jsonschema # jsonschema-specifications - # jupyter-events -regex==2024.4.28 requests==2.31.0 - # via - # azure-core - # cachecontrol - # docker - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # great-expectations - # jupyterlab-server - # kubernetes - # moto - # msal - # requests-oauthlib - # responses - # snowflake-connector-python - # sphinx - # trino -requests-oauthlib==2.0.0 - # via kubernetes -responses==0.25.0 - # via moto -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events + # via feast (setup.py) rich==13.7.1 - # via - # ibis-framework - # typer -rockset==2.1.2 + # via typer rpds-py==0.18.1 # via # jsonschema # referencing -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -ruff==0.4.3 -s3transfer==0.10.1 - # via boto3 -scipy==1.13.0 - # via great-expectations -send2trash==1.8.3 - # via jupyter-server -setuptools==69.5.1 - # via - # grpcio-tools - # kubernetes - # nodeenv - # pip-tools shellingham==1.5.4 # via typer six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # geomet - # happybase - # isodate - # kubernetes - # mock - # python-dateutil - # rfc3339-validator - # thriftpy2 + # via python-dateutil sniffio==1.3.1 # via # anyio # httpx -snowballstemmer==2.2.0 - # via sphinx -snowflake-connector-python[pandas]==3.10.0 -sortedcontainers==2.4.0 - # via snowflake-connector-python -soupsieve==2.5 - # via beautifulsoup4 -sphinx==6.2.1 -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx sqlalchemy[mypy]==2.0.30 - # via - # duckdb-engine - # ibis-framework - # sqlalchemy-views -sqlalchemy-views==0.3.2 - # via ibis-framework -sqlglot==20.11.0 - # via ibis-framework -stack-data==0.6.3 - # via ipython + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) starlette==0.37.2 # via fastapi -substrait==0.17.0 - # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 -terminado==0.18.1 - # via - # jupyter-server - # jupyter-server-terminals -testcontainers==4.4.0 -thriftpy2==0.5.0 - # via happybase -tinycss2==1.3.0 - # via nbconvert + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env -tomlkit==0.12.4 - # via snowflake-connector-python + # via mypy toolz==0.12.1 # via - # altair # dask - # ibis-framework # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado tqdm==4.66.4 - # via great-expectations -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -trino==0.328.0 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli -types-cffi==1.16.0.20240331 - # via types-pyopenssl types-protobuf==3.19.22 # via mypy-protobuf -types-pymysql==1.1.0.20240425 -types-pyopenssl==24.1.0.20240425 - # via types-redis -types-python-dateutil==2.9.0.20240316 - # via arrow -types-pytz==2024.1.0.20240417 -types-pyyaml==6.0.12.20240311 -types-redis==4.6.0.20240425 -types-requests==2.30.0.0 -types-setuptools==69.5.0.20240423 - # via types-cffi -types-tabulate==0.9.0.20240106 -types-urllib3==1.26.25.14 - # via types-requests typing-extensions==4.11.0 # via # anyio - # async-lru - # azure-core - # azure-storage-blob # fastapi - # great-expectations - # ibis-framework - # ipython # mypy # pydantic # pydantic-core - # snowflake-connector-python # sqlalchemy - # testcontainers + # starlette # typeguard # typer # uvicorn tzdata==2024.1 # via pandas -tzlocal==5.2 - # via - # great-expectations - # trino ujson==5.9.0 # via fastapi -uri-template==1.3.0 - # via jsonschema -uritemplate==4.1.1 - # via google-api-python-client urllib3==1.26.18 - # via - # botocore - # docker - # great-expectations - # kubernetes - # minio - # requests - # responses - # rockset - # testcontainers + # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 # via uvicorn -virtualenv==20.23.0 - # via pre-commit watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.13 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.8.0 - # via - # jupyter-server - # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.3 - # via moto -wheel==0.43.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -wrapt==1.16.0 - # via testcontainers -xmltodict==0.13.0 - # via moto zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 56a8259ab43..1344a58f5be 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -63,10 +69,15 @@ idna==3.7 # httpx # requests importlib-metadata==7.1.0 - # via dask + # via + # dask + # typeguard jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -78,13 +89,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -96,20 +110,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -120,6 +143,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -127,6 +151,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -142,11 +167,17 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -154,7 +185,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -167,6 +200,7 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy + # starlette # typeguard # typer # uvicorn @@ -178,6 +212,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 @@ -187,4 +222,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via importlib-metadata \ No newline at end of file + # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 3b76237f599..abf0cc1e484 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,914 +1,225 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -alabaster==0.7.16 - # via sphinx -altair==4.2.2 - # via great-expectations +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-ci-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx - # jupyter-server # starlette # watchfiles -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via isoduration -asn1crypto==1.5.1 - # via snowflake-connector-python -assertpy==1.1 -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -atpublic==4.1.0 - # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing -azure-core==1.30.1 - # via - # azure-identity - # azure-storage-blob -azure-identity==1.16.0 -azure-storage-blob==12.19.1 -babel==2.15.0 - # via - # jupyterlab-server - # sphinx -beautifulsoup4==4.12.3 - # via nbconvert -bidict==0.23.1 - # via ibis-framework -bleach==6.1.0 - # via nbconvert -boto3==1.34.99 - # via moto -botocore==1.34.99 - # via - # boto3 - # moto - # s3transfer -build==1.2.1 - # via pip-tools -cachecontrol==0.14.0 - # via firebase-admin -cachetools==5.3.3 - # via google-auth -cassandra-driver==3.29.1 certifi==2024.2.2 # via # httpcore # httpx - # kubernetes - # minio # requests - # snowflake-connector-python -cffi==1.16.0 - # via - # argon2-cffi-bindings - # cryptography - # snowflake-connector-python -cfgv==3.4.0 - # via pre-commit charset-normalizer==3.3.2 - # via - # requests - # snowflake-connector-python + # via requests click==8.1.7 # via + # feast (setup.py) # dask - # geomet - # great-expectations - # pip-tools # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations -comm==0.2.2 - # via - # ipykernel - # ipywidgets -coverage[toml]==7.5.1 - # via pytest-cov -cryptography==42.0.7 - # via - # azure-identity - # azure-storage-blob - # great-expectations - # moto - # msal - # pyjwt - # pyopenssl - # snowflake-connector-python - # types-pyopenssl - # types-redis + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask -db-dtypes==1.2.0 - # via google-cloud-bigquery -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -deltalake==0.17.3 dill==0.3.8 -distlib==0.3.8 - # via virtualenv + # via feast (setup.py) dnspython==2.6.1 # via email-validator -docker==7.0.0 - # via testcontainers -docutils==0.19 - # via sphinx -duckdb==0.10.2 - # via - # duckdb-engine - # ibis-framework -duckdb-engine==0.12.0 - # via ibis-framework email-validator==2.1.1 # via fastapi -entrypoints==0.4 - # via altair -execnet==2.1.1 - # via pytest-xdist -executing==2.0.1 - # via stack-data +exceptiongroup==1.2.1 + # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via - # snowflake-connector-python - # virtualenv -firebase-admin==5.4.0 -fqdn==1.5.1 - # via jsonschema fsspec==2023.12.2 # via dask -geojson==2.5.0 - # via rockset -geomet==0.2.1.post1 - # via cassandra-driver -google-api-core[grpc]==2.19.0 - # via - # firebase-admin - # google-api-python-client - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-core - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-api-python-client==2.128.0 - # via firebase-admin -google-auth==2.29.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 - # google-cloud-bigquery-storage - # google-cloud-core - # google-cloud-firestore - # google-cloud-storage - # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 -google-cloud-bigquery-storage==2.25.0 -google-cloud-bigtable==2.23.1 -google-cloud-core==2.4.1 - # via - # google-cloud-bigquery - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-cloud-datastore==2.19.0 -google-cloud-firestore==2.16.0 - # via firebase-admin -google-cloud-storage==2.16.0 - # via firebase-admin -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 - # via - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -great-expectations==0.18.13 -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 - # via google-cloud-bigtable -grpcio==1.63.0 - # via - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools -grpcio-health-checking==1.62.2 -grpcio-reflection==1.62.2 -grpcio-status==1.62.2 - # via google-api-core -grpcio-testing==1.62.2 -grpcio-tools==1.62.2 gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -happybase==1.2.0 -hazelcast-python-client==5.3.0 -hiredis==2.3.2 httpcore==1.0.5 # via httpx -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via - # fastapi - # jupyterlab -ibis-framework[duckdb]==8.0.0 - # via ibis-substrait -ibis-substrait==3.2.0 -identify==2.5.36 - # via pre-commit + # via fastapi idna==3.7 # via # anyio # email-validator # httpx - # jsonschema # requests - # snowflake-connector-python -imagesize==1.4.1 - # via sphinx importlib-metadata==7.1.0 - # via dask -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.4 - # via jupyterlab -ipython==8.24.0 # via - # great-expectations - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via great-expectations -isodate==0.6.1 - # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -jedi==0.19.1 - # via ipython + # dask + # typeguard jinja2==3.1.4 # via - # altair + # feast (setup.py) # fastapi - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # moto - # nbconvert - # sphinx -jmespath==1.0.1 - # via - # boto3 - # botocore -json5==0.9.25 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.22.0 - # via - # altair - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat +jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # jupyter-server - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat -jupyter-events==0.10.0 - # via jupyter-server -jupyter-lsp==2.2.5 - # via jupyterlab -jupyter-server==2.14.0 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.5.3 - # via jupyter-server -jupyterlab==4.1.8 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.27.1 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.10 - # via ipywidgets -kubernetes==20.13.0 locket==1.0.0 # via partd -makefun==1.15.2 - # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.21.2 - # via great-expectations -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython + # via jinja2 mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 -mistune==3.0.2 - # via - # great-expectations - # nbconvert mmh3==4.1.0 -mock==2.0.0 -moto==4.2.14 -msal==1.28.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.8 - # via cachecontrol -multipledispatch==1.0.0 - # via ibis-framework + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 -nbclient==0.10.0 - # via nbconvert -nbconvert==7.16.4 - # via jupyter-server -nbformat==5.10.4 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -notebook==7.1.3 - # via great-expectations -notebook-shim==0.2.4 - # via - # jupyterlab - # notebook + # via feast (setup.py) numpy==1.26.4 # via - # altair + # feast (setup.py) # dask - # db-dtypes - # great-expectations - # ibis-framework # pandas # pyarrow - # scipy -oauthlib==3.2.2 - # via requests-oauthlib orjson==3.10.3 # via fastapi -overrides==7.7.0 - # via jupyter-server packaging==24.0 # via - # build # dask - # db-dtypes - # docker - # duckdb-engine - # google-cloud-bigquery - # great-expectations # gunicorn - # ibis-substrait - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # msal-extensions - # nbconvert - # pytest - # snowflake-connector-python - # sphinx pandas==2.2.2 # via - # altair + # feast (setup.py) # dask # dask-expr - # db-dtypes - # google-cloud-bigquery - # great-expectations - # ibis-framework - # snowflake-connector-python -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.4 - # via jedi -parsy==2.1 - # via ibis-framework partd==1.4.2 # via dask -pbr==6.0.0 - # via mock -pexpect==4.9.0 - # via ipython -pip==24.0 - # via pip-tools -pip-tools==7.4.1 -platformdirs==3.11.0 - # via - # jupyter-core - # snowflake-connector-python - # virtualenv -pluggy==1.5.0 - # via pytest -ply==3.11 - # via thriftpy2 -portalocker==2.8.2 - # via msal-extensions -pre-commit==3.3.1 -prometheus-client==0.20.0 - # via jupyter-server -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore protobuf==4.25.3 # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools + # feast (setup.py) # mypy-protobuf - # proto-plus - # substrait -psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -py==1.11.0 -py-cpuinfo==9.0.0 - # via pytest-benchmark -py4j==0.10.9.7 - # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr - # db-dtypes - # deltalake - # google-cloud-bigquery - # ibis-framework - # snowflake-connector-python -pyarrow-hotfix==0.6 - # via - # deltalake - # ibis-framework -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pybindgen==0.22.1 -pycparser==2.22 - # via cffi pydantic==2.7.1 # via + # feast (setup.py) # fastapi - # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via - # ipython - # nbconvert + # feast (setup.py) # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 -pymysql==1.1.0 -pyodbc==5.1.0 -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 -pytest==7.4.4 - # via - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 -pytest-cov==5.0.0 -pytest-env==1.1.3 -pytest-lazy-fixture==0.6.3 -pytest-mock==1.10.4 -pytest-ordering==0.6 -pytest-timeout==1.4.2 -pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # google-cloud-bigquery - # great-expectations - # ibis-framework - # jupyter-client - # kubernetes - # moto - # pandas - # rockset - # trino + # via pandas python-dotenv==1.0.1 # via uvicorn -python-json-logger==2.0.7 - # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via - # great-expectations - # ibis-framework - # pandas - # snowflake-connector-python - # trino + # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask - # ibis-substrait - # jupyter-events - # kubernetes - # pre-commit - # responses # uvicorn -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client - # jupyter-server -redis==4.6.0 referencing==0.35.1 # via # jsonschema # jsonschema-specifications - # jupyter-events -regex==2024.4.28 requests==2.31.0 - # via - # azure-core - # cachecontrol - # docker - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # great-expectations - # jupyterlab-server - # kubernetes - # moto - # msal - # requests-oauthlib - # responses - # snowflake-connector-python - # sphinx - # trino -requests-oauthlib==2.0.0 - # via kubernetes -responses==0.25.0 - # via moto -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events + # via feast (setup.py) rich==13.7.1 - # via - # ibis-framework - # typer -rockset==2.1.2 + # via typer rpds-py==0.18.1 # via # jsonschema # referencing -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -ruff==0.4.3 -s3transfer==0.10.1 - # via boto3 -scipy==1.13.0 - # via great-expectations -send2trash==1.8.3 - # via jupyter-server -setuptools==69.5.1 - # via - # grpcio-tools - # kubernetes - # nodeenv - # pip-tools shellingham==1.5.4 # via typer six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # geomet - # happybase - # isodate - # kubernetes - # mock - # python-dateutil - # rfc3339-validator - # thriftpy2 + # via python-dateutil sniffio==1.3.1 # via # anyio # httpx -snowballstemmer==2.2.0 - # via sphinx -snowflake-connector-python[pandas]==3.10.0 -sortedcontainers==2.4.0 - # via snowflake-connector-python -soupsieve==2.5 - # via beautifulsoup4 -sphinx==6.2.1 -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx sqlalchemy[mypy]==2.0.30 - # via - # duckdb-engine - # ibis-framework - # sqlalchemy-views -sqlalchemy-views==0.3.2 - # via ibis-framework -sqlglot==20.11.0 - # via ibis-framework -stack-data==0.6.3 - # via ipython + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) starlette==0.37.2 # via fastapi -substrait==0.17.0 - # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 -terminado==0.18.1 - # via - # jupyter-server - # jupyter-server-terminals -testcontainers==4.4.0 -thriftpy2==0.5.0 - # via happybase -tinycss2==1.3.0 - # via nbconvert + # via feast (setup.py) toml==0.10.2 -tomlkit==0.12.4 - # via snowflake-connector-python + # via feast (setup.py) +tomli==2.0.1 + # via mypy toolz==0.12.1 # via - # altair # dask - # ibis-framework # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado tqdm==4.66.4 - # via great-expectations -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -trino==0.328.0 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli -types-cffi==1.16.0.20240331 - # via types-pyopenssl types-protobuf==3.19.22 # via mypy-protobuf -types-pymysql==1.1.0.20240425 -types-pyopenssl==24.1.0.20240425 - # via types-redis -types-python-dateutil==2.9.0.20240316 - # via arrow -types-pytz==2024.1.0.20240417 -types-pyyaml==6.0.12.20240311 -types-redis==4.6.0.20240425 -types-requests==2.30.0.0 -types-setuptools==69.5.0.20240423 - # via types-cffi -types-tabulate==0.9.0.20240106 -types-urllib3==1.26.25.14 - # via types-requests typing-extensions==4.11.0 # via - # azure-core - # azure-storage-blob + # anyio # fastapi - # great-expectations - # ibis-framework - # ipython # mypy # pydantic # pydantic-core - # snowflake-connector-python # sqlalchemy - # testcontainers + # starlette # typeguard # typer + # uvicorn tzdata==2024.1 # via pandas -tzlocal==5.2 - # via - # great-expectations - # trino ujson==5.9.0 # via fastapi -uri-template==1.3.0 - # via jsonschema -uritemplate==4.1.1 - # via google-api-python-client urllib3==1.26.18 - # via - # botocore - # docker - # great-expectations - # kubernetes - # minio - # requests - # responses - # rockset - # testcontainers + # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 # via uvicorn -virtualenv==20.23.0 - # via pre-commit watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.13 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.8.0 - # via - # jupyter-server - # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.3 - # via moto -wheel==0.43.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -wrapt==1.16.0 - # via testcontainers -xmltodict==0.13.0 - # via moto zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index c34b610d14c..01c27a435d1 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -20,30 +20,38 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi +exceptiongroup==1.2.1 + # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -61,10 +69,15 @@ idna==3.7 # httpx # requests importlib-metadata==7.1.0 - # via dask + # via + # dask + # typeguard jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -76,13 +89,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -94,20 +110,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -118,6 +143,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -125,6 +151,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -140,30 +167,43 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) +tomli==2.0.1 + # via mypy toolz==0.12.1 # via # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via + # anyio # fastapi # mypy # pydantic # pydantic-core # sqlalchemy + # starlette # typeguard # typer + # uvicorn tzdata==2024.1 # via pandas ujson==5.9.0 @@ -172,6 +212,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index a628f0823db..c5017378284 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,946 +1,225 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -alabaster==0.7.16 - # via sphinx -altair==4.2.2 - # via great-expectations +# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-ci-requirements.txt annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx - # jupyter-server # starlette # watchfiles -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via isoduration -asn1crypto==1.5.1 - # via snowflake-connector-python -assertpy==1.1 -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -async-timeout==4.0.3 - # via redis -atpublic==4.1.0 - # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing -azure-core==1.30.1 - # via - # azure-identity - # azure-storage-blob -azure-identity==1.16.0 -azure-storage-blob==12.19.1 -babel==2.15.0 - # via - # jupyterlab-server - # sphinx -beautifulsoup4==4.12.3 - # via nbconvert -bidict==0.23.1 - # via ibis-framework -bleach==6.1.0 - # via nbconvert -boto3==1.34.99 - # via moto -botocore==1.34.99 - # via - # boto3 - # moto - # s3transfer -build==1.2.1 - # via pip-tools -cachecontrol==0.14.0 - # via firebase-admin -cachetools==5.3.3 - # via google-auth -cassandra-driver==3.29.1 certifi==2024.2.2 # via # httpcore # httpx - # kubernetes - # minio # requests - # snowflake-connector-python -cffi==1.16.0 - # via - # argon2-cffi-bindings - # cryptography - # snowflake-connector-python -cfgv==3.4.0 - # via pre-commit charset-normalizer==3.3.2 - # via - # requests - # snowflake-connector-python + # via requests click==8.1.7 # via + # feast (setup.py) # dask - # geomet - # great-expectations - # pip-tools # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations -comm==0.2.2 - # via - # ipykernel - # ipywidgets -coverage[toml]==7.5.1 - # via pytest-cov -cryptography==42.0.7 - # via - # azure-identity - # azure-storage-blob - # great-expectations - # moto - # msal - # pyjwt - # pyopenssl - # snowflake-connector-python - # types-pyopenssl - # types-redis + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask -db-dtypes==1.2.0 - # via google-cloud-bigquery -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -deltalake==0.17.3 dill==0.3.8 -distlib==0.3.8 - # via virtualenv + # via feast (setup.py) dnspython==2.6.1 # via email-validator -docker==7.0.0 - # via testcontainers -docutils==0.19 - # via sphinx -duckdb==0.10.2 - # via - # duckdb-engine - # ibis-framework -duckdb-engine==0.12.0 - # via ibis-framework email-validator==2.1.1 # via fastapi -entrypoints==0.4 - # via altair exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest -execnet==2.1.1 - # via pytest-xdist -executing==2.0.1 - # via stack-data + # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via - # snowflake-connector-python - # virtualenv -firebase-admin==5.4.0 -fqdn==1.5.1 - # via jsonschema fsspec==2023.12.2 # via dask -geojson==2.5.0 - # via rockset -geomet==0.2.1.post1 - # via cassandra-driver -google-api-core[grpc]==2.19.0 - # via - # firebase-admin - # google-api-python-client - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-core - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-api-python-client==2.128.0 - # via firebase-admin -google-auth==2.29.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 - # google-cloud-bigquery-storage - # google-cloud-core - # google-cloud-firestore - # google-cloud-storage - # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 -google-cloud-bigquery-storage==2.25.0 -google-cloud-bigtable==2.23.1 -google-cloud-core==2.4.1 - # via - # google-cloud-bigquery - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-cloud-datastore==2.19.0 -google-cloud-firestore==2.16.0 - # via firebase-admin -google-cloud-storage==2.16.0 - # via firebase-admin -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos[grpc]==1.63.0 - # via - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -great-expectations==0.18.13 -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 - # via google-cloud-bigtable -grpcio==1.63.0 - # via - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools -grpcio-health-checking==1.62.2 -grpcio-reflection==1.62.2 -grpcio-status==1.62.2 - # via google-api-core -grpcio-testing==1.62.2 -grpcio-tools==1.62.2 gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -happybase==1.2.0 -hazelcast-python-client==5.3.0 -hiredis==2.3.2 httpcore==1.0.5 # via httpx -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via - # fastapi - # jupyterlab -ibis-framework[duckdb]==8.0.0 - # via ibis-substrait -ibis-substrait==3.2.0 -identify==2.5.36 - # via pre-commit + # via fastapi idna==3.7 # via # anyio # email-validator # httpx - # jsonschema # requests - # snowflake-connector-python -imagesize==1.4.1 - # via sphinx importlib-metadata==7.1.0 # via - # build # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx # typeguard -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.4 - # via jupyterlab -ipython==8.18.1 - # via - # great-expectations - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via great-expectations -isodate==0.6.1 - # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -jedi==0.19.1 - # via ipython jinja2==3.1.4 # via - # altair + # feast (setup.py) # fastapi - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # moto - # nbconvert - # sphinx -jmespath==1.0.1 - # via - # boto3 - # botocore -json5==0.9.25 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.22.0 - # via - # altair - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat +jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # jupyter-server - # nbclient -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat -jupyter-events==0.10.0 - # via jupyter-server -jupyter-lsp==2.2.5 - # via jupyterlab -jupyter-server==2.14.0 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.5.3 - # via jupyter-server -jupyterlab==4.1.8 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.27.1 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.10 - # via ipywidgets -kubernetes==20.13.0 locket==1.0.0 # via partd -makefun==1.15.2 - # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.21.2 - # via great-expectations -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython + # via jinja2 mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 -mistune==3.0.2 - # via - # great-expectations - # nbconvert mmh3==4.1.0 -mock==2.0.0 -moto==4.2.14 -msal==1.28.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.8 - # via cachecontrol -multipledispatch==1.0.0 - # via ibis-framework + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 -nbclient==0.10.0 - # via nbconvert -nbconvert==7.16.4 - # via jupyter-server -nbformat==5.10.4 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -notebook==7.1.3 - # via great-expectations -notebook-shim==0.2.4 - # via - # jupyterlab - # notebook + # via feast (setup.py) numpy==1.26.4 # via - # altair + # feast (setup.py) # dask - # db-dtypes - # great-expectations - # ibis-framework # pandas # pyarrow - # scipy -oauthlib==3.2.2 - # via requests-oauthlib orjson==3.10.3 # via fastapi -overrides==7.7.0 - # via jupyter-server packaging==24.0 # via - # build # dask - # db-dtypes - # docker - # duckdb-engine - # google-cloud-bigquery - # great-expectations # gunicorn - # ibis-substrait - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # msal-extensions - # nbconvert - # pytest - # snowflake-connector-python - # sphinx pandas==2.2.2 # via - # altair + # feast (setup.py) # dask # dask-expr - # db-dtypes - # google-cloud-bigquery - # great-expectations - # ibis-framework - # snowflake-connector-python -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.4 - # via jedi -parsy==2.1 - # via ibis-framework partd==1.4.2 # via dask -pbr==6.0.0 - # via mock -pexpect==4.9.0 - # via ipython -pip==24.0 - # via pip-tools -pip-tools==7.4.1 -platformdirs==3.11.0 - # via - # jupyter-core - # snowflake-connector-python - # virtualenv -pluggy==1.5.0 - # via pytest -ply==3.11 - # via thriftpy2 -portalocker==2.8.2 - # via msal-extensions -pre-commit==3.3.1 -prometheus-client==0.20.0 - # via jupyter-server -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore protobuf==4.25.3 # via - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools + # feast (setup.py) # mypy-protobuf - # proto-plus - # substrait -psutil==5.9.0 - # via ipykernel -psycopg2-binary==2.9.9 -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -py==1.11.0 -py-cpuinfo==9.0.0 - # via pytest-benchmark -py4j==0.10.9.7 - # via pyspark pyarrow==15.0.2 # via + # feast (setup.py) # dask-expr - # db-dtypes - # deltalake - # google-cloud-bigquery - # ibis-framework - # snowflake-connector-python -pyarrow-hotfix==0.6 - # via - # deltalake - # ibis-framework -pyasn1==0.6.0 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.0 - # via google-auth -pybindgen==0.22.1 -pycparser==2.22 - # via cffi pydantic==2.7.1 # via + # feast (setup.py) # fastapi - # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via - # ipython - # nbconvert + # feast (setup.py) # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 -pymysql==1.1.0 -pyodbc==5.1.0 -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 -pytest==7.4.4 - # via - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 -pytest-cov==5.0.0 -pytest-env==1.1.3 -pytest-lazy-fixture==0.6.3 -pytest-mock==1.10.4 -pytest-ordering==0.6 -pytest-timeout==1.4.2 -pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # google-cloud-bigquery - # great-expectations - # ibis-framework - # jupyter-client - # kubernetes - # moto - # pandas - # rockset - # trino + # via pandas python-dotenv==1.0.1 # via uvicorn -python-json-logger==2.0.7 - # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via - # great-expectations - # ibis-framework - # pandas - # snowflake-connector-python - # trino + # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask - # ibis-substrait - # jupyter-events - # kubernetes - # pre-commit - # responses # uvicorn -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client - # jupyter-server -redis==4.6.0 referencing==0.35.1 # via # jsonschema # jsonschema-specifications - # jupyter-events -regex==2024.4.28 requests==2.31.0 - # via - # azure-core - # cachecontrol - # docker - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # great-expectations - # jupyterlab-server - # kubernetes - # moto - # msal - # requests-oauthlib - # responses - # snowflake-connector-python - # sphinx - # trino -requests-oauthlib==2.0.0 - # via kubernetes -responses==0.25.0 - # via moto -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events + # via feast (setup.py) rich==13.7.1 - # via - # ibis-framework - # typer -rockset==2.1.2 + # via typer rpds-py==0.18.1 # via # jsonschema # referencing -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml -ruff==0.4.3 -s3transfer==0.10.1 - # via boto3 -scipy==1.13.0 - # via great-expectations -send2trash==1.8.3 - # via jupyter-server -setuptools==69.5.1 - # via - # grpcio-tools - # kubernetes - # nodeenv - # pip-tools shellingham==1.5.4 # via typer six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # geomet - # happybase - # isodate - # kubernetes - # mock - # python-dateutil - # rfc3339-validator - # thriftpy2 + # via python-dateutil sniffio==1.3.1 # via # anyio # httpx -snowballstemmer==2.2.0 - # via sphinx -snowflake-connector-python[pandas]==3.10.0 -sortedcontainers==2.4.0 - # via snowflake-connector-python -soupsieve==2.5 - # via beautifulsoup4 -sphinx==6.2.1 -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx sqlalchemy[mypy]==2.0.30 - # via - # duckdb-engine - # ibis-framework - # sqlalchemy-views -sqlalchemy-views==0.3.2 - # via ibis-framework -sqlglot==20.11.0 - # via ibis-framework -stack-data==0.6.3 - # via ipython + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) starlette==0.37.2 # via fastapi -substrait==0.17.0 - # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 -terminado==0.18.1 - # via - # jupyter-server - # jupyter-server-terminals -testcontainers==4.4.0 -thriftpy2==0.5.0 - # via happybase -tinycss2==1.3.0 - # via nbconvert + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env -tomlkit==0.12.4 - # via snowflake-connector-python + # via mypy toolz==0.12.1 # via - # altair # dask - # ibis-framework # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado tqdm==4.66.4 - # via great-expectations -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -trino==0.328.0 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli -types-cffi==1.16.0.20240331 - # via types-pyopenssl types-protobuf==3.19.22 # via mypy-protobuf -types-pymysql==1.1.0.20240425 -types-pyopenssl==24.1.0.20240425 - # via types-redis -types-python-dateutil==2.9.0.20240316 - # via arrow -types-pytz==2024.1.0.20240417 -types-pyyaml==6.0.12.20240311 -types-redis==4.6.0.20240425 -types-requests==2.30.0.0 -types-setuptools==69.5.0.20240423 - # via types-cffi -types-tabulate==0.9.0.20240106 -types-urllib3==1.26.25.14 - # via types-requests typing-extensions==4.11.0 # via # anyio - # async-lru - # azure-core - # azure-storage-blob # fastapi - # great-expectations - # ibis-framework - # ipython # mypy # pydantic # pydantic-core - # snowflake-connector-python # sqlalchemy # starlette - # testcontainers # typeguard # typer # uvicorn tzdata==2024.1 # via pandas -tzlocal==5.2 - # via - # great-expectations - # trino ujson==5.9.0 # via fastapi -uri-template==1.3.0 - # via jsonschema -uritemplate==4.1.1 - # via google-api-python-client urllib3==1.26.18 - # via - # botocore - # docker - # great-expectations - # kubernetes - # minio - # requests - # responses - # rockset - # snowflake-connector-python - # testcontainers + # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 # via uvicorn -virtualenv==20.23.0 - # via pre-commit watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.13 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.8.0 - # via - # jupyter-server - # kubernetes websockets==12.0 # via uvicorn -werkzeug==3.0.3 - # via moto -wheel==0.43.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -wrapt==1.16.0 - # via testcontainers -xmltodict==0.13.0 - # via moto zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 1092aac9d09..68801152d38 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -20,17 +20,22 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via + # feast (setup.py) # dask # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 + # via feast (setup.py) dask[dataframe]==2024.5.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.0 # via dask dill==0.3.8 + # via feast (setup.py) dnspython==2.6.1 # via email-validator email-validator==2.1.1 @@ -38,14 +43,15 @@ email-validator==2.1.1 exceptiongroup==1.2.1 # via anyio fastapi==0.111.0 - # via fastapi-cli + # via + # feast (setup.py) + # fastapi-cli fastapi-cli==0.0.2 # via fastapi fsspec==2024.3.1 # via dask -greenlet==3.0.3 - # via sqlalchemy gunicorn==22.0.0 + # via feast (setup.py) h11==0.14.0 # via # httpcore @@ -67,8 +73,11 @@ importlib-metadata==7.1.0 # dask # typeguard jinja2==3.1.4 - # via fastapi + # via + # feast (setup.py) + # fastapi jsonschema==4.22.0 + # via feast (setup.py) jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 @@ -80,13 +89,16 @@ markupsafe==2.1.5 mdurl==0.1.2 # via markdown-it-py mmh3==4.1.0 + # via feast (setup.py) mypy==1.10.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.6.0 + # via feast (setup.py) numpy==1.26.4 # via + # feast (setup.py) # dask # pandas # pyarrow @@ -98,20 +110,29 @@ packaging==24.0 # gunicorn pandas==2.2.2 # via + # feast (setup.py) # dask # dask-expr partd==1.4.2 # via dask protobuf==4.25.3 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf pyarrow==16.0.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr pydantic==2.7.1 - # via fastapi + # via + # feast (setup.py) + # fastapi pydantic-core==2.18.2 # via pydantic pygments==2.18.0 - # via rich + # via + # feast (setup.py) + # rich python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 @@ -122,6 +143,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via + # feast (setup.py) # dask # uvicorn referencing==0.35.1 @@ -129,6 +151,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications requests==2.31.0 + # via feast (setup.py) rich==13.7.1 # via typer rpds-py==0.18.1 @@ -144,11 +167,17 @@ sniffio==1.3.1 # anyio # httpx sqlalchemy[mypy]==2.0.30 + # via feast (setup.py) +sqlite-vss==0.1.2 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 + # via feast (setup.py) tenacity==8.3.0 + # via feast (setup.py) toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via mypy toolz==0.12.1 @@ -156,7 +185,9 @@ toolz==0.12.1 # dask # partd tqdm==4.66.4 + # via feast (setup.py) typeguard==4.2.1 + # via feast (setup.py) typer==0.12.3 # via fastapi-cli types-protobuf==5.26.0.20240422 @@ -181,6 +212,7 @@ urllib3==2.2.1 # via requests uvicorn[standard]==0.29.0 # via + # feast (setup.py) # fastapi # fastapi-cli uvloop==0.19.0 @@ -190,4 +222,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via importlib-metadata \ No newline at end of file + # via importlib-metadata From 995a9c3d481e6fc653e0f7622717ef540fa5c85f Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 19 May 2024 04:45:02 -0400 Subject: [PATCH 10/58] moving to sqlite-vec Signed-off-by: Francisco Javier Arceo --- sdk/python/requirements/py-ci-requirements.txt | 2 +- sdk/python/requirements/py-requirements.txt | 2 +- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/python/requirements/py-ci-requirements.txt b/sdk/python/requirements/py-ci-requirements.txt index f6c6987fd63..483fda2d644 100644 --- a/sdk/python/requirements/py-ci-requirements.txt +++ b/sdk/python/requirements/py-ci-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.29 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py-requirements.txt b/sdk/python/requirements/py-requirements.txt index 7edb05a30f2..5ae4f7fa3c8 100644 --- a/sdk/python/requirements/py-requirements.txt +++ b/sdk/python/requirements/py-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.29 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 913db69ed5e..f8813d5aef4 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 1344a58f5be..050c67f7cc4 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index abf0cc1e484..acbc643be9a 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 01c27a435d1..626a36b3e29 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index c5017378284..69177f5c94e 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 68801152d38..ca657582332 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vss==0.1.2 +sqlite-vec==0.0.1a9 # via feast (setup.py) starlette==0.37.2 # via fastapi From ceb7395eaf0ae5d135797805dc455c3f965e658e Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 19 May 2024 13:57:10 -0400 Subject: [PATCH 11/58] updating sqlite.py Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 53 +++++++++++++------ setup.py | 2 +- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 91d3291645b..ba30d196a8d 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -15,7 +15,8 @@ import os import json import sqlite3 -import sqlite_vss +import sqlite_vec +import struct from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple @@ -81,7 +82,7 @@ def _get_conn(self, config: RepoConfig): if config.online_store.vss_enabled: db = sqlite3.connect(":memory:") db.enable_load_extension(True) - sqlite_vss.load(db) + sqlite_vec.load(db) return self._conn @@ -110,8 +111,9 @@ def online_write_batch( for feature_name, val in values.items(): if config.online_store.vss_enabled: - print('using vector search') - vector_val = json.dumps(val) + vector_val = json.loads(val.string_val) # assuming val.string_val contains the JSON-serialized vector + vector_bin = serialize_f32(vector_val) + conn.execute( f""" UPDATE {_table_id(project, table)} @@ -120,8 +122,8 @@ def online_write_batch( """, ( # SET - str(val), - vector_val, + val.SerializeToString(), + vector_bin, timestamp, created_ts, # WHERE @@ -137,14 +139,13 @@ def online_write_batch( ( entity_key_bin, feature_name, - str(val), - vector_val, + val.SerializeToString(), + vector_bin, timestamp, created_ts, ), ) else: - print('not using vector search') conn.execute( f""" UPDATE {_table_id(project, table)} @@ -316,17 +317,34 @@ def retrieve_online_documents( # Convert the embedding to a string to be used in postgres vector search query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" + + # Convert the embedding to a binary format + query_embedding_bin = serialize_f32(embedding) + table_name = _table_id(project, table) + cur.execute( f""" - SELECT entity_key, feature_name, value, vector_value, vector_value <=> ? AS distance, event_ts - FROM {_table_id(project, table)} - WHERE feature_name = ? - ORDER BY distance - LIMIT ? + WITH similar_matches AS ( + SELECT rowid, vector_value <-> ? AS distance + FROM {table_name} + WHERE feature_name = ? + ORDER BY distance + LIMIT ? + ) + SELECT + sm.distance, + t.entity_key, + t.feature_name, + t.value, + t.vector_value, + t.event_ts + FROM similar_matches sm + LEFT JOIN {table_name} t ON t.rowid = sm.rowid; """, - (query_embedding_str, requested_feature, top_k), + (query_embedding_bin, requested_feature, top_k), ) + rows = cur.fetchall() result: List[ @@ -370,6 +388,11 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" +def serialize_f32(vector: List[float]) -> bytes: + """serializes a list of floats into a compact "raw bytes" format""" + return struct.pack("%sf" % len(vector), *vector) + + class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. diff --git a/setup.py b/setup.py index 236affeaa5c..4622b640640 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", - "sqlite-vss", + "sqlite-vec", "SQLAlchemy[mypy]>1", "tabulate>=0.8.0,<1", "tenacity>=7,<9", From 2cf4020094b3d544b1dd6c6b01249089e4b2fdd0 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 19 May 2024 23:10:57 -0400 Subject: [PATCH 12/58] checking in progress Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index ba30d196a8d..ec702965a3f 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import itertools -import os import json +import os import sqlite3 -import sqlite_vec import struct from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +import sqlite_vec from pydantic import StrictStr from feast import Entity @@ -111,8 +111,7 @@ def online_write_batch( for feature_name, val in values.items(): if config.online_store.vss_enabled: - vector_val = json.loads(val.string_val) # assuming val.string_val contains the JSON-serialized vector - vector_bin = serialize_f32(vector_val) + vector_bin = [serialize_f32(v) for v in val] # serializing the value conn.execute( f""" @@ -122,7 +121,7 @@ def online_write_batch( """, ( # SET - val.SerializeToString(), + str(val), vector_bin, timestamp, created_ts, @@ -139,7 +138,7 @@ def online_write_batch( ( entity_key_bin, feature_name, - val.SerializeToString(), + str(val), vector_bin, timestamp, created_ts, From 494e12fcb918ad23fe90f657ec2e5e707dd1a6b2 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 20 May 2024 15:03:16 -0400 Subject: [PATCH 13/58] updated test type Signed-off-by: Francisco Javier Arceo --- .../online_store/test_online_retrieval.py | 79 ++++++++++--------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index b31fc0ce9b7..ebefe7d92a6 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -10,7 +10,7 @@ from feast import FeatureStore, RepoConfig from feast.errors import FeatureViewNotFoundException from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto, FloatList from feast.repo_config import RegistryConfig from tests.utils.cli_repo_creator import CliRunner, get_example_repo @@ -440,7 +440,7 @@ def test_get_online_documents() -> None: item_key, { "Embeddings": [ - [ + FloatList(val=[ 0.17517076, -0.1259909, 0.01954236, @@ -451,43 +451,43 @@ def test_get_online_documents() -> None: 0.01173803, -0.0573408, 0.02616226, - ], - [ - 0.18517076, - -0.1259909, - 0.01954236, - 0.03045186, - -0.00074535, - -0.02715777, - -0.04582673, - 0.01173803, - -0.0573408, - 0.02616226, - ], - [ - 0.19517076, - -0.1259909, - 0.01954236, - 0.03045186, - -0.00074535, - -0.02715777, - -0.04582673, - 0.01173803, - -0.0573408, - 0.02616226, - ], - [ - 0.20517076, - -0.1259909, - 0.01954236, - 0.03045186, - -0.00074535, - -0.02715777, - -0.04582673, - 0.01173803, - -0.0573408, - 0.02616226, - ] + ]), + # FloatList(val=[ + # 0.18517076, + # -0.1259909, + # 0.01954236, + # 0.03045186, + # -0.00074535, + # -0.02715777, + # -0.04582673, + # 0.01173803, + # -0.0573408, + # 0.02616226, + # ]), + # FloatList(val=[ + # 0.19517076, + # -0.1259909, + # 0.01954236, + # 0.03045186, + # -0.00074535, + # -0.02715777, + # -0.04582673, + # 0.01173803, + # -0.0573408, + # 0.02616226, + # ]), + # FloatList(val=[ + # 0.20517076, + # -0.1259909, + # 0.01954236, + # 0.03045186, + # -0.00074535, + # -0.02715777, + # -0.04582673, + # 0.01173803, + # -0.0573408, + # 0.02616226, + # ]) ], }, datetime.utcnow(), @@ -522,3 +522,4 @@ def test_get_online_documents() -> None: assert "Embeddings" in result assert result["driver_id"] == [0] + print(result) \ No newline at end of file From 714b9c74f2bd979a64fa34162a240718e1ea67f4 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Tue, 21 May 2024 22:03:24 -0400 Subject: [PATCH 14/58] got the initialization working, nice Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 31 ++++++++++--------- .../online_store/test_online_retrieval.py | 6 ++++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index ec702965a3f..b2746979379 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -79,10 +79,8 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if config.online_store.vss_enabled: - db = sqlite3.connect(":memory:") - db.enable_load_extension(True) - sqlite_vec.load(db) + self._conn.enable_load_extension(True) + sqlite_vec.load(self._conn) return self._conn @@ -109,20 +107,21 @@ def online_write_batch( if created_ts is not None: created_ts = to_naive_utc(created_ts) + table_name = _table_id(project, table) for feature_name, val in values.items(): if config.online_store.vss_enabled: - vector_bin = [serialize_f32(v) for v in val] # serializing the value + vector_bin = [serialize_f32(v.val) for v in val] # serializing the value conn.execute( f""" - UPDATE {_table_id(project, table)} + UPDATE {table_name} SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? WHERE (entity_key = ? AND feature_name = ?) """, ( # SET - str(val), - vector_bin, + val[0].SerializeToString(), + vector_bin[0], timestamp, created_ts, # WHERE @@ -132,14 +131,14 @@ def online_write_batch( ) conn.execute( - f"""INSERT OR IGNORE INTO {_table_id(project, table)} + f"""INSERT OR IGNORE INTO {table_name} (entity_key, feature_name, value, vector_value, event_ts, created_ts) VALUES (?, ?, ?, ?, ?, ?)""", ( entity_key_bin, feature_name, - str(val), - vector_bin, + val[0].SerializeToString(), + vector_bin[0], timestamp, created_ts, ), @@ -147,7 +146,7 @@ def online_write_batch( else: conn.execute( f""" - UPDATE {_table_id(project, table)} + UPDATE {table_name} SET value = ?, event_ts = ?, created_ts = ? WHERE (entity_key = ? AND feature_name = ?) """, @@ -163,7 +162,7 @@ def online_write_batch( ) conn.execute( - f"""INSERT OR IGNORE INTO {_table_id(project, table)} + f"""INSERT OR IGNORE INTO {table_name} (entity_key, feature_name, value, event_ts, created_ts) VALUES (?, ?, ?, ?, ?)""", ( @@ -438,12 +437,16 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: ) def update(self): + self.conn.enable_load_extension(True) + sqlite_vec.load(self.conn) self.conn.execute( f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" ) - + self.conn.execute( + f"CREATE VIRTUAL TABLE {self.name}_embeddings USING vec0(embedding float[64])" + ) def teardown(self): self.conn.execute(f"DROP TABLE IF EXISTS {self.name}") diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index ebefe7d92a6..b3c09e519a2 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -427,6 +427,12 @@ def test_get_online_documents() -> None: get_example_repo("example_feature_repo_1.py"), "file" ) as store: store.config.online_store.vss_enabled = True + import sqlite3 + import sqlite_vec + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + sqlite_vec.load(db) + # Write some data to two tables document_embeddings_fv = store.get_feature_view(name="document_embeddings") From cd41bf01a7bc4465244a44e37d1ef29577e14f73 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 22 May 2024 07:01:50 -0400 Subject: [PATCH 15/58] checking in progress from last night Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 47 ++++++++++--------- .../online_store/test_online_retrieval.py | 6 --- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index b2746979379..80788ee9b97 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -322,26 +322,31 @@ def retrieve_online_documents( cur.execute( f""" - WITH similar_matches AS ( - SELECT rowid, vector_value <-> ? AS distance - FROM {table_name} - WHERE feature_name = ? - ORDER BY distance - LIMIT ? - ) - SELECT - sm.distance, - t.entity_key, - t.feature_name, - t.value, - t.vector_value, - t.event_ts - FROM similar_matches sm - LEFT JOIN {table_name} t ON t.rowid = sm.rowid; - """, - (query_embedding_bin, requested_feature, top_k), + create virtual table vec_example using vec0( + vector_value float[10] + ); + """) + + cur.execute( + f""" + INSERT INTO vec_example(vector_value) + VALUES (?) + """, + (query_embedding_bin) ) + cur.execute( + f""" + select + vector_value, + distance + from vec_example + where vector_value match ? + order by distance + limit ?; + """, + (query_embedding_bin, top_k) + ) rows = cur.fetchall() @@ -445,8 +450,8 @@ def update(self): self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" ) - self.conn.execute( - f"CREATE VIRTUAL TABLE {self.name}_embeddings USING vec0(embedding float[64])" - ) + # self.conn.execute( + # f"CREATE VIRTUAL TABLE {self.name}_embeddings USING vec0(embedding float[10])" + # ) def teardown(self): self.conn.execute(f"DROP TABLE IF EXISTS {self.name}") diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index b3c09e519a2..ebefe7d92a6 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -427,12 +427,6 @@ def test_get_online_documents() -> None: get_example_repo("example_feature_repo_1.py"), "file" ) as store: store.config.online_store.vss_enabled = True - import sqlite3 - import sqlite_vec - db = sqlite3.connect(":memory:") - db.enable_load_extension(True) - sqlite_vec.load(db) - # Write some data to two tables document_embeddings_fv = store.get_feature_view(name="document_embeddings") From ae40c81ce951d8015d281cd6e5b4a39a002fa31e Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 22 May 2024 22:21:17 -0400 Subject: [PATCH 16/58] removing unnecessary stuff Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 80788ee9b97..63a7ca53122 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -322,7 +322,7 @@ def retrieve_online_documents( cur.execute( f""" - create virtual table vec_example using vec0( + CREATE VIRTUAL TABLE vec_example using vec0( vector_value float[10] ); """) @@ -450,8 +450,6 @@ def update(self): self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" ) - # self.conn.execute( - # f"CREATE VIRTUAL TABLE {self.name}_embeddings USING vec0(embedding float[10])" - # ) + def teardown(self): self.conn.execute(f"DROP TABLE IF EXISTS {self.name}") From dd5b7e9c0c3f4dc55d209da3c3c144fc91c45a30 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 27 May 2024 06:51:17 -0400 Subject: [PATCH 17/58] fixing merge conflicts Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 6 ++-- .../example_repos/example_feature_repo_1.py | 3 +- .../online_store/test_online_retrieval.py | 36 +++++++------------ 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 63a7ca53122..bbfca24bc52 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -329,10 +329,10 @@ def retrieve_online_documents( cur.execute( f""" - INSERT INTO vec_example(vector_value) - VALUES (?) + INSERT INTO vec_example(rowid, vector_value) + VALUES (?, ?) """, - (query_embedding_bin) + (0, query_embedding_bin) ) cur.execute( diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index 20a8ad7bd86..795e6a28432 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -4,7 +4,7 @@ from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Array, Float32, Int64, String +from feast.types import Float32, Int64, String, Array # Note that file source paths are not validated, so there doesn't actually need to be any data # at the paths for these file sources. Since these paths are effectively fake, this example @@ -122,7 +122,6 @@ ttl=timedelta(hours=24), ) - @on_demand_feature_view( sources=[customer_profile], schema=[Field(name="on_demand_age", dtype=Int64)], diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index ebefe7d92a6..91dfebeec0d 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import numpy as np import pytest from pandas.testing import assert_frame_equal @@ -21,7 +22,7 @@ def test_get_online_features() -> None: """ runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write some data to two tables driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -278,7 +279,7 @@ def test_online_to_df(): runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write three tables to online store driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -435,7 +436,10 @@ def test_get_online_documents() -> None: item_key = EntityKeyProto( join_keys=["item_id"], entity_values=[ValueProto(int64_val=0)] ) - data = [ + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=[ ( item_key, { @@ -493,33 +497,17 @@ def test_get_online_documents() -> None: datetime.utcnow(), datetime.utcnow(), ) - ] - provider.online_write_batch( - config=store.config, - table=document_embeddings_fv, - data=data, + ], progress=None, ) - query_embedding = np.array( - [ - 0.17517076, - -0.1259909, - 0.01954236, - 0.03045186, - -0.00074535, - -0.02715777, - -0.04582673, - 0.01173803, - -0.0573408, - 0.02616226, - ] - ) + query = np.array([ 0.17517076, -0.1259909 , 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408 , 0.02616226]) # Retrieve two features using two keys, one valid one non-existing result = store.retrieve_online_documents( - feature="document_embeddings:Embeddings", query=query_embedding, top_k=3 + feature="document_embeddings:Embeddings", + query=query, + top_k=3 ).to_dict() assert "Embeddings" in result assert result["driver_id"] == [0] - print(result) \ No newline at end of file From 59e8b4077fd842d3c27e45265be9f648cdf1a55a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 27 May 2024 06:53:10 -0400 Subject: [PATCH 18/58] removing files changed accidentally] Signed-off-by: Francisco Javier Arceo --- .../requirements/py-ci-requirements.txt | 225 ------------------ sdk/python/requirements/py-requirements.txt | 225 ------------------ 2 files changed, 450 deletions(-) delete mode 100644 sdk/python/requirements/py-ci-requirements.txt delete mode 100644 sdk/python/requirements/py-requirements.txt diff --git a/sdk/python/requirements/py-ci-requirements.txt b/sdk/python/requirements/py-ci-requirements.txt deleted file mode 100644 index 483fda2d644..00000000000 --- a/sdk/python/requirements/py-ci-requirements.txt +++ /dev/null @@ -1,225 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py-ci-requirements.txt -annotated-types==0.6.0 - # via pydantic -anyio==4.3.0 - # via - # httpx - # starlette - # watchfiles -attrs==23.2.0 - # via - # jsonschema - # referencing -certifi==2024.2.2 - # via - # httpcore - # httpx - # requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # feast (setup.py) - # dask - # typer - # uvicorn -cloudpickle==3.0.0 - # via dask -colorama==0.4.6 - # via feast (setup.py) -dask[dataframe]==2024.5.0 - # via - # feast (setup.py) - # dask-expr -dask-expr==1.1.0 - # via dask -dill==0.3.8 - # via feast (setup.py) -dnspython==2.6.1 - # via email-validator -email-validator==2.1.1 - # via fastapi -exceptiongroup==1.2.1 - # via anyio -fastapi==0.111.0 - # via - # feast (setup.py) - # fastapi-cli -fastapi-cli==0.0.2 - # via fastapi -fsspec==2023.12.2 - # via dask -gunicorn==22.0.0 - # via feast (setup.py) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.5 - # via httpx -httptools==0.6.1 - # via uvicorn -httpx==0.27.0 - # via fastapi -idna==3.7 - # via - # anyio - # email-validator - # httpx - # requests -importlib-metadata==7.1.0 - # via - # dask - # typeguard -jinja2==3.1.3 - # via - # feast (setup.py) - # fastapi -jsonschema==4.22.0 - # via feast (setup.py) -jsonschema-specifications==2023.12.1 - # via jsonschema -locket==1.0.0 - # via partd -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via jinja2 -mdurl==0.1.2 - # via markdown-it-py -mmh3==4.1.0 - # via feast (setup.py) -mypy==1.10.0 - # via sqlalchemy -mypy-extensions==1.0.0 - # via mypy -mypy-protobuf==3.3.0 - # via feast (setup.py) -numpy==1.26.4 - # via - # feast (setup.py) - # dask - # pandas - # pyarrow -orjson==3.10.3 - # via fastapi -packaging==24.0 - # via - # dask - # gunicorn -pandas==2.2.2 - # via - # feast (setup.py) - # dask - # dask-expr -partd==1.4.1 - # via dask -protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf -pyarrow==16.0.0 - # via - # feast (setup.py) - # dask-expr -pydantic==2.7.1 - # via - # feast (setup.py) - # fastapi -pydantic-core==2.18.2 - # via pydantic -pygments==2.18.0 - # via - # feast (setup.py) - # rich -python-dateutil==2.9.0.post0 - # via pandas -python-dotenv==1.0.1 - # via uvicorn -python-multipart==0.0.9 - # via fastapi -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via - # feast (setup.py) - # dask - # uvicorn -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via feast (setup.py) -rich==13.7.1 - # via typer -rpds-py==0.18.0 - # via - # jsonschema - # referencing -shellingham==1.5.4 - # via typer -six==1.16.0 - # via python-dateutil -sniffio==1.3.1 - # via - # anyio - # httpx -sqlalchemy[mypy]==2.0.29 - # via feast (setup.py) -sqlite-vec==0.0.1a9 - # via feast (setup.py) -starlette==0.37.2 - # via fastapi -tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) -toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via mypy -toolz==0.12.1 - # via - # dask - # partd -tqdm==4.66.4 - # via feast (setup.py) -typeguard==4.2.1 - # via feast (setup.py) -typer==0.12.3 - # via fastapi-cli -types-protobuf==3.19.22 - # via mypy-protobuf -typing-extensions==4.11.0 - # via - # anyio - # fastapi - # mypy - # pydantic - # pydantic-core - # sqlalchemy - # starlette - # typeguard - # typer - # uvicorn -tzdata==2024.1 - # via pandas -ujson==5.9.0 - # via fastapi -urllib3==1.26.18 - # via requests -uvicorn[standard]==0.29.0 - # via - # feast (setup.py) - # fastapi - # fastapi-cli -uvloop==0.19.0 - # via uvicorn -watchfiles==0.21.0 - # via uvicorn -websockets==12.0 - # via uvicorn -zipp==3.18.1 - # via importlib-metadata diff --git a/sdk/python/requirements/py-requirements.txt b/sdk/python/requirements/py-requirements.txt deleted file mode 100644 index 5ae4f7fa3c8..00000000000 --- a/sdk/python/requirements/py-requirements.txt +++ /dev/null @@ -1,225 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py-requirements.txt -annotated-types==0.6.0 - # via pydantic -anyio==4.3.0 - # via - # httpx - # starlette - # watchfiles -attrs==23.2.0 - # via - # jsonschema - # referencing -certifi==2024.2.2 - # via - # httpcore - # httpx - # requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # feast (setup.py) - # dask - # typer - # uvicorn -cloudpickle==3.0.0 - # via dask -colorama==0.4.6 - # via feast (setup.py) -dask[dataframe]==2024.5.0 - # via - # feast (setup.py) - # dask-expr -dask-expr==1.1.0 - # via dask -dill==0.3.8 - # via feast (setup.py) -dnspython==2.6.1 - # via email-validator -email-validator==2.1.1 - # via fastapi -exceptiongroup==1.2.1 - # via anyio -fastapi==0.111.0 - # via - # feast (setup.py) - # fastapi-cli -fastapi-cli==0.0.2 - # via fastapi -fsspec==2024.3.1 - # via dask -gunicorn==22.0.0 - # via feast (setup.py) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.5 - # via httpx -httptools==0.6.1 - # via uvicorn -httpx==0.27.0 - # via fastapi -idna==3.7 - # via - # anyio - # email-validator - # httpx - # requests -importlib-metadata==7.1.0 - # via - # dask - # typeguard -jinja2==3.1.3 - # via - # feast (setup.py) - # fastapi -jsonschema==4.22.0 - # via feast (setup.py) -jsonschema-specifications==2023.12.1 - # via jsonschema -locket==1.0.0 - # via partd -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via jinja2 -mdurl==0.1.2 - # via markdown-it-py -mmh3==4.1.0 - # via feast (setup.py) -mypy==1.10.0 - # via sqlalchemy -mypy-extensions==1.0.0 - # via mypy -mypy-protobuf==3.6.0 - # via feast (setup.py) -numpy==1.26.4 - # via - # feast (setup.py) - # dask - # pandas - # pyarrow -orjson==3.10.3 - # via fastapi -packaging==24.0 - # via - # dask - # gunicorn -pandas==2.2.2 - # via - # feast (setup.py) - # dask - # dask-expr -partd==1.4.1 - # via dask -protobuf==4.25.3 - # via - # feast (setup.py) - # mypy-protobuf -pyarrow==16.0.0 - # via - # feast (setup.py) - # dask-expr -pydantic==2.7.1 - # via - # feast (setup.py) - # fastapi -pydantic-core==2.18.2 - # via pydantic -pygments==2.18.0 - # via - # feast (setup.py) - # rich -python-dateutil==2.9.0.post0 - # via pandas -python-dotenv==1.0.1 - # via uvicorn -python-multipart==0.0.9 - # via fastapi -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via - # feast (setup.py) - # dask - # uvicorn -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via feast (setup.py) -rich==13.7.1 - # via typer -rpds-py==0.18.0 - # via - # jsonschema - # referencing -shellingham==1.5.4 - # via typer -six==1.16.0 - # via python-dateutil -sniffio==1.3.1 - # via - # anyio - # httpx -sqlalchemy[mypy]==2.0.29 - # via feast (setup.py) -sqlite-vec==0.0.1a9 - # via feast (setup.py) -starlette==0.37.2 - # via fastapi -tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) -toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via mypy -toolz==0.12.1 - # via - # dask - # partd -tqdm==4.66.4 - # via feast (setup.py) -typeguard==4.2.1 - # via feast (setup.py) -typer==0.12.3 - # via fastapi-cli -types-protobuf==5.26.0.20240422 - # via mypy-protobuf -typing-extensions==4.11.0 - # via - # anyio - # fastapi - # mypy - # pydantic - # pydantic-core - # sqlalchemy - # starlette - # typeguard - # typer - # uvicorn -tzdata==2024.1 - # via pandas -ujson==5.9.0 - # via fastapi -urllib3==2.2.1 - # via requests -uvicorn[standard]==0.29.0 - # via - # feast (setup.py) - # fastapi - # fastapi-cli -uvloop==0.19.0 - # via uvicorn -watchfiles==0.21.0 - # via uvicorn -websockets==12.0 - # via uvicorn -zipp==3.18.1 - # via importlib-metadata From 642b68e8cf566dd0cdb52b6602c6242d4eed76fc Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 27 May 2024 23:18:33 -0400 Subject: [PATCH 19/58] uploading current progress...things run but need to update the virtual table insertion Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 41 +++++---- .../online_store/test_online_retrieval.py | 91 ++++++------------- 2 files changed, 51 insertions(+), 81 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index bbfca24bc52..b98903cf73a 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -18,7 +18,7 @@ import struct from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union import sqlite_vec from pydantic import StrictStr @@ -32,7 +32,7 @@ from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto, FloatList as FloatListProto from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.utils import to_naive_utc @@ -89,7 +89,7 @@ def online_write_batch( config: RepoConfig, table: FeatureView, data: List[ - Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + Tuple[EntityKeyProto, Dict[str, Union[FloatListProto, ValueProto]], datetime, Optional[datetime]] ], progress: Optional[Callable[[int], Any]], ) -> None: @@ -110,18 +110,17 @@ def online_write_batch( table_name = _table_id(project, table) for feature_name, val in values.items(): if config.online_store.vss_enabled: - vector_bin = [serialize_f32(v.val) for v in val] # serializing the value - + vector_bin = serialize_f32(val.val) conn.execute( f""" - UPDATE {table_name} - SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? - WHERE (entity_key = ? AND feature_name = ?) - """, + UPDATE {table_name} + SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, ( # SET - val[0].SerializeToString(), - vector_bin[0], + val.SerializeToString(), + vector_bin, timestamp, created_ts, # WHERE @@ -137,12 +136,13 @@ def online_write_batch( ( entity_key_bin, feature_name, - val[0].SerializeToString(), - vector_bin[0], + val.SerializeToString(), + vector_bin, timestamp, created_ts, ), ) + else: conn.execute( f""" @@ -316,12 +316,12 @@ def retrieve_online_documents( query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" - # Convert the embedding to a binary format + # Convert the embedding to a binary format instead of using SerializeToString() query_embedding_bin = serialize_f32(embedding) table_name = _table_id(project, table) cur.execute( - f""" + """ CREATE VIRTUAL TABLE vec_example using vec0( vector_value float[10] ); @@ -338,8 +338,11 @@ def retrieve_online_documents( cur.execute( f""" select + rowid as entity_key, + null as value, vector_value, - distance + distance, + datetime('now') event_ts from vec_example where vector_value match ? order by distance @@ -359,11 +362,11 @@ def retrieve_online_documents( ] ] = [] - for entity_key, feature_name, val_bin, vector_val, distance, event_ts in rows: + for entity_key, val_bin, vector_val, distance, event_ts in rows: feature_value_proto = ValueProto() - feature_value_proto.ParseFromString(val_bin) + # feature_value_proto.ParseFromString(val_bin) - vector_value_proto = ValueProto(string_val=vector_val) + vector_value_proto = FloatListProto(val=vector_val) distance_value_proto = ValueProto(float_val=distance) result.append( diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 91dfebeec0d..03c9bf36ee8 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -423,9 +423,10 @@ def test_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ + vector_length = 10 runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: store.config.online_store.vss_enabled = True # Write some data to two tables @@ -433,75 +434,41 @@ def test_get_online_documents() -> None: provider = store._get_provider() - item_key = EntityKeyProto( - join_keys=["item_id"], entity_values=[ValueProto(int64_val=0)] - ) - provider.online_write_batch( - config=store.config, - table=document_embeddings_fv, - data=[ + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + ) for i in range(10) + ] + data = [] + for item_key in item_keys: + data.append( ( item_key, - { - "Embeddings": [ - FloatList(val=[ - 0.17517076, - -0.1259909, - 0.01954236, - 0.03045186, - -0.00074535, - -0.02715777, - -0.04582673, - 0.01173803, - -0.0573408, - 0.02616226, - ]), - # FloatList(val=[ - # 0.18517076, - # -0.1259909, - # 0.01954236, - # 0.03045186, - # -0.00074535, - # -0.02715777, - # -0.04582673, - # 0.01173803, - # -0.0573408, - # 0.02616226, - # ]), - # FloatList(val=[ - # 0.19517076, - # -0.1259909, - # 0.01954236, - # 0.03045186, - # -0.00074535, - # -0.02715777, - # -0.04582673, - # 0.01173803, - # -0.0573408, - # 0.02616226, - # ]), - # FloatList(val=[ - # 0.20517076, - # -0.1259909, - # 0.01954236, - # 0.03045186, - # -0.00074535, - # -0.02715777, - # -0.04582673, - # 0.01173803, - # -0.0573408, - # 0.02616226, - # ]) - ], - }, + {"Embeddings": FloatList(val=np.random.random(vector_length, ))}, datetime.utcnow(), datetime.utcnow(), ) - ], + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, progress=None, ) - query = np.array([ 0.17517076, -0.1259909 , 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, -0.0573408 , 0.02616226]) + document_table = store._provider._online_store._conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' and name like '%_document_embeddings';" + ).fetchall() + assert len(document_table) == 1 + document_table_name = document_table[0][0] + record_count = len( + store._provider._online_store._conn.execute(f"select * from {document_table_name}").fetchall()) + assert record_count == len(data) + + query = np.array( + [0.17517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, + -0.0573408, 0.02616226]) # Retrieve two features using two keys, one valid one non-existing result = store.retrieve_online_documents( feature="document_embeddings:Embeddings", From 2a3dfd25739d38096143c8a1090b0ec082ec66a8 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 27 May 2024 23:35:24 -0400 Subject: [PATCH 20/58] linted Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 8 ++++---- sdk/python/tests/example_repos/example_feature_repo_1.py | 2 +- .../tests/unit/online_store/test_online_retrieval.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index b98903cf73a..4eb5d9a76f1 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import itertools -import json import os import sqlite3 import struct @@ -32,7 +31,8 @@ from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.protos.feast.types.Value_pb2 import Value as ValueProto, FloatList as FloatListProto +from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.utils import to_naive_utc @@ -328,7 +328,7 @@ def retrieve_online_documents( """) cur.execute( - f""" + """ INSERT INTO vec_example(rowid, vector_value) VALUES (?, ?) """, @@ -336,7 +336,7 @@ def retrieve_online_documents( ) cur.execute( - f""" + """ select rowid as entity_key, null as value, diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index 795e6a28432..a331cba62e1 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -4,7 +4,7 @@ from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Int64, String, Array +from feast.types import Array, Float32, Int64, String # Note that file source paths are not validated, so there doesn't actually need to be any data # at the paths for these file sources. Since these paths are effectively fake, this example diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 03c9bf36ee8..ef34f6fd714 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -4,14 +4,14 @@ import numpy as np import pandas as pd -import numpy as np import pytest from pandas.testing import assert_frame_equal from feast import FeatureStore, RepoConfig from feast.errors import FeatureViewNotFoundException from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.protos.feast.types.Value_pb2 import Value as ValueProto, FloatList +from feast.protos.feast.types.Value_pb2 import FloatList +from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig from tests.utils.cli_repo_creator import CliRunner, get_example_repo From 345526bdc35e835248610ee68ba6ac7068e4feb7 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Tue, 28 May 2024 06:15:41 -0400 Subject: [PATCH 21/58] adding working notes Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 4eb5d9a76f1..851b2b4c36d 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -335,6 +335,8 @@ def retrieve_online_documents( (0, query_embedding_bin) ) + # Have to join this with the {table_name} to get the feature name and entity_key + # Also the `top_k` doesn't appear to be working for some reason cur.execute( """ select From 4872d2dce083946d4d0aa9d855d0a275f3102414 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Tue, 28 May 2024 14:13:26 -0400 Subject: [PATCH 22/58] found a bug, original feature_store.py was only grabbing first feature view, adjusted Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/feature_store.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 343aa04d604..136a2d998e1 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1880,11 +1880,9 @@ def _retrieve_online_documents( raise ValueError( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) - ( - requested_feature_views, - _, - ) = self._get_feature_views_to_use( - features=[feature], allow_cache=True, hide_dummy_entity=False + + requested_feature_view = ( + feature.split(":")[0] if isinstance(feature, str) else feature ) requested_feature = ( feature.split(":")[1] if isinstance(feature, str) else feature @@ -1892,7 +1890,7 @@ def _retrieve_online_documents( provider = self._get_provider() document_features = self._retrieve_from_online_store( provider, - requested_feature_views[0], + requested_feature_view, requested_feature, query, top_k, From 5f8ff900ece4b8e911f36d812c3fc45297931116 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Tue, 28 May 2024 20:53:42 -0400 Subject: [PATCH 23/58] cant use a string have to verify it is a proper FeatureView object Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/feature_store.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 136a2d998e1..8114645b743 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1880,10 +1880,24 @@ def _retrieve_online_documents( raise ValueError( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) - + ( + requested_feature_views, + _, + ) = self._get_feature_views_to_use( + features=[feature], allow_cache=True, hide_dummy_entity=False + ) requested_feature_view = ( feature.split(":")[0] if isinstance(feature, str) else feature ) + feature_view_validated = False + for feature_view in requested_feature_views: + if feature_view.name == requested_feature_view: + feature_view_validated = True + requested_feature_view = feature_view + if not feature_view_validated: + raise ValueError( + f"Feature view {requested_feature_view} not found in the registry." + ) requested_feature = ( feature.split(":")[1] if isinstance(feature, str) else feature ) From 2a98513ba43a4c375930a5e23ec2e837e452a3a3 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 29 May 2024 22:57:38 -0400 Subject: [PATCH 24/58] updated got it working, need to fix some other stuff still Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/feature_store.py | 6 +++--- sdk/python/feast/infra/online_stores/sqlite.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 8114645b743..cddae08914c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1881,7 +1881,7 @@ def _retrieve_online_documents( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) ( - requested_feature_views, + available_feature_views, _, ) = self._get_feature_views_to_use( features=[feature], allow_cache=True, hide_dummy_entity=False @@ -1890,8 +1890,8 @@ def _retrieve_online_documents( feature.split(":")[0] if isinstance(feature, str) else feature ) feature_view_validated = False - for feature_view in requested_feature_views: - if feature_view.name == requested_feature_view: + for feature_view in available_feature_views: + if isinstance(requested_feature_view, str) and feature_view.name == requested_feature_view: feature_view_validated = True requested_feature_view = feature_view if not feature_view_validated: diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 851b2b4c36d..27794eef8ee 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -327,6 +327,11 @@ def retrieve_online_documents( ); """) + cur.execute( + f""" + INSERT INTO vec_example(rowid, vector_value) + select rowid, vector_value from {table_name} + """) cur.execute( """ INSERT INTO vec_example(rowid, vector_value) From 1db84468f0a4254d53ae15fcd58ce45537685ada Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 06:42:22 -0400 Subject: [PATCH 25/58] working Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 43 ++++++++++++------- .../online_store/test_online_retrieval.py | 3 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 27794eef8ee..09d11e076fc 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -327,6 +327,7 @@ def retrieve_online_documents( ); """) + # Currently I can only insert the embedding value without crashing SQLite, will report a bug cur.execute( f""" INSERT INTO vec_example(rowid, vector_value) @@ -343,17 +344,26 @@ def retrieve_online_documents( # Have to join this with the {table_name} to get the feature name and entity_key # Also the `top_k` doesn't appear to be working for some reason cur.execute( - """ - select - rowid as entity_key, - null as value, - vector_value, - distance, - datetime('now') event_ts - from vec_example - where vector_value match ? - order by distance - limit ?; + f""" + select + fv.entity_key, + f.rowid, + f.vector_value, + fv.value, + f.distance, + fv.event_ts + from ( + select + rowid, + vector_value, + distance + from vec_example + where vector_value match ? + order by distance + limit ? + ) f + left join {table_name} fv + on f.rowid = fv.rowid """, (query_embedding_bin, top_k) ) @@ -369,11 +379,11 @@ def retrieve_online_documents( ] ] = [] - for entity_key, val_bin, vector_val, distance, event_ts in rows: + for entity_key, _, vector_val, string_value, distance, event_ts in rows: + deserialized_val = deserialize_f32(vector_val) feature_value_proto = ValueProto() - # feature_value_proto.ParseFromString(val_bin) - - vector_value_proto = FloatListProto(val=vector_val) + feature_value_proto.ParseFromString(string_value if string_value else b'') + vector_value_proto = FloatListProto(val=deserialized_val) distance_value_proto = ValueProto(float_val=distance) result.append( @@ -405,6 +415,9 @@ def serialize_f32(vector: List[float]) -> bytes: """serializes a list of floats into a compact "raw bytes" format""" return struct.pack("%sf" % len(vector), *vector) +def deserialize_f32(byte_vector: bytes) -> List[float]: + """deserializes a list of floats from a compact "raw bytes" format""" + return list(struct.unpack(f"{len(byte_vector) // 4}f", byte_vector)) class SqliteTable(InfraObject): """ diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index ef34f6fd714..23d67648e2c 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -477,4 +477,5 @@ def test_get_online_documents() -> None: ).to_dict() assert "Embeddings" in result - assert result["driver_id"] == [0] + assert "distance" in result + assert len(result["distance"]) == 3 From d48ce6da1c1d62cd8a473fc6ccab4ea5f3b9ad48 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 06:44:42 -0400 Subject: [PATCH 26/58] linter Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/feature_store.py | 7 +++- .../feast/infra/online_stores/sqlite.py | 35 +++++++++------- .../example_repos/example_feature_repo_1.py | 1 + .../online_store/test_online_retrieval.py | 41 ++++++++++++++----- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index cddae08914c..f3f367c4187 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1891,13 +1891,16 @@ def _retrieve_online_documents( ) feature_view_validated = False for feature_view in available_feature_views: - if isinstance(requested_feature_view, str) and feature_view.name == requested_feature_view: + if ( + isinstance(requested_feature_view, str) + and feature_view.name == requested_feature_view + ): feature_view_validated = True requested_feature_view = feature_view if not feature_view_validated: raise ValueError( f"Feature view {requested_feature_view} not found in the registry." - ) + ) requested_feature = ( feature.split(":")[1] if isinstance(feature, str) else feature ) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 09d11e076fc..7016c6fb684 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -89,7 +89,12 @@ def online_write_batch( config: RepoConfig, table: FeatureView, data: List[ - Tuple[EntityKeyProto, Dict[str, Union[FloatListProto, ValueProto]], datetime, Optional[datetime]] + Tuple[ + EntityKeyProto, + Dict[str, Union[FloatListProto, ValueProto]], + datetime, + Optional[datetime], + ] ], progress: Optional[Callable[[int], Any]], ) -> None: @@ -312,10 +317,6 @@ def retrieve_online_documents( conn = self._get_conn(config) cur = conn.cursor() - # Convert the embedding to a string to be used in postgres vector search - query_embedding_str = f"[{','.join(str(el) for el in embedding)}]" - - # Convert the embedding to a binary format instead of using SerializeToString() query_embedding_bin = serialize_f32(embedding) table_name = _table_id(project, table) @@ -325,20 +326,22 @@ def retrieve_online_documents( CREATE VIRTUAL TABLE vec_example using vec0( vector_value float[10] ); - """) + """ + ) # Currently I can only insert the embedding value without crashing SQLite, will report a bug cur.execute( f""" - INSERT INTO vec_example(rowid, vector_value) + INSERT INTO vec_example(rowid, vector_value) select rowid, vector_value from {table_name} - """) + """ + ) cur.execute( """ - INSERT INTO vec_example(rowid, vector_value) + INSERT INTO vec_example(rowid, vector_value) VALUES (?, ?) """, - (0, query_embedding_bin) + (0, query_embedding_bin), ) # Have to join this with the {table_name} to get the feature name and entity_key @@ -349,7 +352,7 @@ def retrieve_online_documents( fv.entity_key, f.rowid, f.vector_value, - fv.value, + fv.value, f.distance, fv.event_ts from ( @@ -360,12 +363,12 @@ def retrieve_online_documents( from vec_example where vector_value match ? order by distance - limit ? + limit ? ) f left join {table_name} fv - on f.rowid = fv.rowid + on f.rowid = fv.rowid """, - (query_embedding_bin, top_k) + (query_embedding_bin, top_k), ) rows = cur.fetchall() @@ -382,7 +385,7 @@ def retrieve_online_documents( for entity_key, _, vector_val, string_value, distance, event_ts in rows: deserialized_val = deserialize_f32(vector_val) feature_value_proto = ValueProto() - feature_value_proto.ParseFromString(string_value if string_value else b'') + feature_value_proto.ParseFromString(string_value if string_value else b"") vector_value_proto = FloatListProto(val=deserialized_val) distance_value_proto = ValueProto(float_val=distance) @@ -415,10 +418,12 @@ def serialize_f32(vector: List[float]) -> bytes: """serializes a list of floats into a compact "raw bytes" format""" return struct.pack("%sf" % len(vector), *vector) + def deserialize_f32(byte_vector: bytes) -> List[float]: """deserializes a list of floats from a compact "raw bytes" format""" return list(struct.unpack(f"{len(byte_vector) // 4}f", byte_vector)) + class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index a331cba62e1..20a8ad7bd86 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -122,6 +122,7 @@ ttl=timedelta(hours=24), ) + @on_demand_feature_view( sources=[customer_profile], schema=[Field(name="on_demand_age", dtype=Int64)], diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 23d67648e2c..5292e080773 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -22,7 +22,7 @@ def test_get_online_features() -> None: """ runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write some data to two tables driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -279,7 +279,7 @@ def test_online_to_df(): runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: # Write three tables to online store driver_locations_fv = store.get_feature_view(name="driver_locations") @@ -426,7 +426,7 @@ def test_get_online_documents() -> None: vector_length = 10 runner = CliRunner() with runner.local_repo( - get_example_repo("example_feature_repo_1.py"), "file" + get_example_repo("example_feature_repo_1.py"), "file" ) as store: store.config.online_store.vss_enabled = True # Write some data to two tables @@ -437,14 +437,21 @@ def test_get_online_documents() -> None: item_keys = [ EntityKeyProto( join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] - ) for i in range(10) + ) + for i in range(10) ] data = [] for item_key in item_keys: data.append( ( item_key, - {"Embeddings": FloatList(val=np.random.random(vector_length, ))}, + { + "Embeddings": FloatList( + val=np.random.random( + vector_length, + ) + ) + }, datetime.utcnow(), datetime.utcnow(), ) @@ -463,17 +470,29 @@ def test_get_online_documents() -> None: assert len(document_table) == 1 document_table_name = document_table[0][0] record_count = len( - store._provider._online_store._conn.execute(f"select * from {document_table_name}").fetchall()) + store._provider._online_store._conn.execute( + f"select * from {document_table_name}" + ).fetchall() + ) assert record_count == len(data) query = np.array( - [0.17517076, -0.1259909, 0.01954236, 0.03045186, -0.00074535, -0.02715777, -0.04582673, 0.01173803, - -0.0573408, 0.02616226]) + [ + 0.17517076, + -0.1259909, + 0.01954236, + 0.03045186, + -0.00074535, + -0.02715777, + -0.04582673, + 0.01173803, + -0.0573408, + 0.02616226, + ] + ) # Retrieve two features using two keys, one valid one non-existing result = store.retrieve_online_documents( - feature="document_embeddings:Embeddings", - query=query, - top_k=3 + feature="document_embeddings:Embeddings", query=query, top_k=3 ).to_dict() assert "Embeddings" in result From 3fd1979fca63a1159ad2f26a263a110c45839407 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 06:51:32 -0400 Subject: [PATCH 27/58] fixing some type issues Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/feature_store.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f3f367c4187..f99247bff95 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1886,18 +1886,13 @@ def _retrieve_online_documents( ) = self._get_feature_views_to_use( features=[feature], allow_cache=True, hide_dummy_entity=False ) - requested_feature_view = ( + requested_feature_view_name = ( feature.split(":")[0] if isinstance(feature, str) else feature ) - feature_view_validated = False for feature_view in available_feature_views: - if ( - isinstance(requested_feature_view, str) - and feature_view.name == requested_feature_view - ): - feature_view_validated = True + if feature_view.name == requested_feature_view_name: requested_feature_view = feature_view - if not feature_view_validated: + if not requested_feature_view: raise ValueError( f"Feature view {requested_feature_view} not found in the registry." ) From 773e581405606b469f9f056ef11c4f64fe878810 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 11:29:32 -0400 Subject: [PATCH 28/58] fixed typing and lint issues Signed-off-by: Francisco Javier Arceo --- .../feast/infra/online_stores/sqlite.py | 42 ++++++++------ .../online_store/test_online_retrieval.py | 55 +++++++++++-------- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 7016c6fb684..bb25bc0edb9 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -17,7 +17,7 @@ import struct from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import sqlite_vec from pydantic import StrictStr @@ -48,9 +48,12 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ - vss_enabled: Optional[bool] = False + vec_enabled: Optional[bool] = False """ (optional) Enable or disable sqlite-vss for vector search""" + vector_len: Optional[int] = 512 + """ (optional) Length of the vector to be stored in the database""" + class SqliteOnlineStore(OnlineStore): """ @@ -79,7 +82,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - self._conn.enable_load_extension(True) + self._conn.enable_load_extension(True) # type: ignore sqlite_vec.load(self._conn) return self._conn @@ -91,7 +94,7 @@ def online_write_batch( data: List[ Tuple[ EntityKeyProto, - Dict[str, Union[FloatListProto, ValueProto]], + Dict[str, ValueProto], datetime, Optional[datetime], ] @@ -114,8 +117,10 @@ def online_write_batch( table_name = _table_id(project, table) for feature_name, val in values.items(): - if config.online_store.vss_enabled: - vector_bin = serialize_f32(val.val) + if config.online_store.vec_enabled: + vector_bin = serialize_f32( + val.float_list_val.val, config.online_store.vector_len + ) # type: ignore conn.execute( f""" UPDATE {table_name} @@ -311,20 +316,20 @@ def retrieve_online_documents( """ project = config.project - if not config.online_store.vss_enabled: + if not config.online_store.vec_enabled: raise ValueError("sqlite-vss is not enabled in the online store config") conn = self._get_conn(config) cur = conn.cursor() # Convert the embedding to a binary format instead of using SerializeToString() - query_embedding_bin = serialize_f32(embedding) + query_embedding_bin = serialize_f32(embedding, config.online_store.vector_len) table_name = _table_id(project, table) cur.execute( - """ + f""" CREATE VIRTUAL TABLE vec_example using vec0( - vector_value float[10] + vector_value float[{config.online_store.vector_len}] ); """ ) @@ -350,7 +355,6 @@ def retrieve_online_documents( f""" select fv.entity_key, - f.rowid, f.vector_value, fv.value, f.distance, @@ -382,11 +386,12 @@ def retrieve_online_documents( ] ] = [] - for entity_key, _, vector_val, string_value, distance, event_ts in rows: - deserialized_val = deserialize_f32(vector_val) + for entity_key, _, string_value, distance, event_ts in rows: feature_value_proto = ValueProto() feature_value_proto.ParseFromString(string_value if string_value else b"") - vector_value_proto = FloatListProto(val=deserialized_val) + vector_value_proto = ValueProto( + float_list_val=FloatListProto(val=embedding) + ) distance_value_proto = ValueProto(float_val=distance) result.append( @@ -414,14 +419,15 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" -def serialize_f32(vector: List[float]) -> bytes: +def serialize_f32(vector: List[float], vector_length: int) -> bytes: """serializes a list of floats into a compact "raw bytes" format""" - return struct.pack("%sf" % len(vector), *vector) + return struct.pack(f"{vector_length}f", *vector) -def deserialize_f32(byte_vector: bytes) -> List[float]: +def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: """deserializes a list of floats from a compact "raw bytes" format""" - return list(struct.unpack(f"{len(byte_vector) // 4}f", byte_vector)) + num_floats = vector_length // 4 # 4 bytes per float + return list(struct.unpack(f"{num_floats}f", byte_vector)) class SqliteTable(InfraObject): diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 5292e080773..ec66fe283c9 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -10,7 +10,7 @@ from feast import FeatureStore, RepoConfig from feast.errors import FeatureViewNotFoundException from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto -from feast.protos.feast.types.Value_pb2 import FloatList +from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig from tests.utils.cli_repo_creator import CliRunner, get_example_repo @@ -423,12 +423,14 @@ def test_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ - vector_length = 10 + n = 10 # number of samples - note: we'll actually double it + vector_length = 8 runner = CliRunner() with runner.local_repo( get_example_repo("example_feature_repo_1.py"), "file" ) as store: - store.config.online_store.vss_enabled = True + store.config.online_store.vec_enabled = True + store.config.online_store.vector_len = vector_length # Write some data to two tables document_embeddings_fv = store.get_feature_view(name="document_embeddings") @@ -438,7 +440,7 @@ def test_get_online_documents() -> None: EntityKeyProto( join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] ) - for i in range(10) + for i in range(n) ] data = [] for item_key in item_keys: @@ -446,9 +448,11 @@ def test_get_online_documents() -> None: ( item_key, { - "Embeddings": FloatList( - val=np.random.random( - vector_length, + "Embeddings": ValueProto( + float_list_val=FloatListProto( + val=np.random.random( + vector_length, + ) ) ) }, @@ -463,6 +467,23 @@ def test_get_online_documents() -> None: data=data, progress=None, ) + documents_df = pd.DataFrame( + { + "item_id": [str(i) for i in range(n)], + "Embeddings": [ + np.random.random( + vector_length, + ) + for i in range(n) + ], + "event_timestamp": [datetime.utcnow() for _ in range(n)], + } + ) + + store.write_to_online_store( + feature_view_name="document_embeddings", + df=documents_df, + ) document_table = store._provider._online_store._conn.execute( "SELECT name FROM sqlite_master WHERE type='table' and name like '%_document_embeddings';" @@ -474,23 +495,11 @@ def test_get_online_documents() -> None: f"select * from {document_table_name}" ).fetchall() ) - assert record_count == len(data) - - query = np.array( - [ - 0.17517076, - -0.1259909, - 0.01954236, - 0.03045186, - -0.00074535, - -0.02715777, - -0.04582673, - 0.01173803, - -0.0573408, - 0.02616226, - ] + assert record_count == len(data) + documents_df.shape[0] + + query = np.random.random( + vector_length, ) - # Retrieve two features using two keys, one valid one non-existing result = store.retrieve_online_documents( feature="document_embeddings:Embeddings", query=query, top_k=3 ).to_dict() From ccf5277b35492668ca14512b287f6ee13dccbaaa Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 11:47:14 -0400 Subject: [PATCH 29/58] updated dependencies Signed-off-by: Francisco Javier Arceo --- .../requirements/py3.10-ci-requirements.txt | 37 ------------------- .../requirements/py3.11-ci-requirements.txt | 37 ------------------- .../requirements/py3.9-ci-requirements.txt | 37 ------------------- 3 files changed, 111 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 5276492c329..f8813d5aef4 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -133,43 +133,6 @@ pygments==2.18.0 # via # feast (setup.py) # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 -pymysql==1.1.1 -pyodbc==5.1.0 -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 -pytest==7.4.4 - # via - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 -pytest-cov==5.0.0 -pytest-env==1.1.3 -pytest-lazy-fixture==0.6.3 -pytest-mock==1.10.4 -pytest-ordering==0.6 -pytest-timeout==1.4.2 -pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 39caeee7f87..acbc643be9a 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -133,43 +133,6 @@ pygments==2.18.0 # via # feast (setup.py) # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 -pymysql==1.1.1 -pyodbc==5.1.0 -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 -pytest==7.4.4 - # via - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 -pytest-cov==5.0.0 -pytest-env==1.1.3 -pytest-lazy-fixture==0.6.3 -pytest-mock==1.10.4 -pytest-ordering==0.6 -pytest-timeout==1.4.2 -pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index be32c5b0cb5..69177f5c94e 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -133,43 +133,6 @@ pygments==2.18.0 # via # feast (setup.py) # rich - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.3.0 -pymysql==1.1.1 -pyodbc==5.1.0 -pyopenssl==24.1.0 - # via snowflake-connector-python -pyparsing==3.1.2 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.1.0 - # via - # build - # pip-tools -pyspark==3.5.1 -pytest==7.4.4 - # via - # pytest-benchmark - # pytest-cov - # pytest-env - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 -pytest-cov==5.0.0 -pytest-env==1.1.3 -pytest-lazy-fixture==0.6.3 -pytest-mock==1.10.4 -pytest-ordering==0.6 -pytest-timeout==1.4.2 -pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 # via pandas python-dotenv==1.0.1 From 4b55f38b7ebcc5c8ec6a6c6f9d885407bd3c7a39 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 14:57:35 -0400 Subject: [PATCH 30/58] fix for pixi and updating requirements Signed-off-by: Francisco Javier Arceo --- infra/scripts/pixi/pixi.lock | 306 +++++- infra/scripts/pixi/pixi.toml | 2 +- .../requirements/py3.10-ci-requirements.txt | 872 ++++++++++++++++- .../requirements/py3.10-requirements.txt | 5 +- .../requirements/py3.11-ci-requirements.txt | 861 ++++++++++++++++- .../requirements/py3.11-requirements.txt | 11 +- .../requirements/py3.9-ci-requirements.txt | 879 +++++++++++++++++- 7 files changed, 2853 insertions(+), 83 deletions(-) diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock index 19a32f32ae8..f1ce2d26585 100644 --- a/infra/scripts/pixi/pixi.lock +++ b/infra/scripts/pixi/pixi.lock @@ -1,4 +1,4 @@ -version: 4 +version: 5 environments: default: channels: @@ -11,6 +11,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda py310: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -38,6 +41,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py311: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -66,6 +84,22 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py39: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -93,6 +127,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 packages: - kind: conda name: _libgcc_mutex @@ -123,6 +172,19 @@ packages: license_family: BSD size: 23621 timestamp: 1650670423406 +- kind: conda + name: bzip2 + version: 1.0.8 + build: h93a5062_5 + build_number: 5 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + sha256: bfa84296a638bea78a8bb29abc493ee95f2a0218775642474a840411b950fe5f + md5: 1bbc659ca658bfd49a481b5ef7a0f40f + license: bzip2-1.0.6 + license_family: BSD + size: 122325 + timestamp: 1699280294368 - kind: conda name: bzip2 version: 1.0.8 @@ -149,6 +211,17 @@ packages: license: ISC size: 155432 timestamp: 1706843687645 +- kind: conda + name: ca-certificates + version: 2024.2.2 + build: hf0a4a13_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + sha256: 49bc3439816ac72d0c0e0f144b8cc870fdcc4adec2e861407ec818d8116b2204 + md5: fb416a1795f18dcc5a038bc2dc54edf9 + license: ISC + size: 155725 + timestamp: 1706844034242 - kind: conda name: ld_impl_linux-64 version: '2.40' @@ -177,6 +250,20 @@ packages: license_family: GPL size: 713322 timestamp: 1713651222435 +- kind: conda + name: libcxx + version: 17.0.6 + build: h5f092b4_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + sha256: 119d3d9306f537d4c89dc99ed99b94c396d262f0b06f7833243646f68884f2c2 + md5: a96fd5dda8ce56c86a971e0fa02751d0 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + size: 1248885 + timestamp: 1715020154867 - kind: conda name: libexpat version: 2.6.2 @@ -193,6 +280,33 @@ packages: license_family: MIT size: 73730 timestamp: 1710362120304 +- kind: conda + name: libexpat + version: 2.6.2 + build: hebf3989_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda + sha256: ba7173ac30064ea901a4c9fb5a51846dcc25512ceb565759be7d18cbf3e5415e + md5: e3cde7cfa87f82f7cb13d482d5e0ad09 + constrains: + - expat 2.6.2.* + license: MIT + license_family: MIT + size: 63655 + timestamp: 1710362424980 +- kind: conda + name: libffi + version: 3.4.2 + build: h3422bc3_5 + build_number: 5 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + sha256: 41b3d13efb775e340e4dba549ab5c029611ea6918703096b2eaa9c015c0750ca + md5: 086914b672be056eb70fd4285b6783b6 + license: MIT + license_family: MIT + size: 39020 + timestamp: 1636488587153 - kind: conda name: libffi version: 3.4.2 @@ -288,6 +402,19 @@ packages: license_family: GPL size: 33408 timestamp: 1697359010159 +- kind: conda + name: libsqlite + version: 3.45.3 + build: h091b4b1_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + sha256: 4337f466eb55bbdc74e168b52ec8c38f598e3664244ec7a2536009036e2066cc + md5: c8c1186c7f3351f6ffddb97b1f54fc58 + depends: + - libzlib >=1.2.13,<2.0.0a0 + license: Unlicense + size: 824794 + timestamp: 1713367748819 - kind: conda name: libsqlite version: 3.45.3 @@ -360,6 +487,23 @@ packages: license_family: Other size: 61588 timestamp: 1686575217516 +- kind: conda + name: libzlib + version: 1.2.13 + build: hfb2fe0b_6 + build_number: 6 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + sha256: 8b29a2386d99b8f58178951dcf19117b532cd9c4aa07623bf1667eae99755d32 + md5: 9c4e121cd926cab631bd1c4a61d18b17 + depends: + - __osx >=11.0 + constrains: + - zlib 1.2.13 *_6 + license: Zlib + license_family: Other + size: 46768 + timestamp: 1716874151980 - kind: conda name: ncurses version: 6.4.20240210 @@ -373,6 +517,17 @@ packages: license: X11 AND BSD-3-Clause size: 895669 timestamp: 1710866638986 +- kind: conda + name: ncurses + version: '6.5' + build: hb89a1cb_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + sha256: 87d7cf716d9d930dab682cb57b3b8d3a61940b47d6703f3529a155c938a6990a + md5: b13ad5724ac9ae98b6b4fd87e4500ba4 + license: X11 AND BSD-3-Clause + size: 795131 + timestamp: 1715194898402 - kind: conda name: openssl version: 3.2.1 @@ -408,6 +563,24 @@ packages: license_family: Apache size: 2895187 timestamp: 1714466138265 +- kind: conda + name: openssl + version: 3.3.0 + build: hfb2fe0b_3 + build_number: 3 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + sha256: 6f41c163ab57e7499dff092be4498614651f0f6432e12c2b9f06859a8bc39b75 + md5: 730f618b008b3c13c1e3f973408ddd67 + depends: + - __osx >=11.0 + - ca-certificates + constrains: + - pyopenssl >=22.1 + license: Apache-2.0 + license_family: Apache + size: 2893954 + timestamp: 1716468329572 - kind: conda name: python version: 3.9.19 @@ -437,6 +610,54 @@ packages: license: Python-2.0 size: 23800555 timestamp: 1710940120866 +- kind: conda + name: python + version: 3.9.19 + build: hd7ebdb9_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + sha256: 3b93f7a405f334043758dfa8aaca050429a954a37721a6462ebd20e94ef7c5a0 + md5: 45c4d173b12154f746be3b49b1190634 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.9.* *_cp39 + license: Python-2.0 + size: 11847835 + timestamp: 1710939779164 +- kind: conda + name: python + version: 3.10.14 + build: h2469fbe_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + sha256: 454d609fe25daedce9e886efcbfcadad103ed0362e7cb6d2bcddec90b1ecd3ee + md5: 4ae999c8227c6d8c7623d32d51d25ea9 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.10.* *_cp310 + license: Python-2.0 + size: 12336005 + timestamp: 1710939659384 - kind: conda name: python version: 3.10.14 @@ -466,6 +687,32 @@ packages: license: Python-2.0 size: 25517742 timestamp: 1710939725109 +- kind: conda + name: python + version: 3.11.9 + build: h932a869_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + sha256: a436ceabde1f056a0ac3e347dadc780ee2a135a421ddb6e9a469370769829e3c + md5: 293e0713ae804b5527a673e7605c04fc + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.6.2,<3.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 14644189 + timestamp: 1713552154779 - kind: conda name: python version: 3.11.9 @@ -512,6 +759,36 @@ packages: license_family: GPL size: 281456 timestamp: 1679532220005 +- kind: conda + name: readline + version: '8.2' + build: h92ec313_1 + build_number: 1 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + sha256: a1dfa679ac3f6007362386576a704ad2d0d7a02e98f5d0b115f207a2da63e884 + md5: 8cbb776a2f641b943d413b3e19df71f4 + depends: + - ncurses >=6.3,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 250351 + timestamp: 1679532511311 +- kind: conda + name: tk + version: 8.6.13 + build: h5083fa2_1 + build_number: 1 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + sha256: 72457ad031b4c048e5891f3f6cb27a53cb479db68a52d965f796910e71a403a8 + md5: b50a57ba89c32b62428b71a875291c9b + depends: + - libzlib >=1.2.13,<1.3.0a0 + license: TCL + license_family: BSD + size: 3145523 + timestamp: 1699202432999 - kind: conda name: tk version: 8.6.13 @@ -554,6 +831,22 @@ packages: license: Apache-2.0 OR MIT size: 11891252 timestamp: 1714233659570 +- kind: conda + name: uv + version: 0.1.45 + build: hc069d6b_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + sha256: 80dfc19f2ef473e86e718361847d1d598e95ffd0c0f5de7d07cda35d25f6aef5 + md5: 9192238a60bc6da9c41092990c31eb41 + depends: + - __osx >=11.0 + - libcxx >=16 + constrains: + - __osx >=11.0 + license: Apache-2.0 OR MIT + size: 9231858 + timestamp: 1716265232676 - kind: conda name: xz version: 5.2.6 @@ -567,3 +860,14 @@ packages: license: LGPL-2.1 and GPL-2.0 size: 418368 timestamp: 1660346797927 +- kind: conda + name: xz + version: 5.2.6 + build: h57fd34a_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 + sha256: 59d78af0c3e071021cfe82dc40134c19dab8cdf804324b62940f5c8cd71803ec + md5: 39c6b54e94014701dd157f4f576ed211 + license: LGPL-2.1 and GPL-2.0 + size: 235693 + timestamp: 1660346961024 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index f0d360fff3d..10179339f70 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -1,7 +1,7 @@ [project] name = "pixi-feast" channels = ["conda-forge"] -platforms = ["linux-64"] +platforms = ["linux-64", "osx-arm64"] [tasks] diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index f8813d5aef4..bda20c74216 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,215 +1,1017 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt +alabaster==0.7.16 + # via sphinx +altair==4.2.2 + # via great-expectations annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx + # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asn1crypto==1.5.1 + # via snowflake-connector-python +assertpy==1.1 + # via feast (setup.py) +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 + # via redis +atpublic==4.1.0 + # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing +azure-core==1.30.1 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.16.0 + # via feast (setup.py) +azure-storage-blob==12.20.0 + # via feast (setup.py) +babel==2.15.0 + # via + # jupyterlab-server + # sphinx +beautifulsoup4==4.12.3 + # via nbconvert +bidict==0.23.1 + # via ibis-framework +bleach==6.1.0 + # via nbconvert +boto3==1.34.116 + # via + # feast (setup.py) + # moto +botocore==1.34.116 + # via + # boto3 + # moto + # s3transfer +build==1.2.1 + # via + # feast (setup.py) + # pip-tools +cachecontrol==0.14.0 + # via firebase-admin +cachetools==5.3.3 + # via google-auth +cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.2.2 # via + # elastic-transport # httpcore # httpx + # kubernetes + # minio # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # argon2-cffi-bindings + # cryptography + # snowflake-connector-python +cfgv==3.4.0 + # via pre-commit charset-normalizer==3.3.2 - # via requests + # via + # requests + # snowflake-connector-python click==8.1.7 # via # feast (setup.py) # dask + # geomet + # great-expectations + # pip-tools # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) + # via + # feast (setup.py) + # great-expectations +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.5.3 + # via pytest-cov +cryptography==42.0.7 + # via + # feast (setup.py) + # azure-identity + # azure-storage-blob + # great-expectations + # moto + # msal + # pyjwt + # pyopenssl + # snowflake-connector-python + # types-pyopenssl + # types-redis dask[dataframe]==2024.5.0 # via # feast (setup.py) # dask-expr dask-expr==1.1.0 # via dask +db-dtypes==1.2.0 + # via google-cloud-bigquery +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +deltalake==0.17.4 + # via feast (setup.py) dill==0.3.8 # via feast (setup.py) +distlib==0.3.8 + # via virtualenv dnspython==2.6.1 # via email-validator +docker==7.1.0 + # via + # feast (setup.py) + # testcontainers +docutils==0.19 + # via sphinx +duckdb==0.10.3 + # via + # duckdb-engine + # ibis-framework +duckdb-engine==0.12.1 + # via ibis-framework +elastic-transport==8.13.1 + # via elasticsearch +elasticsearch==8.13.2 + # via feast (setup.py) email-validator==2.1.1 # via fastapi +entrypoints==0.4 + # via altair exceptiongroup==1.2.1 - # via anyio + # via + # anyio + # ipython + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.0.1 + # via stack-data fastapi==0.111.0 # via # feast (setup.py) # fastapi-cli fastapi-cli==0.0.2 # via fastapi +fastjsonschema==2.19.1 + # via nbformat +filelock==3.14.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 + # via feast (setup.py) +fqdn==1.5.1 + # via jsonschema fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask +geojson==2.5.0 + # via rockset +geomet==0.2.1.post1 + # via cassandra-driver +google-api-core[grpc]==2.19.0 + # via + # feast (setup.py) + # firebase-admin + # google-api-python-client + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-api-python-client==2.131.0 + # via firebase-admin +google-auth==2.29.0 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-bigquery-storage + # google-cloud-core + # google-cloud-firestore + # google-cloud-storage + # kubernetes +google-auth-httplib2==0.2.0 + # via google-api-python-client +google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-core==2.4.1 + # via + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-cloud-datastore==2.19.0 + # via feast (setup.py) +google-cloud-firestore==2.16.0 + # via firebase-admin +google-cloud-storage==2.16.0 + # via + # feast (setup.py) + # firebase-admin +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.63.0 + # via + # feast (setup.py) + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +great-expectations==0.18.15 + # via feast (setup.py) +grpc-google-iam-v1==0.13.0 + # via google-cloud-bigtable +grpcio==1.64.0 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools +grpcio-health-checking==1.62.2 + # via feast (setup.py) +grpcio-reflection==1.62.2 + # via feast (setup.py) +grpcio-status==1.62.2 + # via google-api-core +grpcio-testing==1.62.2 + # via feast (setup.py) +grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn +happybase==1.2.0 + # via feast (setup.py) +hazelcast-python-client==5.4.0 + # via feast (setup.py) +hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx +httplib2==0.22.0 + # via + # google-api-python-client + # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via fastapi + # via + # feast (setup.py) + # fastapi + # jupyterlab +ibis-framework[duckdb]==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.36 + # via pre-commit idna==3.7 # via # anyio # email-validator # httpx + # jsonschema # requests + # snowflake-connector-python +imagesize==1.4.1 + # via sphinx importlib-metadata==7.1.0 + # via dask +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.4 + # via jupyterlab +ipython==8.25.0 # via - # dask - # typeguard + # great-expectations + # ipykernel + # ipywidgets +ipywidgets==8.1.3 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython jinja2==3.1.4 # via # feast (setup.py) + # altair # fastapi -jsonschema==4.22.0 - # via feast (setup.py) + # great-expectations + # jupyter-server + # jupyterlab + # jupyterlab-server + # moto + # nbconvert + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +json5==0.9.25 + # via jupyterlab-server +jsonpatch==1.33 + # via great-expectations +jsonpointer==2.4 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.22.0 + # via + # feast (setup.py) + # altair + # great-expectations + # jupyter-events + # jupyterlab-server + # nbformat jsonschema-specifications==2023.12.1 # via jsonschema +jupyter-client==8.6.2 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.1 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.2.1 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.2 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.11 + # via ipywidgets +kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd +makefun==1.15.2 + # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via jinja2 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.21.2 + # via great-expectations +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython mdurl==0.1.2 # via markdown-it-py +minio==7.1.0 + # via feast (setup.py) +mistune==3.0.2 + # via + # great-expectations + # nbconvert mmh3==4.1.0 # via feast (setup.py) +mock==2.0.0 + # via feast (setup.py) +moto==4.2.14 + # via feast (setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +msgpack==1.0.8 + # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework mypy==1.10.0 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # great-expectations + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.9.0 + # via pre-commit +notebook==7.2.0 + # via great-expectations +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook numpy==1.26.4 # via # feast (setup.py) + # altair # dask + # db-dtypes + # great-expectations + # ibis-framework # pandas # pyarrow + # scipy +oauthlib==3.2.2 + # via requests-oauthlib orjson==3.10.3 # via fastapi +overrides==7.7.0 + # via jupyter-server packaging==24.0 # via + # build # dask + # db-dtypes + # duckdb-engine + # google-cloud-bigquery + # great-expectations # gunicorn + # ibis-substrait + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # msal-extensions + # nbconvert + # pytest + # snowflake-connector-python + # sphinx pandas==2.2.2 # via # feast (setup.py) + # altair # dask # dask-expr + # db-dtypes + # google-cloud-bigquery + # great-expectations + # ibis-framework + # snowflake-connector-python +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.2 # via dask +pbr==6.0.0 + # via mock +pexpect==4.9.0 + # via ipython +pip==24.0 + # via pip-tools +pip-tools==7.4.1 + # via feast (setup.py) +platformdirs==3.11.0 + # via + # jupyter-core + # snowflake-connector-python + # virtualenv +pluggy==1.5.0 + # via pytest +ply==3.11 + # via thriftpy2 +portalocker==2.8.2 + # via msal-extensions +pre-commit==3.3.1 + # via feast (setup.py) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.45 + # via ipython +proto-plus==1.23.0 + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore protobuf==4.25.3 # via # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools # mypy-protobuf + # proto-plus + # substrait +psutil==5.9.0 + # via + # feast (setup.py) + # ipykernel +psycopg2-binary==2.9.9 + # via feast (setup.py) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via feast (setup.py) +py-cpuinfo==9.0.0 + # via pytest-benchmark +py4j==0.10.9.7 + # via pyspark pyarrow==15.0.2 # via # feast (setup.py) # dask-expr + # db-dtypes + # deltalake + # google-cloud-bigquery + # ibis-framework + # snowflake-connector-python +pyarrow-hotfix==0.6 + # via + # deltalake + # ibis-framework +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pybindgen==0.22.1 + # via feast (setup.py) +pycparser==2.22 + # via cffi pydantic==2.7.1 # via # feast (setup.py) # fastapi + # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # feast (setup.py) + # ipython + # nbconvert # rich + # sphinx +pyjwt[crypto]==2.8.0 + # via + # msal + # snowflake-connector-python +pymssql==2.3.0 + # via feast (setup.py) +pymysql==1.1.1 + # via feast (setup.py) +pyodbc==5.1.0 + # via feast (setup.py) +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via + # great-expectations + # httplib2 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pyspark==3.5.1 + # via feast (setup.py) +pytest==7.4.4 + # via + # feast (setup.py) + # pytest-benchmark + # pytest-cov + # pytest-env + # pytest-lazy-fixture + # pytest-mock + # pytest-ordering + # pytest-timeout + # pytest-xdist +pytest-benchmark==3.4.1 + # via feast (setup.py) +pytest-cov==5.0.0 + # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) +pytest-lazy-fixture==0.6.3 + # via feast (setup.py) +pytest-mock==1.10.4 + # via feast (setup.py) +pytest-ordering==0.6 + # via feast (setup.py) +pytest-timeout==1.4.2 + # via feast (setup.py) +pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 - # via pandas + # via + # arrow + # botocore + # google-cloud-bigquery + # great-expectations + # ibis-framework + # jupyter-client + # kubernetes + # moto + # pandas + # rockset + # trino python-dotenv==1.0.1 # via uvicorn +python-json-logger==2.0.7 + # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via pandas + # via + # great-expectations + # ibis-framework + # pandas + # snowflake-connector-python + # trino pyyaml==6.0.1 # via # feast (setup.py) # dask + # ibis-substrait + # jupyter-events + # kubernetes + # pre-commit + # responses # uvicorn +pyzmq==26.0.3 + # via + # ipykernel + # jupyter-client + # jupyter-server +redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 + # jupyter-events +regex==2024.5.15 # via feast (setup.py) +requests==2.31.0 + # via + # feast (setup.py) + # azure-core + # cachecontrol + # docker + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # great-expectations + # jupyterlab-server + # kubernetes + # moto + # msal + # requests-oauthlib + # responses + # snowflake-connector-python + # sphinx + # trino +requests-oauthlib==2.0.0 + # via kubernetes +responses==0.25.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events rich==13.7.1 - # via typer + # via + # ibis-framework + # typer +rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema # referencing +rsa==4.9 + # via google-auth +ruamel-yaml==0.17.17 + # via great-expectations +ruff==0.4.6 + # via feast (setup.py) +s3transfer==0.10.1 + # via boto3 +scipy==1.13.1 + # via great-expectations +send2trash==1.8.3 + # via jupyter-server +setuptools==70.0.0 + # via + # grpcio-tools + # kubernetes + # pip-tools shellingham==1.5.4 # via typer six==1.16.0 - # via python-dateutil + # via + # asttokens + # azure-core + # bleach + # geomet + # happybase + # isodate + # kubernetes + # mock + # python-dateutil + # rfc3339-validator + # thriftpy2 sniffio==1.3.1 # via # anyio # httpx -sqlalchemy[mypy]==2.0.30 +snowballstemmer==2.2.0 + # via sphinx +snowflake-connector-python[pandas]==3.10.1 # via feast (setup.py) +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sphinx==6.2.1 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy[mypy]==2.0.30 + # via + # feast (setup.py) + # duckdb-engine + # ibis-framework + # sqlalchemy-views +sqlalchemy-views==0.3.2 + # via ibis-framework +sqlglot==20.11.0 + # via ibis-framework sqlite-vec==0.0.1a9 # via feast (setup.py) +stack-data==0.6.3 + # via ipython starlette==0.37.2 # via fastapi +substrait==0.19.0 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.3.0 # via feast (setup.py) +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +testcontainers==4.4.0 + # via feast (setup.py) +thriftpy2==0.5.0 + # via happybase +tinycss2==1.3.0 + # via nbconvert toml==0.10.2 # via feast (setup.py) tomli==2.0.1 - # via mypy + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pytest + # pytest-env +tomlkit==0.12.5 + # via snowflake-connector-python toolz==0.12.1 # via + # altair # dask + # ibis-framework # partd +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado tqdm==4.66.4 + # via + # feast (setup.py) + # great-expectations +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +trino==0.328.0 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) typer==0.12.3 # via fastapi-cli +types-cffi==1.16.0.20240331 + # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240524 + # via feast (setup.py) +types-pyopenssl==24.1.0.20240425 + # via types-redis +types-python-dateutil==2.9.0.20240316 + # via + # feast (setup.py) + # arrow +types-pytz==2024.1.0.20240417 + # via feast (setup.py) +types-pyyaml==6.0.12.20240311 + # via feast (setup.py) +types-redis==4.6.0.20240425 + # via feast (setup.py) +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==70.0.0.20240524 + # via + # feast (setup.py) + # types-cffi +types-tabulate==0.9.0.20240106 + # via feast (setup.py) +types-urllib3==1.26.25.14 + # via types-requests typing-extensions==4.11.0 # via # anyio + # async-lru + # azure-core + # azure-storage-blob # fastapi + # great-expectations + # ibis-framework + # ipython # mypy # pydantic # pydantic-core + # snowflake-connector-python # sqlalchemy - # starlette + # testcontainers # typeguard # typer # uvicorn tzdata==2024.1 # via pandas +tzlocal==5.2 + # via + # great-expectations + # trino ujson==5.9.0 # via fastapi +uri-template==1.3.0 + # via jsonschema +uritemplate==4.1.1 + # via google-api-python-client urllib3==1.26.18 - # via requests + # via + # feast (setup.py) + # botocore + # docker + # elastic-transport + # great-expectations + # kubernetes + # minio + # requests + # responses + # rockset + # testcontainers uvicorn[standard]==0.29.0 # via # feast (setup.py) @@ -217,9 +1019,35 @@ uvicorn[standard]==0.29.0 # fastapi-cli uvloop==0.19.0 # via uvicorn +virtualenv==20.23.0 + # via + # feast (setup.py) + # pre-commit watchfiles==0.21.0 # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via + # jupyter-server + # kubernetes websockets==12.0 # via uvicorn +werkzeug==3.0.3 + # via moto +wheel==0.43.0 + # via pip-tools +widgetsnbextension==4.0.11 + # via ipywidgets +wrapt==1.16.0 + # via testcontainers +xmltodict==0.13.0 + # via moto zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 050c67f7cc4..c779fb58b50 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -69,9 +69,7 @@ idna==3.7 # httpx # requests importlib-metadata==7.1.0 - # via - # dask - # typeguard + # via dask jinja2==3.1.4 # via # feast (setup.py) @@ -200,7 +198,6 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy - # starlette # typeguard # typer # uvicorn diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index acbc643be9a..ce59e5e80d7 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,215 +1,998 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt +alabaster==0.7.16 + # via sphinx +altair==4.2.2 + # via great-expectations annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx + # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asn1crypto==1.5.1 + # via snowflake-connector-python +assertpy==1.1 + # via feast (setup.py) +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +atpublic==4.1.0 + # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing +azure-core==1.30.1 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.16.0 + # via feast (setup.py) +azure-storage-blob==12.20.0 + # via feast (setup.py) +babel==2.15.0 + # via + # jupyterlab-server + # sphinx +beautifulsoup4==4.12.3 + # via nbconvert +bidict==0.23.1 + # via ibis-framework +bleach==6.1.0 + # via nbconvert +boto3==1.34.116 + # via + # feast (setup.py) + # moto +botocore==1.34.116 + # via + # boto3 + # moto + # s3transfer +build==1.2.1 + # via + # feast (setup.py) + # pip-tools +cachecontrol==0.14.0 + # via firebase-admin +cachetools==5.3.3 + # via google-auth +cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.2.2 # via + # elastic-transport # httpcore # httpx + # kubernetes + # minio # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # argon2-cffi-bindings + # cryptography + # snowflake-connector-python +cfgv==3.4.0 + # via pre-commit charset-normalizer==3.3.2 - # via requests + # via + # requests + # snowflake-connector-python click==8.1.7 # via # feast (setup.py) # dask + # geomet + # great-expectations + # pip-tools # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) + # via + # feast (setup.py) + # great-expectations +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.5.3 + # via pytest-cov +cryptography==42.0.7 + # via + # feast (setup.py) + # azure-identity + # azure-storage-blob + # great-expectations + # moto + # msal + # pyjwt + # pyopenssl + # snowflake-connector-python + # types-pyopenssl + # types-redis dask[dataframe]==2024.5.0 # via # feast (setup.py) # dask-expr dask-expr==1.1.0 # via dask +db-dtypes==1.2.0 + # via google-cloud-bigquery +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +deltalake==0.17.4 + # via feast (setup.py) dill==0.3.8 # via feast (setup.py) +distlib==0.3.8 + # via virtualenv dnspython==2.6.1 # via email-validator +docker==7.1.0 + # via + # feast (setup.py) + # testcontainers +docutils==0.19 + # via sphinx +duckdb==0.10.3 + # via + # duckdb-engine + # ibis-framework +duckdb-engine==0.12.1 + # via ibis-framework +elastic-transport==8.13.1 + # via elasticsearch +elasticsearch==8.13.2 + # via feast (setup.py) email-validator==2.1.1 # via fastapi -exceptiongroup==1.2.1 - # via anyio +entrypoints==0.4 + # via altair +execnet==2.1.1 + # via pytest-xdist +executing==2.0.1 + # via stack-data fastapi==0.111.0 # via # feast (setup.py) # fastapi-cli fastapi-cli==0.0.2 # via fastapi +fastjsonschema==2.19.1 + # via nbformat +filelock==3.14.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 + # via feast (setup.py) +fqdn==1.5.1 + # via jsonschema fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask +geojson==2.5.0 + # via rockset +geomet==0.2.1.post1 + # via cassandra-driver +google-api-core[grpc]==2.19.0 + # via + # feast (setup.py) + # firebase-admin + # google-api-python-client + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-api-python-client==2.131.0 + # via firebase-admin +google-auth==2.29.0 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-bigquery-storage + # google-cloud-core + # google-cloud-firestore + # google-cloud-storage + # kubernetes +google-auth-httplib2==0.2.0 + # via google-api-python-client +google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-core==2.4.1 + # via + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-cloud-datastore==2.19.0 + # via feast (setup.py) +google-cloud-firestore==2.16.0 + # via firebase-admin +google-cloud-storage==2.16.0 + # via + # feast (setup.py) + # firebase-admin +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.63.0 + # via + # feast (setup.py) + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +great-expectations==0.18.15 + # via feast (setup.py) +grpc-google-iam-v1==0.13.0 + # via google-cloud-bigtable +grpcio==1.64.0 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools +grpcio-health-checking==1.62.2 + # via feast (setup.py) +grpcio-reflection==1.62.2 + # via feast (setup.py) +grpcio-status==1.62.2 + # via google-api-core +grpcio-testing==1.62.2 + # via feast (setup.py) +grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn +happybase==1.2.0 + # via feast (setup.py) +hazelcast-python-client==5.4.0 + # via feast (setup.py) +hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx +httplib2==0.22.0 + # via + # google-api-python-client + # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via fastapi + # via + # feast (setup.py) + # fastapi + # jupyterlab +ibis-framework[duckdb]==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.36 + # via pre-commit idna==3.7 # via # anyio # email-validator # httpx + # jsonschema # requests + # snowflake-connector-python +imagesize==1.4.1 + # via sphinx importlib-metadata==7.1.0 + # via dask +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.4 + # via jupyterlab +ipython==8.25.0 # via - # dask - # typeguard + # great-expectations + # ipykernel + # ipywidgets +ipywidgets==8.1.3 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython jinja2==3.1.4 # via # feast (setup.py) + # altair # fastapi -jsonschema==4.22.0 - # via feast (setup.py) + # great-expectations + # jupyter-server + # jupyterlab + # jupyterlab-server + # moto + # nbconvert + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +json5==0.9.25 + # via jupyterlab-server +jsonpatch==1.33 + # via great-expectations +jsonpointer==2.4 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.22.0 + # via + # feast (setup.py) + # altair + # great-expectations + # jupyter-events + # jupyterlab-server + # nbformat jsonschema-specifications==2023.12.1 # via jsonschema +jupyter-client==8.6.2 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.1 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.2.1 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.2 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.11 + # via ipywidgets +kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd +makefun==1.15.2 + # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via jinja2 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.21.2 + # via great-expectations +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython mdurl==0.1.2 # via markdown-it-py +minio==7.1.0 + # via feast (setup.py) +mistune==3.0.2 + # via + # great-expectations + # nbconvert mmh3==4.1.0 # via feast (setup.py) +mock==2.0.0 + # via feast (setup.py) +moto==4.2.14 + # via feast (setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +msgpack==1.0.8 + # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework mypy==1.10.0 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # great-expectations + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.9.0 + # via pre-commit +notebook==7.2.0 + # via great-expectations +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook numpy==1.26.4 # via # feast (setup.py) + # altair # dask + # db-dtypes + # great-expectations + # ibis-framework # pandas # pyarrow + # scipy +oauthlib==3.2.2 + # via requests-oauthlib orjson==3.10.3 # via fastapi +overrides==7.7.0 + # via jupyter-server packaging==24.0 # via + # build # dask + # db-dtypes + # duckdb-engine + # google-cloud-bigquery + # great-expectations # gunicorn + # ibis-substrait + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # msal-extensions + # nbconvert + # pytest + # snowflake-connector-python + # sphinx pandas==2.2.2 # via # feast (setup.py) + # altair # dask # dask-expr + # db-dtypes + # google-cloud-bigquery + # great-expectations + # ibis-framework + # snowflake-connector-python +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.2 # via dask +pbr==6.0.0 + # via mock +pexpect==4.9.0 + # via ipython +pip==24.0 + # via pip-tools +pip-tools==7.4.1 + # via feast (setup.py) +platformdirs==3.11.0 + # via + # jupyter-core + # snowflake-connector-python + # virtualenv +pluggy==1.5.0 + # via pytest +ply==3.11 + # via thriftpy2 +portalocker==2.8.2 + # via msal-extensions +pre-commit==3.3.1 + # via feast (setup.py) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.45 + # via ipython +proto-plus==1.23.0 + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore protobuf==4.25.3 # via # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools # mypy-protobuf + # proto-plus + # substrait +psutil==5.9.0 + # via + # feast (setup.py) + # ipykernel +psycopg2-binary==2.9.9 + # via feast (setup.py) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via feast (setup.py) +py-cpuinfo==9.0.0 + # via pytest-benchmark +py4j==0.10.9.7 + # via pyspark pyarrow==15.0.2 # via # feast (setup.py) # dask-expr + # db-dtypes + # deltalake + # google-cloud-bigquery + # ibis-framework + # snowflake-connector-python +pyarrow-hotfix==0.6 + # via + # deltalake + # ibis-framework +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pybindgen==0.22.1 + # via feast (setup.py) +pycparser==2.22 + # via cffi pydantic==2.7.1 # via # feast (setup.py) # fastapi + # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # feast (setup.py) + # ipython + # nbconvert # rich + # sphinx +pyjwt[crypto]==2.8.0 + # via + # msal + # snowflake-connector-python +pymssql==2.3.0 + # via feast (setup.py) +pymysql==1.1.1 + # via feast (setup.py) +pyodbc==5.1.0 + # via feast (setup.py) +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via + # great-expectations + # httplib2 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pyspark==3.5.1 + # via feast (setup.py) +pytest==7.4.4 + # via + # feast (setup.py) + # pytest-benchmark + # pytest-cov + # pytest-env + # pytest-lazy-fixture + # pytest-mock + # pytest-ordering + # pytest-timeout + # pytest-xdist +pytest-benchmark==3.4.1 + # via feast (setup.py) +pytest-cov==5.0.0 + # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) +pytest-lazy-fixture==0.6.3 + # via feast (setup.py) +pytest-mock==1.10.4 + # via feast (setup.py) +pytest-ordering==0.6 + # via feast (setup.py) +pytest-timeout==1.4.2 + # via feast (setup.py) +pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 - # via pandas + # via + # arrow + # botocore + # google-cloud-bigquery + # great-expectations + # ibis-framework + # jupyter-client + # kubernetes + # moto + # pandas + # rockset + # trino python-dotenv==1.0.1 # via uvicorn +python-json-logger==2.0.7 + # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via pandas + # via + # great-expectations + # ibis-framework + # pandas + # snowflake-connector-python + # trino pyyaml==6.0.1 # via # feast (setup.py) # dask + # ibis-substrait + # jupyter-events + # kubernetes + # pre-commit + # responses # uvicorn +pyzmq==26.0.3 + # via + # ipykernel + # jupyter-client + # jupyter-server +redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 + # jupyter-events +regex==2024.5.15 # via feast (setup.py) +requests==2.31.0 + # via + # feast (setup.py) + # azure-core + # cachecontrol + # docker + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # great-expectations + # jupyterlab-server + # kubernetes + # moto + # msal + # requests-oauthlib + # responses + # snowflake-connector-python + # sphinx + # trino +requests-oauthlib==2.0.0 + # via kubernetes +responses==0.25.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events rich==13.7.1 - # via typer + # via + # ibis-framework + # typer +rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema # referencing +rsa==4.9 + # via google-auth +ruamel-yaml==0.17.17 + # via great-expectations +ruff==0.4.6 + # via feast (setup.py) +s3transfer==0.10.1 + # via boto3 +scipy==1.13.1 + # via great-expectations +send2trash==1.8.3 + # via jupyter-server +setuptools==70.0.0 + # via + # grpcio-tools + # kubernetes + # pip-tools shellingham==1.5.4 # via typer six==1.16.0 - # via python-dateutil + # via + # asttokens + # azure-core + # bleach + # geomet + # happybase + # isodate + # kubernetes + # mock + # python-dateutil + # rfc3339-validator + # thriftpy2 sniffio==1.3.1 # via # anyio # httpx -sqlalchemy[mypy]==2.0.30 +snowballstemmer==2.2.0 + # via sphinx +snowflake-connector-python[pandas]==3.10.1 + # via feast (setup.py) +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sphinx==6.2.1 # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy[mypy]==2.0.30 + # via + # feast (setup.py) + # duckdb-engine + # ibis-framework + # sqlalchemy-views +sqlalchemy-views==0.3.2 + # via ibis-framework +sqlglot==20.11.0 + # via ibis-framework sqlite-vec==0.0.1a9 # via feast (setup.py) +stack-data==0.6.3 + # via ipython starlette==0.37.2 # via fastapi +substrait==0.19.0 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.3.0 # via feast (setup.py) +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +testcontainers==4.4.0 + # via feast (setup.py) +thriftpy2==0.5.0 + # via happybase +tinycss2==1.3.0 + # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy +tomlkit==0.12.5 + # via snowflake-connector-python toolz==0.12.1 # via + # altair # dask + # ibis-framework # partd +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado tqdm==4.66.4 + # via + # feast (setup.py) + # great-expectations +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +trino==0.328.0 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) typer==0.12.3 # via fastapi-cli +types-cffi==1.16.0.20240331 + # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240524 + # via feast (setup.py) +types-pyopenssl==24.1.0.20240425 + # via types-redis +types-python-dateutil==2.9.0.20240316 + # via + # feast (setup.py) + # arrow +types-pytz==2024.1.0.20240417 + # via feast (setup.py) +types-pyyaml==6.0.12.20240311 + # via feast (setup.py) +types-redis==4.6.0.20240425 + # via feast (setup.py) +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==70.0.0.20240524 + # via + # feast (setup.py) + # types-cffi +types-tabulate==0.9.0.20240106 + # via feast (setup.py) +types-urllib3==1.26.25.14 + # via types-requests typing-extensions==4.11.0 # via - # anyio + # azure-core + # azure-storage-blob # fastapi + # great-expectations + # ibis-framework + # ipython # mypy # pydantic # pydantic-core + # snowflake-connector-python # sqlalchemy - # starlette + # testcontainers # typeguard # typer - # uvicorn tzdata==2024.1 # via pandas +tzlocal==5.2 + # via + # great-expectations + # trino ujson==5.9.0 # via fastapi +uri-template==1.3.0 + # via jsonschema +uritemplate==4.1.1 + # via google-api-python-client urllib3==1.26.18 - # via requests + # via + # feast (setup.py) + # botocore + # docker + # elastic-transport + # great-expectations + # kubernetes + # minio + # requests + # responses + # rockset + # testcontainers uvicorn[standard]==0.29.0 # via # feast (setup.py) @@ -217,9 +1000,35 @@ uvicorn[standard]==0.29.0 # fastapi-cli uvloop==0.19.0 # via uvicorn +virtualenv==20.23.0 + # via + # feast (setup.py) + # pre-commit watchfiles==0.21.0 # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via + # jupyter-server + # kubernetes websockets==12.0 # via uvicorn +werkzeug==3.0.3 + # via moto +wheel==0.43.0 + # via pip-tools +widgetsnbextension==4.0.11 + # via ipywidgets +wrapt==1.16.0 + # via testcontainers +xmltodict==0.13.0 + # via moto zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 626a36b3e29..479402e6e0a 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -40,8 +40,6 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi -exceptiongroup==1.2.1 - # via anyio fastapi==0.111.0 # via # feast (setup.py) @@ -69,9 +67,7 @@ idna==3.7 # httpx # requests importlib-metadata==7.1.0 - # via - # dask - # typeguard + # via dask jinja2==3.1.4 # via # feast (setup.py) @@ -178,8 +174,6 @@ tenacity==8.3.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -194,16 +188,13 @@ types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic # pydantic-core # sqlalchemy - # starlette # typeguard # typer - # uvicorn tzdata==2024.1 # via pandas ujson==5.9.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 69177f5c94e..43c4bdfdb1a 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,215 +1,1030 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt +alabaster==0.7.16 + # via sphinx +altair==4.2.2 + # via great-expectations annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via # httpx + # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asn1crypto==1.5.1 + # via snowflake-connector-python +assertpy==1.1 + # via feast (setup.py) +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 + # via redis +atpublic==4.1.0 + # via ibis-framework attrs==23.2.0 # via # jsonschema # referencing +azure-core==1.30.1 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.16.0 + # via feast (setup.py) +azure-storage-blob==12.20.0 + # via feast (setup.py) +babel==2.15.0 + # via + # jupyterlab-server + # sphinx +beautifulsoup4==4.12.3 + # via nbconvert +bidict==0.23.1 + # via ibis-framework +bleach==6.1.0 + # via nbconvert +boto3==1.34.116 + # via + # feast (setup.py) + # moto +botocore==1.34.116 + # via + # boto3 + # moto + # s3transfer +build==1.2.1 + # via + # feast (setup.py) + # pip-tools +cachecontrol==0.14.0 + # via firebase-admin +cachetools==5.3.3 + # via google-auth +cassandra-driver==3.29.1 + # via feast (setup.py) certifi==2024.2.2 # via + # elastic-transport # httpcore # httpx + # kubernetes + # minio # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # argon2-cffi-bindings + # cryptography + # snowflake-connector-python +cfgv==3.4.0 + # via pre-commit charset-normalizer==3.3.2 - # via requests + # via + # requests + # snowflake-connector-python click==8.1.7 # via # feast (setup.py) # dask + # geomet + # great-expectations + # pip-tools # typer # uvicorn cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via feast (setup.py) + # via + # feast (setup.py) + # great-expectations +comm==0.2.2 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.5.3 + # via pytest-cov +cryptography==42.0.7 + # via + # feast (setup.py) + # azure-identity + # azure-storage-blob + # great-expectations + # moto + # msal + # pyjwt + # pyopenssl + # snowflake-connector-python + # types-pyopenssl + # types-redis dask[dataframe]==2024.5.0 # via # feast (setup.py) # dask-expr dask-expr==1.1.0 # via dask +db-dtypes==1.2.0 + # via google-cloud-bigquery +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +deltalake==0.17.4 + # via feast (setup.py) dill==0.3.8 # via feast (setup.py) +distlib==0.3.8 + # via virtualenv dnspython==2.6.1 # via email-validator +docker==7.1.0 + # via + # feast (setup.py) + # testcontainers +docutils==0.19 + # via sphinx +duckdb==0.10.3 + # via + # duckdb-engine + # ibis-framework +duckdb-engine==0.12.1 + # via ibis-framework +elastic-transport==8.13.1 + # via elasticsearch +elasticsearch==8.13.2 + # via feast (setup.py) email-validator==2.1.1 # via fastapi +entrypoints==0.4 + # via altair exceptiongroup==1.2.1 - # via anyio + # via + # anyio + # ipython + # pytest +execnet==2.1.1 + # via pytest-xdist +executing==2.0.1 + # via stack-data fastapi==0.111.0 # via # feast (setup.py) # fastapi-cli fastapi-cli==0.0.2 # via fastapi +fastjsonschema==2.19.1 + # via nbformat +filelock==3.14.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 + # via feast (setup.py) +fqdn==1.5.1 + # via jsonschema fsspec==2023.12.2 - # via dask + # via + # feast (setup.py) + # dask +geojson==2.5.0 + # via rockset +geomet==0.2.1.post1 + # via cassandra-driver +google-api-core[grpc]==2.19.0 + # via + # feast (setup.py) + # firebase-admin + # google-api-python-client + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-api-python-client==2.131.0 + # via firebase-admin +google-auth==2.29.0 + # via + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-cloud-bigquery-storage + # google-cloud-core + # google-cloud-firestore + # google-cloud-storage + # kubernetes +google-auth-httplib2==0.2.0 + # via google-api-python-client +google-cloud-bigquery[pandas]==3.12.0 + # via feast (setup.py) +google-cloud-bigquery-storage==2.25.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.1 + # via feast (setup.py) +google-cloud-core==2.4.1 + # via + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # google-cloud-storage +google-cloud-datastore==2.19.0 + # via feast (setup.py) +google-cloud-firestore==2.16.0 + # via firebase-admin +google-cloud-storage==2.16.0 + # via + # feast (setup.py) + # firebase-admin +google-crc32c==1.5.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos[grpc]==1.63.0 + # via + # feast (setup.py) + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +great-expectations==0.18.15 + # via feast (setup.py) +grpc-google-iam-v1==0.13.0 + # via google-cloud-bigtable +grpcio==1.64.0 + # via + # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools +grpcio-health-checking==1.62.2 + # via feast (setup.py) +grpcio-reflection==1.62.2 + # via feast (setup.py) +grpcio-status==1.62.2 + # via google-api-core +grpcio-testing==1.62.2 + # via feast (setup.py) +grpcio-tools==1.62.2 + # via feast (setup.py) gunicorn==22.0.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn +happybase==1.2.0 + # via feast (setup.py) +hazelcast-python-client==5.4.0 + # via feast (setup.py) +hiredis==2.3.2 + # via feast (setup.py) httpcore==1.0.5 # via httpx +httplib2==0.22.0 + # via + # google-api-python-client + # google-auth-httplib2 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via fastapi + # via + # feast (setup.py) + # fastapi + # jupyterlab +ibis-framework[duckdb]==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.36 + # via pre-commit idna==3.7 # via # anyio # email-validator # httpx + # jsonschema # requests + # snowflake-connector-python +imagesize==1.4.1 + # via sphinx importlib-metadata==7.1.0 # via + # build # dask + # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # nbconvert + # sphinx # typeguard +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.4 + # via jupyterlab +ipython==8.18.1 + # via + # great-expectations + # ipykernel + # ipywidgets +ipywidgets==8.1.3 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython jinja2==3.1.4 # via # feast (setup.py) + # altair # fastapi -jsonschema==4.22.0 - # via feast (setup.py) + # great-expectations + # jupyter-server + # jupyterlab + # jupyterlab-server + # moto + # nbconvert + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +json5==0.9.25 + # via jupyterlab-server +jsonpatch==1.33 + # via great-expectations +jsonpointer==2.4 + # via + # jsonpatch + # jsonschema +jsonschema[format-nongpl]==4.22.0 + # via + # feast (setup.py) + # altair + # great-expectations + # jupyter-events + # jupyterlab-server + # nbformat jsonschema-specifications==2023.12.1 # via jsonschema +jupyter-client==8.6.2 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.1 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.2.1 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.2 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.11 + # via ipywidgets +kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd +makefun==1.15.2 + # via great-expectations markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 - # via jinja2 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.21.2 + # via great-expectations +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython mdurl==0.1.2 # via markdown-it-py +minio==7.1.0 + # via feast (setup.py) +mistune==3.0.2 + # via + # great-expectations + # nbconvert mmh3==4.1.0 # via feast (setup.py) +mock==2.0.0 + # via feast (setup.py) +moto==4.2.14 + # via feast (setup.py) +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +msgpack==1.0.8 + # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework mypy==1.10.0 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # great-expectations + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +nodeenv==1.9.0 + # via pre-commit +notebook==7.2.0 + # via great-expectations +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook numpy==1.26.4 # via # feast (setup.py) + # altair # dask + # db-dtypes + # great-expectations + # ibis-framework # pandas # pyarrow + # scipy +oauthlib==3.2.2 + # via requests-oauthlib orjson==3.10.3 # via fastapi +overrides==7.7.0 + # via jupyter-server packaging==24.0 # via + # build # dask + # db-dtypes + # duckdb-engine + # google-cloud-bigquery + # great-expectations # gunicorn + # ibis-substrait + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # marshmallow + # msal-extensions + # nbconvert + # pytest + # snowflake-connector-python + # sphinx pandas==2.2.2 # via # feast (setup.py) + # altair # dask # dask-expr + # db-dtypes + # google-cloud-bigquery + # great-expectations + # ibis-framework + # snowflake-connector-python +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.2 # via dask +pbr==6.0.0 + # via mock +pexpect==4.9.0 + # via ipython +pip==24.0 + # via pip-tools +pip-tools==7.4.1 + # via feast (setup.py) +platformdirs==3.11.0 + # via + # jupyter-core + # snowflake-connector-python + # virtualenv +pluggy==1.5.0 + # via pytest +ply==3.11 + # via thriftpy2 +portalocker==2.8.2 + # via msal-extensions +pre-commit==3.3.1 + # via feast (setup.py) +prometheus-client==0.20.0 + # via jupyter-server +prompt-toolkit==3.0.45 + # via ipython +proto-plus==1.23.0 + # via + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore protobuf==4.25.3 # via # feast (setup.py) + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-firestore + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-health-checking + # grpcio-reflection + # grpcio-status + # grpcio-testing + # grpcio-tools # mypy-protobuf + # proto-plus + # substrait +psutil==5.9.0 + # via + # feast (setup.py) + # ipykernel +psycopg2-binary==2.9.9 + # via feast (setup.py) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via feast (setup.py) +py-cpuinfo==9.0.0 + # via pytest-benchmark +py4j==0.10.9.7 + # via pyspark pyarrow==15.0.2 # via # feast (setup.py) # dask-expr + # db-dtypes + # deltalake + # google-cloud-bigquery + # ibis-framework + # snowflake-connector-python +pyarrow-hotfix==0.6 + # via + # deltalake + # ibis-framework +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pybindgen==0.22.1 + # via feast (setup.py) +pycparser==2.22 + # via cffi pydantic==2.7.1 # via # feast (setup.py) # fastapi + # great-expectations pydantic-core==2.18.2 # via pydantic pygments==2.18.0 # via # feast (setup.py) + # ipython + # nbconvert # rich + # sphinx +pyjwt[crypto]==2.8.0 + # via + # msal + # snowflake-connector-python +pymssql==2.3.0 + # via feast (setup.py) +pymysql==1.1.1 + # via feast (setup.py) +pyodbc==5.1.0 + # via feast (setup.py) +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via + # great-expectations + # httplib2 +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pyspark==3.5.1 + # via feast (setup.py) +pytest==7.4.4 + # via + # feast (setup.py) + # pytest-benchmark + # pytest-cov + # pytest-env + # pytest-lazy-fixture + # pytest-mock + # pytest-ordering + # pytest-timeout + # pytest-xdist +pytest-benchmark==3.4.1 + # via feast (setup.py) +pytest-cov==5.0.0 + # via feast (setup.py) +pytest-env==1.1.3 + # via feast (setup.py) +pytest-lazy-fixture==0.6.3 + # via feast (setup.py) +pytest-mock==1.10.4 + # via feast (setup.py) +pytest-ordering==0.6 + # via feast (setup.py) +pytest-timeout==1.4.2 + # via feast (setup.py) +pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 - # via pandas + # via + # arrow + # botocore + # google-cloud-bigquery + # great-expectations + # ibis-framework + # jupyter-client + # kubernetes + # moto + # pandas + # rockset + # trino python-dotenv==1.0.1 # via uvicorn +python-json-logger==2.0.7 + # via jupyter-events python-multipart==0.0.9 # via fastapi pytz==2024.1 - # via pandas + # via + # great-expectations + # ibis-framework + # pandas + # snowflake-connector-python + # trino pyyaml==6.0.1 # via # feast (setup.py) # dask + # ibis-substrait + # jupyter-events + # kubernetes + # pre-commit + # responses # uvicorn +pyzmq==26.0.3 + # via + # ipykernel + # jupyter-client + # jupyter-server +redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 + # jupyter-events +regex==2024.5.15 # via feast (setup.py) +requests==2.31.0 + # via + # feast (setup.py) + # azure-core + # cachecontrol + # docker + # google-api-core + # google-cloud-bigquery + # google-cloud-storage + # great-expectations + # jupyterlab-server + # kubernetes + # moto + # msal + # requests-oauthlib + # responses + # snowflake-connector-python + # sphinx + # trino +requests-oauthlib==2.0.0 + # via kubernetes +responses==0.25.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events rich==13.7.1 - # via typer + # via + # ibis-framework + # typer +rockset==2.1.2 + # via feast (setup.py) rpds-py==0.18.1 # via # jsonschema # referencing +rsa==4.9 + # via google-auth +ruamel-yaml==0.17.17 + # via great-expectations +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +ruff==0.4.6 + # via feast (setup.py) +s3transfer==0.10.1 + # via boto3 +scipy==1.13.1 + # via great-expectations +send2trash==1.8.3 + # via jupyter-server +setuptools==70.0.0 + # via + # grpcio-tools + # kubernetes + # pip-tools shellingham==1.5.4 # via typer six==1.16.0 - # via python-dateutil + # via + # asttokens + # azure-core + # bleach + # geomet + # happybase + # isodate + # kubernetes + # mock + # python-dateutil + # rfc3339-validator + # thriftpy2 sniffio==1.3.1 # via # anyio # httpx -sqlalchemy[mypy]==2.0.30 +snowballstemmer==2.2.0 + # via sphinx +snowflake-connector-python[pandas]==3.10.1 # via feast (setup.py) +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sphinx==6.2.1 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy[mypy]==2.0.30 + # via + # feast (setup.py) + # duckdb-engine + # ibis-framework + # sqlalchemy-views +sqlalchemy-views==0.3.2 + # via ibis-framework +sqlglot==20.11.0 + # via ibis-framework sqlite-vec==0.0.1a9 # via feast (setup.py) +stack-data==0.6.3 + # via ipython starlette==0.37.2 # via fastapi +substrait==0.19.0 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.3.0 # via feast (setup.py) +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +testcontainers==4.4.0 + # via feast (setup.py) +thriftpy2==0.5.0 + # via happybase +tinycss2==1.3.0 + # via nbconvert toml==0.10.2 # via feast (setup.py) tomli==2.0.1 - # via mypy + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pytest + # pytest-env +tomlkit==0.12.5 + # via snowflake-connector-python toolz==0.12.1 # via + # altair # dask + # ibis-framework # partd +tornado==6.4 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado tqdm==4.66.4 + # via + # feast (setup.py) + # great-expectations +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +trino==0.328.0 # via feast (setup.py) typeguard==4.2.1 # via feast (setup.py) typer==0.12.3 # via fastapi-cli +types-cffi==1.16.0.20240331 + # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf +types-pymysql==1.1.0.20240524 + # via feast (setup.py) +types-pyopenssl==24.1.0.20240425 + # via types-redis +types-python-dateutil==2.9.0.20240316 + # via + # feast (setup.py) + # arrow +types-pytz==2024.1.0.20240417 + # via feast (setup.py) +types-pyyaml==6.0.12.20240311 + # via feast (setup.py) +types-redis==4.6.0.20240425 + # via feast (setup.py) +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==70.0.0.20240524 + # via + # feast (setup.py) + # types-cffi +types-tabulate==0.9.0.20240106 + # via feast (setup.py) +types-urllib3==1.26.25.14 + # via types-requests typing-extensions==4.11.0 # via # anyio + # async-lru + # azure-core + # azure-storage-blob # fastapi + # great-expectations + # ibis-framework + # ipython # mypy # pydantic # pydantic-core + # snowflake-connector-python # sqlalchemy # starlette + # testcontainers # typeguard # typer # uvicorn tzdata==2024.1 # via pandas +tzlocal==5.2 + # via + # great-expectations + # trino ujson==5.9.0 # via fastapi +uri-template==1.3.0 + # via jsonschema +uritemplate==4.1.1 + # via google-api-python-client urllib3==1.26.18 - # via requests + # via + # feast (setup.py) + # botocore + # docker + # elastic-transport + # great-expectations + # kubernetes + # minio + # requests + # responses + # rockset + # snowflake-connector-python + # testcontainers uvicorn[standard]==0.29.0 # via # feast (setup.py) @@ -217,9 +1032,35 @@ uvicorn[standard]==0.29.0 # fastapi-cli uvloop==0.19.0 # via uvicorn +virtualenv==20.23.0 + # via + # feast (setup.py) + # pre-commit watchfiles==0.21.0 # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via + # jupyter-server + # kubernetes websockets==12.0 # via uvicorn +werkzeug==3.0.3 + # via moto +wheel==0.43.0 + # via pip-tools +widgetsnbextension==4.0.11 + # via ipywidgets +wrapt==1.16.0 + # via testcontainers +xmltodict==0.13.0 + # via moto zipp==3.18.1 # via importlib-metadata From c0fcb06ba4a1b936e4d7c87b8259e6dcfbc26df3 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 15:04:34 -0400 Subject: [PATCH 31/58] fixed type Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index bb25bc0edb9..3e4b61a561b 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -17,7 +17,7 @@ import struct from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union import sqlite_vec from pydantic import StrictStr @@ -35,6 +35,7 @@ from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.utils import to_naive_utc +from google.protobuf.internal.containers import RepeatedScalarFieldContainer class SqliteOnlineStoreConfig(FeastConfigBaseModel): @@ -419,7 +420,7 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" -def serialize_f32(vector: List[float], vector_length: int) -> bytes: +def serialize_f32(vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int) -> bytes: """serializes a list of floats into a compact "raw bytes" format""" return struct.pack(f"{vector_length}f", *vector) From 34e4d29685de557817a644effc57389f81f91fba Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 31 May 2024 15:07:30 -0400 Subject: [PATCH 32/58] linter Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 3e4b61a561b..8733a760845 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -20,6 +20,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union import sqlite_vec +from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr from feast import Entity @@ -35,7 +36,6 @@ from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.utils import to_naive_utc -from google.protobuf.internal.containers import RepeatedScalarFieldContainer class SqliteOnlineStoreConfig(FeastConfigBaseModel): @@ -420,7 +420,9 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" -def serialize_f32(vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int) -> bytes: +def serialize_f32( + vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int +) -> bytes: """serializes a list of floats into a compact "raw bytes" format""" return struct.pack(f"{vector_length}f", *vector) From 21bbb7d04968b92cff8ba2bd1bc9caa6d222fe63 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 1 Jun 2024 14:20:14 -0400 Subject: [PATCH 33/58] testing sqlite_vec import Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/unit/online_store/test_online_retrieval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index ec66fe283c9..140126637a8 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import sqlite_vec import pytest from pandas.testing import assert_frame_equal From 482df7ea1adc71886f717ba9b820adb728adc5a2 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 1 Jun 2024 14:28:10 -0400 Subject: [PATCH 34/58] adding minimal example test Signed-off-by: Francisco Javier Arceo --- .../online_store/test_online_retrieval.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 140126637a8..31a0aad0eb0 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import sqlite3 import sqlite_vec import pytest from pandas.testing import assert_frame_equal @@ -508,3 +509,34 @@ def test_get_online_documents() -> None: assert "Embeddings" in result assert "distance" in result assert len(result["distance"]) == 3 + + +def test_sqlite_vec_import() -> None: + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + sqlite_vec.load(db) + + db.execute(""" + create virtual table vec_examples using vec0( + sample_embedding float[8] + ); + """) + db.execute(""" + insert into vec_examples(rowid, sample_embedding) + values + (1, '[-0.200, 0.250, 0.341, -0.211, 0.645, 0.935, -0.316, -0.924]'), + (2, '[0.443, -0.501, 0.355, -0.771, 0.707, -0.708, -0.185, 0.362]'), + (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), + (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]'); + """) + result = db.execute(""" + select + rowid, + distance + from vec_examples + where sample_embedding match '[0.890, 0.544, 0.825, 0.961, 0.358, 0.0196, 0.521, 0.175]' + order by distance + limit 2; + """).fetchall() + result = [(rowid, round(distance, 2)) for rowid, distance in result] + assert result == [(2, 2.39), (1, 2.39)] \ No newline at end of file From 1d8601fc4cabcb27976c7bbba64fe4b5f16577cc Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 1 Jun 2024 14:33:46 -0400 Subject: [PATCH 35/58] lint Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/unit/online_store/test_online_retrieval.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 31a0aad0eb0..1fef39a8d12 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -1,12 +1,12 @@ import os +import sqlite3 import time from datetime import datetime import numpy as np import pandas as pd -import sqlite3 -import sqlite_vec import pytest +import sqlite_vec from pandas.testing import assert_frame_equal from feast import FeatureStore, RepoConfig @@ -539,4 +539,4 @@ def test_sqlite_vec_import() -> None: limit 2; """).fetchall() result = [(rowid, round(distance, 2)) for rowid, distance in result] - assert result == [(2, 2.39), (1, 2.39)] \ No newline at end of file + assert result == [(2, 2.39), (1, 2.39)] From 75b04c15bec4b5641abb24121ea729786bf6b65e Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 2 Jun 2024 06:20:11 -0400 Subject: [PATCH 36/58] testing raw sqlite Signed-off-by: Francisco Javier Arceo --- .../online_store/test_online_retrieval.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 1fef39a8d12..c4ab13f5179 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -499,16 +499,16 @@ def test_get_online_documents() -> None: ) assert record_count == len(data) + documents_df.shape[0] - query = np.random.random( - vector_length, - ) - result = store.retrieve_online_documents( - feature="document_embeddings:Embeddings", query=query, top_k=3 - ).to_dict() - - assert "Embeddings" in result - assert "distance" in result - assert len(result["distance"]) == 3 + # query = np.random.random( + # vector_length, + # ) + # result = store.retrieve_online_documents( + # feature="document_embeddings:Embeddings", query=query, top_k=3 + # ).to_dict() + # + # assert "Embeddings" in result + # assert "distance" in result + # assert len(result["distance"]) == 3 def test_sqlite_vec_import() -> None: From 13da55d494afd4acedf1b340fd1271c3d0b91fc3 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Sun, 2 Jun 2024 11:59:03 -0400 Subject: [PATCH 37/58] Printing package version --- .../tests/unit/online_store/test_online_retrieval.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index c4ab13f5179..2e91ff7befe 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -521,6 +521,7 @@ def test_sqlite_vec_import() -> None: sample_embedding float[8] ); """) + db.execute(""" insert into vec_examples(rowid, sample_embedding) values @@ -529,6 +530,12 @@ def test_sqlite_vec_import() -> None: (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]'); """) + + sqlite_version, vec_version = db.execute( + "select sqlite_version(), vec_version()" + ).fetchone() + print(f"sqlite_version={sqlite_version}, vec_version={vec_version}") + result = db.execute(""" select rowid, From 1b9a3bbdb3be2b3bbc43625dad7c23c12b2c9cc6 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 2 Jun 2024 21:25:44 -0400 Subject: [PATCH 38/58] printing version Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/unit/online_store/test_online_retrieval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 2e91ff7befe..1d53eca7f4f 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -534,6 +534,7 @@ def test_sqlite_vec_import() -> None: sqlite_version, vec_version = db.execute( "select sqlite_version(), vec_version()" ).fetchone() + assert vec_version == 'v0.0.1-alpha.10' print(f"sqlite_version={sqlite_version}, vec_version={vec_version}") result = db.execute(""" From 2eedda8a82fb4e796832f491b862dcf88b25acb6 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 2 Jun 2024 21:36:38 -0400 Subject: [PATCH 39/58] updated requirements --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- setup.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index bda20c74216..6dc0cd39627 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -864,7 +864,7 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework -sqlite-vec==0.0.1a9 +sqlite-vec==0.0.1a10 # via feast (setup.py) stack-data==0.6.3 # via ipython diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index c779fb58b50..23bd94feb5c 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -166,7 +166,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vec==0.0.1a9 +sqlite-vec==0.0.1a10 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index ce59e5e80d7..2d2106bf532 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -857,7 +857,7 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework -sqlite-vec==0.0.1a9 +sqlite-vec==0.0.1a10 # via feast (setup.py) stack-data==0.6.3 # via ipython diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 479402e6e0a..9698eea6dff 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -164,7 +164,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vec==0.0.1a9 +sqlite-vec==0.0.1a10 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 43c4bdfdb1a..a52b78bf78a 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -875,7 +875,7 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework -sqlite-vec==0.0.1a9 +sqlite-vec==0.0.1a10 # via feast (setup.py) stack-data==0.6.3 # via ipython diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index ca657582332..579f39135e3 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -168,7 +168,7 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) -sqlite-vec==0.0.1a9 +sqlite-vec==0.0.1a10 # via feast (setup.py) starlette==0.37.2 # via fastapi diff --git a/setup.py b/setup.py index 4622b640640..3d310852e80 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", - "sqlite-vec", + "sqlite-vec==v0.0.1-alpha.10", "SQLAlchemy[mypy]>1", "tabulate>=0.8.0,<1", "tenacity>=7,<9", From bc90a1c3b72e32b2f88320e6e5e0510234c8fb53 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 8 Jun 2024 12:35:07 -0400 Subject: [PATCH 40/58] rebuilding requirments Signed-off-by: Francisco Javier Arceo --- .../requirements/py3.11-ci-requirements.txt | 21 ------------------- .../requirements/py3.11-requirements.txt | 6 ------ .../requirements/py3.9-ci-requirements.txt | 15 ++++++++++++- .../requirements/py3.9-requirements.txt | 5 ++++- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 636fe97d330..643e3715c60 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -36,10 +36,6 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via - # aiohttp - # redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -181,11 +177,6 @@ email-validator==2.1.1 # via fastapi entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -910,15 +901,6 @@ tinycss2==1.3.0 toml==0.10.2 # via feast (setup.py) tomlkit==0.12.5 -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env # via snowflake-connector-python toolz==0.12.1 # via @@ -991,8 +973,6 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi @@ -1007,7 +987,6 @@ typing-extensions==4.11.0 # testcontainers # typeguard # typer - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 6896d2a6460..9698eea6dff 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -40,8 +40,6 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi -exceptiongroup==1.2.1 - # via anyio fastapi==0.111.0 # via # feast (setup.py) @@ -176,8 +174,6 @@ tenacity==8.3.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -192,7 +188,6 @@ types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic @@ -200,7 +195,6 @@ typing-extensions==4.11.0 # sqlalchemy # typeguard # typer - # uvicorn tzdata==2024.1 # via pandas ujson==5.9.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 2db60d56c4c..8aca7006961 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -350,7 +350,16 @@ idna==3.7 imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 - # via dask + # via + # build + # dask + # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # nbconvert + # sphinx + # typeguard iniconfig==2.0.0 # via pytest ipykernel==6.29.4 @@ -993,6 +1002,7 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # aioitertools # anyio # async-lru # azure-core @@ -1000,11 +1010,13 @@ typing-extensions==4.11.0 # fastapi # great-expectations # ibis-framework + # ipython # mypy # pydantic # pydantic-core # snowflake-connector-python # sqlalchemy + # starlette # testcontainers # typeguard # typer @@ -1033,6 +1045,7 @@ urllib3==1.26.18 # requests # responses # rockset + # snowflake-connector-python # testcontainers uvicorn[standard]==0.29.0 # via diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 1895b6f3fb5..579f39135e3 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -69,7 +69,9 @@ idna==3.7 # httpx # requests importlib-metadata==7.1.0 - # via dask + # via + # dask + # typeguard jinja2==3.1.4 # via # feast (setup.py) @@ -198,6 +200,7 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy + # starlette # typeguard # typer # uvicorn From c50f1d22e4873b1f45491eb77d77fac6cc742c73 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 8 Jun 2024 12:43:05 -0400 Subject: [PATCH 41/58] only going to run this on 3.10 for now Signed-off-by: Francisco Javier Arceo --- .../online_store/test_online_retrieval.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 1d53eca7f4f..8aa6cc22758 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -1,5 +1,7 @@ import os +import platform import sqlite3 +import sys import time from datetime import datetime @@ -421,7 +423,8 @@ def test_online_to_df(): assert_frame_equal(result_df[ordered_column], expected_df) -def test_get_online_documents() -> None: +@pytest.mark.skipif(sys.version_info != (3, 10) and platform.platform != "Darwin") +def test_sqlite_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ @@ -499,18 +502,19 @@ def test_get_online_documents() -> None: ) assert record_count == len(data) + documents_df.shape[0] - # query = np.random.random( - # vector_length, - # ) - # result = store.retrieve_online_documents( - # feature="document_embeddings:Embeddings", query=query, top_k=3 - # ).to_dict() - # - # assert "Embeddings" in result - # assert "distance" in result - # assert len(result["distance"]) == 3 + query = np.random.random( + vector_length, + ) + result = store.retrieve_online_documents( + feature="document_embeddings:Embeddings", query=query, top_k=3 + ).to_dict() + + assert "Embeddings" in result + assert "distance" in result + assert len(result["distance"]) == 3 +@pytest.mark.skipif(sys.version_info != (3, 10) and platform.platform != "Darwin") def test_sqlite_vec_import() -> None: db = sqlite3.connect(":memory:") db.enable_load_extension(True) @@ -521,7 +525,7 @@ def test_sqlite_vec_import() -> None: sample_embedding float[8] ); """) - + db.execute(""" insert into vec_examples(rowid, sample_embedding) values @@ -530,13 +534,13 @@ def test_sqlite_vec_import() -> None: (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]'); """) - + sqlite_version, vec_version = db.execute( "select sqlite_version(), vec_version()" ).fetchone() - assert vec_version == 'v0.0.1-alpha.10' + assert vec_version == "v0.0.1-alpha.10" print(f"sqlite_version={sqlite_version}, vec_version={vec_version}") - + result = db.execute(""" select rowid, From c4eafab0ff54879f789d97506780cd61d55292b0 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 8 Jun 2024 12:49:14 -0400 Subject: [PATCH 42/58] updated docs for sqlite caveats Signed-off-by: Francisco Javier Arceo --- docs/reference/alpha-vector-database.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index cdabf195f16..d3861404314 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -13,7 +13,9 @@ Below are supported vector databases and implemented features: | Elasticsearch | [x] | [x] | | Milvus | [ ] | [ ] | | Faiss | [ ] | [ ] | +| SQLite | [x] | [ ] | +Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. ## Example From eb3712105455c7e5d18e032efb4cd7cad5b6954a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 8 Jun 2024 13:00:45 -0400 Subject: [PATCH 43/58] adding reason Signed-off-by: Francisco Javier Arceo --- .../tests/unit/online_store/test_online_retrieval.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 8aa6cc22758..3d8aa5183d8 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -423,7 +423,10 @@ def test_online_to_df(): assert_frame_equal(result_df[ordered_column], expected_df) -@pytest.mark.skipif(sys.version_info != (3, 10) and platform.platform != "Darwin") +@pytest.mark.skipif( + sys.version_info != (3, 10) and platform.platform != "Darwin", + reason="Only works on Python 3.10 and MacOS", +) def test_sqlite_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. @@ -514,7 +517,10 @@ def test_sqlite_get_online_documents() -> None: assert len(result["distance"]) == 3 -@pytest.mark.skipif(sys.version_info != (3, 10) and platform.platform != "Darwin") +@pytest.mark.skipif( + sys.version_info != (3, 10) and platform.platform != "Darwin", + reason="Only works on Python 3.10 and MacOS", +) def test_sqlite_vec_import() -> None: db = sqlite3.connect(":memory:") db.enable_load_extension(True) From cfd761170b7f0a5c5e5180a2514540ca4a243744 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 8 Jun 2024 13:30:09 -0400 Subject: [PATCH 44/58] skipping Signed-off-by: Francisco Javier Arceo --- .../tests/unit/online_store/test_online_retrieval.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 3d8aa5183d8..9b552474358 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -424,7 +424,7 @@ def test_online_to_df(): @pytest.mark.skipif( - sys.version_info != (3, 10) and platform.platform != "Darwin", + sys.version_info[0:2] != (3, 10) and platform.system != "Darwin", reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_get_online_documents() -> None: @@ -518,10 +518,16 @@ def test_sqlite_get_online_documents() -> None: @pytest.mark.skipif( - sys.version_info != (3, 10) and platform.platform != "Darwin", + sys.version_info[0:2] != (3, 10) and platform.system != "Darwin", reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_vec_import() -> None: + print( + sys.version_info[0:2] != (3, 10), + sys.version_info[0:2], + platform.platform(), + platform.system(), + ) db = sqlite3.connect(":memory:") db.enable_load_extension(True) sqlite_vec.load(db) From 58d595a5bfb413ea8c645d1dbe751bb2dc93527d Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sat, 8 Jun 2024 15:02:09 -0400 Subject: [PATCH 45/58] updated tests Signed-off-by: Francisco Javier Arceo --- .../tests/integration/registration/test_universal_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/integration/registration/test_universal_cli.py b/sdk/python/tests/integration/registration/test_universal_cli.py index c16b26fee65..fc90108d787 100644 --- a/sdk/python/tests/integration/registration/test_universal_cli.py +++ b/sdk/python/tests/integration/registration/test_universal_cli.py @@ -74,13 +74,13 @@ def test_universal_cli(): cwd=repo_path, ) assertpy.assert_that(result.returncode).is_equal_to(0) - assertpy.assert_that(fs.list_feature_views()).is_length(4) + assertpy.assert_that(fs.list_feature_views()).is_length(5) result = runner.run( ["data-sources", "describe", "customer_profile_source"], cwd=repo_path, ) assertpy.assert_that(result.returncode).is_equal_to(0) - assertpy.assert_that(fs.list_data_sources()).is_length(4) + assertpy.assert_that(fs.list_data_sources()).is_length(5) # entity & feature view describe commands should fail when objects don't exist result = runner.run(["entities", "describe", "foo"], cwd=repo_path) From 8db355c116197963eb11ab287660b8fcb43b27e3 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 12:43:21 -0400 Subject: [PATCH 46/58] removing print Signed-off-by: Francisco Javier Arceo --- .../tests/unit/online_store/test_online_retrieval.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 9b552474358..40be40435b6 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -505,11 +505,11 @@ def test_sqlite_get_online_documents() -> None: ) assert record_count == len(data) + documents_df.shape[0] - query = np.random.random( + query_embedding = np.random.random( vector_length, ) result = store.retrieve_online_documents( - feature="document_embeddings:Embeddings", query=query, top_k=3 + feature="document_embeddings:Embeddings", query=query_embedding, top_k=3 ).to_dict() assert "Embeddings" in result @@ -522,12 +522,6 @@ def test_sqlite_get_online_documents() -> None: reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_vec_import() -> None: - print( - sys.version_info[0:2] != (3, 10), - sys.version_info[0:2], - platform.platform(), - platform.system(), - ) db = sqlite3.connect(":memory:") db.enable_load_extension(True) sqlite_vec.load(db) From 9335fa50026d543799820f31c9aa9da4f6b0d7d7 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 21:38:57 -0400 Subject: [PATCH 47/58] added method call Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/unit/online_store/test_online_retrieval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 40be40435b6..1377082b0a6 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -424,7 +424,7 @@ def test_online_to_df(): @pytest.mark.skipif( - sys.version_info[0:2] != (3, 10) and platform.system != "Darwin", + sys.version_info[0:2] != (3, 10) and platform.system() != "Darwin", reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_get_online_documents() -> None: @@ -518,7 +518,7 @@ def test_sqlite_get_online_documents() -> None: @pytest.mark.skipif( - sys.version_info[0:2] != (3, 10) and platform.system != "Darwin", + sys.version_info[0:2] != (3, 10) and platform.system() != "Darwin", reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_vec_import() -> None: From 511291a638a7a0a8a07dd2705ac1a43f5c340d53 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 21:47:21 -0400 Subject: [PATCH 48/58] added prubt Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/unit/online_store/test_online_retrieval.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 1377082b0a6..2f91112ffdb 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -431,6 +431,7 @@ def test_sqlite_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ + print(sys.version_info, platform.system()) n = 10 # number of samples - note: we'll actually double it vector_length = 8 runner = CliRunner() @@ -522,6 +523,7 @@ def test_sqlite_get_online_documents() -> None: reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_vec_import() -> None: + print(sys.version_info, platform.system()) db = sqlite3.connect(":memory:") db.enable_load_extension(True) sqlite_vec.load(db) From b544527ecdb8eaed6505796c0b5c8bc58dafce82 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:02:16 -0400 Subject: [PATCH 49/58] added print Signed-off-by: Francisco Javier Arceo --- sdk/python/tests/unit/online_store/test_online_retrieval.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 2f91112ffdb..c1a2dd86b07 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -431,7 +431,10 @@ def test_sqlite_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ - print(sys.version_info, platform.system()) + print( + sys.version_info, platform.system(), + sys.version_info[0:2] != (3, 10) and platform.system() != "Darwin", + ) n = 10 # number of samples - note: we'll actually double it vector_length = 8 runner = CliRunner() From efad816c893f652ab4f5e83c8eb07c995ec36ab7 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:11:17 -0400 Subject: [PATCH 50/58] removing print Signed-off-by: Francisco Javier Arceo --- .../tests/unit/online_store/test_online_retrieval.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index c1a2dd86b07..13b220fbb97 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -424,17 +424,13 @@ def test_online_to_df(): @pytest.mark.skipif( - sys.version_info[0:2] != (3, 10) and platform.system() != "Darwin", + sys.version_info[0:2] != (3, 10) or platform.system() != "Darwin", reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode. """ - print( - sys.version_info, platform.system(), - sys.version_info[0:2] != (3, 10) and platform.system() != "Darwin", - ) n = 10 # number of samples - note: we'll actually double it vector_length = 8 runner = CliRunner() @@ -522,11 +518,10 @@ def test_sqlite_get_online_documents() -> None: @pytest.mark.skipif( - sys.version_info[0:2] != (3, 10) and platform.system() != "Darwin", + sys.version_info[0:2] != (3, 10) or platform.system() != "Darwin", reason="Only works on Python 3.10 and MacOS", ) def test_sqlite_vec_import() -> None: - print(sys.version_info, platform.system()) db = sqlite3.connect(":memory:") db.enable_load_extension(True) sqlite_vec.load(db) From 436b89aa94696816fbc6c994568ba539fe0f1d49 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:22:24 -0400 Subject: [PATCH 51/58] adding check in sqlite Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 8733a760845..491a8f29802 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -13,6 +13,7 @@ # limitations under the License. import itertools import os +import sys import sqlite3 import struct from datetime import datetime @@ -83,8 +84,9 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - self._conn.enable_load_extension(True) # type: ignore - sqlite_vec.load(self._conn) + if sys.version_info[0] == 3 and sys.version_info < 11: + self._conn.enable_load_extension(True) # type: ignore + sqlite_vec.load(self._conn) return self._conn From 263ec43bc06b8ef23bd79a639fb55e0b1a519454 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:27:27 -0400 Subject: [PATCH 52/58] missed an = Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 491a8f29802..81aab98405f 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -13,9 +13,9 @@ # limitations under the License. import itertools import os -import sys import sqlite3 import struct +import sys from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union @@ -84,7 +84,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if sys.version_info[0] == 3 and sys.version_info < 11: + if sys.version_info.major == 3 and sys.version_info.minor < 11: self._conn.enable_load_extension(True) # type: ignore sqlite_vec.load(self._conn) From 908b7f38d37e02e29dcc5fce3e681e6590875812 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:35:43 -0400 Subject: [PATCH 53/58] still running on 3.11 Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 81aab98405f..ec9e958f30d 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -84,7 +84,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if sys.version_info.major == 3 and sys.version_info.minor < 11: + if sys.version_info.major[0:2] != (3, 10): self._conn.enable_load_extension(True) # type: ignore sqlite_vec.load(self._conn) From 65952dbf64e4fd6d9fb84d7ada2e85a65e51a2c4 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:40:18 -0400 Subject: [PATCH 54/58] typo Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index ec9e958f30d..3304649f404 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -84,7 +84,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if sys.version_info.major[0:2] != (3, 10): + if sys.version_info[0:2] != (3, 10): self._conn.enable_load_extension(True) # type: ignore sqlite_vec.load(self._conn) From 39493b762442adb50c5dc57b9f96368d76b22e08 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:50:53 -0400 Subject: [PATCH 55/58] fix Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 3304649f404..c90c5c6411c 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -84,7 +84,7 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) - if sys.version_info[0:2] != (3, 10): + if sys.version_info[0:2] == (3, 10): self._conn.enable_load_extension(True) # type: ignore sqlite_vec.load(self._conn) From 834e4f6daca02e7e830b1a318e14893717a6bfd4 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 9 Jun 2024 22:58:55 -0400 Subject: [PATCH 56/58] fix Signed-off-by: Francisco Javier Arceo --- sdk/python/feast/infra/online_stores/sqlite.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index c90c5c6411c..41af14aaf16 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -481,8 +481,9 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: ) def update(self): - self.conn.enable_load_extension(True) - sqlite_vec.load(self.conn) + if sys.version_info[0:2] == (3, 10): + self.conn.enable_load_extension(True) + sqlite_vec.load(self.conn) self.conn.execute( f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) From 1c14c978dd5fd4ba81ba30aca0f2195537ed03bb Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 10 Jun 2024 06:26:53 -0400 Subject: [PATCH 57/58] updated setup and docs Signed-off-by: Francisco Javier Arceo --- docs/reference/alpha-vector-database.md | 6 +++++- setup.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index d3861404314..b9ce7f408a0 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -121,5 +121,9 @@ If you are using `pyenv` to manage your Python versions, you can install the SQL PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" \ CPPFLAGS="-I/opt/homebrew/opt/sqlite/include" \ - pyenv install + pyenv install 3.10.14 +``` +And you can the Feast install package via: +```bash +pip install feast[sqlite_vec] ``` \ No newline at end of file diff --git a/setup.py b/setup.py index e691be05172..c940ad2c237 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", - "sqlite-vec==v0.0.1-alpha.10", "SQLAlchemy[mypy]>1", "tabulate>=0.8.0,<1", "tenacity>=7,<9", @@ -97,6 +96,9 @@ "pyspark>=3.0.0,<4", ] +SQLITE_REQUIRED = [ + "-vec==v0.0.1-alpha.10", +] TRINO_REQUIRED = ["trino>=0.305.0,<0.400.0", "regex"] POSTGRES_REQUIRED = [ @@ -215,6 +217,7 @@ + DUCKDB_REQUIRED + DELTA_REQUIRED + ELASTICSEARCH_REQUIRED + + SQLITE_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -382,6 +385,7 @@ def run(self): "ikv": IKV_REQUIRED, "delta": DELTA_REQUIRED, "elasticsearch": ELASTICSEARCH_REQUIRED, + "sqlite_vec": SQLITE_REQUIRED, }, include_package_data=True, license="Apache", From ec60ea339cf0a850ed850aa337e0d0d02564c0ff Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 10 Jun 2024 06:27:56 -0400 Subject: [PATCH 58/58] renamed things Signed-off-by: Francisco Javier Arceo --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index c940ad2c237..9b3d0e55e62 100644 --- a/setup.py +++ b/setup.py @@ -96,8 +96,8 @@ "pyspark>=3.0.0,<4", ] -SQLITE_REQUIRED = [ - "-vec==v0.0.1-alpha.10", +SQLITE_VEC_REQUIRED = [ + "sqlite-vec==v0.0.1-alpha.10", ] TRINO_REQUIRED = ["trino>=0.305.0,<0.400.0", "regex"] @@ -217,7 +217,7 @@ + DUCKDB_REQUIRED + DELTA_REQUIRED + ELASTICSEARCH_REQUIRED - + SQLITE_REQUIRED + + SQLITE_VEC_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -385,7 +385,7 @@ def run(self): "ikv": IKV_REQUIRED, "delta": DELTA_REQUIRED, "elasticsearch": ELASTICSEARCH_REQUIRED, - "sqlite_vec": SQLITE_REQUIRED, + "sqlite_vec": SQLITE_VEC_REQUIRED, }, include_package_data=True, license="Apache",