From 0d91021ca07f5e8a911e8da5b355afcb20176c4d Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 2 Sep 2022 21:30:13 -0700 Subject: [PATCH 1/3] fix: Fix push API to respect feature view's already inferred entity types Signed-off-by: Danny Chiao --- Makefile | 6 +----- docs/reference/data-sources/push.md | 2 -- sdk/python/feast/feature_store.py | 7 +------ sdk/python/feast/infra/passthrough_provider.py | 6 ++++-- sdk/python/feast/infra/provider.py | 2 -- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 990604da575..ef90cc393cb 100644 --- a/Makefile +++ b/Makefile @@ -356,15 +356,11 @@ lint-go: compile-protos-go compile-go-lib # Docker -build-docker: build-ci-docker build-feature-server-python-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker +build-docker: build-feature-server-python-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) -# TODO(adchia): consider removing. This doesn't run successfully right now -build-ci-docker: - docker buildx build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile --load . - push-feature-server-python-docker: docker push $(REGISTRY)/feature-server:$$VERSION diff --git a/docs/reference/data-sources/push.md b/docs/reference/data-sources/push.md index a585de5688d..6dad690c16a 100644 --- a/docs/reference/data-sources/push.md +++ b/docs/reference/data-sources/push.md @@ -1,7 +1,5 @@ # Push source -**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. - ## Description Push sources allow feature values to be pushed to the online store and offline store in real time. This allows fresh feature values to be made available to applications. Push sources supercede the diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 23600e7c64f..9350220c21c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1478,13 +1478,8 @@ def write_to_online_store( feature_view = self.get_feature_view( feature_view_name, allow_registry_cache=allow_registry_cache ) - entities = [] - for entity_name in feature_view.entities: - entities.append( - self.get_entity(entity_name, allow_registry_cache=allow_registry_cache) - ) provider = self._get_provider() - provider.ingest_df(feature_view, entities, df) + provider.ingest_df(feature_view, df) @log_exceptions_and_usage def write_to_offline_store( diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index bb5cd38a835..28b10c12595 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -193,7 +193,6 @@ def online_read( def ingest_df( self, feature_view: FeatureView, - entities: List[Entity], df: pd.DataFrame, ): set_usage_attribute("provider", self.__class__.__name__) @@ -204,7 +203,10 @@ def ingest_df( table, feature_view.batch_source.field_mapping ) - join_keys = {entity.join_key: entity.value_type for entity in entities} + join_keys = { + entity.name: entity.dtype.to_value_type() + for entity in feature_view.entity_columns + } rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys) self.online_write_batch( diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 7d3c37e4c2e..82879b264af 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -123,7 +123,6 @@ def online_write_batch( def ingest_df( self, feature_view: FeatureView, - entities: List[Entity], df: pd.DataFrame, ): """ @@ -131,7 +130,6 @@ def ingest_df( Args: feature_view: The feature view to which the dataframe corresponds. - entities: The entities that are referenced by the dataframe. df: The dataframe to be persisted. """ pass From 546e82c25ff86c117996154d98715ede5eb2f385 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 2 Sep 2022 21:42:21 -0700 Subject: [PATCH 2/3] fix roadmap Signed-off-by: Danny Chiao --- README.md | 6 +++--- docs/roadmap.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b663533710b..c8adfa5f22c 100644 --- a/README.md +++ b/README.md @@ -185,8 +185,8 @@ The list below contains the functionality that contributors are planning to deve * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [x] [Push based streaming data ingestion to online store (Alpha)](https://docs.feast.dev/reference/data-sources/push) - * [x] [Push based streaming data ingestion to offline store (Alpha)](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to online store](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to offline store](https://docs.feast.dev/reference/data-sources/push) * **Deployments** * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [x] Kubernetes (See [guide](https://docs.feast.dev/how-to-guides/running-feast-in-production#4.3.-java-based-feature-server-deployed-on-kubernetes)) @@ -202,7 +202,7 @@ The list below contains the functionality that contributors are planning to deve * [x] Model-centric feature tracking (feature services) * [x] Amundsen integration (see [Feast extractor](https://github.com/amundsen-io/amundsen/blob/main/databuilder/databuilder/extractor/feast_extractor.py)) * [x] DataHub integration (see [DataHub Feast docs](https://datahubproject.io/docs/generated/ingestion/sources/feast/)) - * [x] Feast Web UI (Alpha release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) + * [x] Feast Web UI (Beta release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) ## 🎓 Important Resources diff --git a/docs/roadmap.md b/docs/roadmap.md index dc1d9ae1ab8..32d0917ff2d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -43,8 +43,8 @@ The list below contains the functionality that contributors are planning to deve * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [x] [Push based streaming data ingestion to online store (Alpha)](https://docs.feast.dev/reference/data-sources/push) - * [x] [Push based streaming data ingestion to offline store (Alpha)](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to online store](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to offline store](https://docs.feast.dev/reference/data-sources/push) * **Deployments** * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [x] Kubernetes (See [guide](https://docs.feast.dev/how-to-guides/running-feast-in-production#4.3.-java-based-feature-server-deployed-on-kubernetes)) @@ -60,4 +60,4 @@ The list below contains the functionality that contributors are planning to deve * [x] Model-centric feature tracking (feature services) * [x] Amundsen integration (see [Feast extractor](https://github.com/amundsen-io/amundsen/blob/main/databuilder/databuilder/extractor/feast_extractor.py)) * [x] DataHub integration (see [DataHub Feast docs](https://datahubproject.io/docs/generated/ingestion/sources/feast/)) - * [x] Feast Web UI (Alpha release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) + * [x] Feast Web UI (Beta release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) From 9ff27940a6876f42ea3cb4378325c360a09ea8f5 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 2 Sep 2022 21:45:21 -0700 Subject: [PATCH 3/3] fix helm Signed-off-by: Danny Chiao --- docs/roadmap.md | 3 ++- infra/charts/feast/README.md | 4 ++-- infra/charts/feast/README.md.gotmpl | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 32d0917ff2d..30f4317054b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -51,7 +51,8 @@ The list below contains the functionality that contributors are planning to deve * **Feature Serving** * [x] Python Client * [x] [Python feature server](https://docs.feast.dev/reference/feature-servers/python-feature-server) - * [x] [Go feature server](https://docs.feast.dev/reference/feature-servers/go-feature-server) + * [x] [Java feature server (alpha)](https://github.com/feast-dev/feast/blob/master/infra/charts/feast/README.md) + * [x] [Go feature server (alpha)](https://docs.feast.dev/reference/feature-servers/go-feature-server) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * **Feature Discovery and Governance** diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 2dd8c62250d..484c19e6d8d 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -1,6 +1,6 @@ -# Feast Helm Charts +# Feast Java Helm Charts (alpha) -This repo contains Helm charts for Feast components that are being installed on Kubernetes: +This repo contains Helm charts for Feast Java components that are being installed on Kubernetes: * Feast (root chart): The complete Helm chart containing all Feast components and dependencies. Most users will use this chart, but can selectively enable/disable subcharts using the values.yaml file. * [Feature Server](charts/feature-server): High performant JVM-based implementation of feature server. * [Transformation Service](charts/transformation-service): Transformation server for calculating on-demand features diff --git a/infra/charts/feast/README.md.gotmpl b/infra/charts/feast/README.md.gotmpl index 007dc854531..362f921d7ef 100644 --- a/infra/charts/feast/README.md.gotmpl +++ b/infra/charts/feast/README.md.gotmpl @@ -1,6 +1,6 @@ -# Feast Helm Charts +# Feast Java Helm Charts (alpha) -This repo contains Helm charts for Feast components that are being installed on Kubernetes: +This repo contains Helm charts for Feast Java components that are being installed on Kubernetes: * Feast (root chart): The complete Helm chart containing all Feast components and dependencies. Most users will use this chart, but can selectively enable/disable subcharts using the values.yaml file. * [Feature Server](charts/feature-server): High performant JVM-based implementation of feature server. * [Transformation Service](charts/transformation-service): Transformation server for calculating on-demand features