diff --git a/ui/package.json b/ui/package.json index 94bc5f6fc6a..0a0f129389c 100644 --- a/ui/package.json +++ b/ui/package.json @@ -37,6 +37,7 @@ "query-string": "^7.1.1", "react-app-polyfill": "^3.0.0", "react-code-blocks": "^0.1.6", + "react-markdown": "^10.1.0", "react-query": "^3.39.3", "react-router-dom": "^6.28.0", "reactflow": "^11.11.4", diff --git a/ui/public/docs/reference/alpha-vector-database.md b/ui/public/docs/reference/alpha-vector-database.md new file mode 100644 index 00000000000..861c3fcb114 --- /dev/null +++ b/ui/public/docs/reference/alpha-vector-database.md @@ -0,0 +1,225 @@ +# [Alpha] Vector Database +**Warning**: This is an _experimental_ feature. To our knowledge, this is stable, but there are still rough edges in the experience. Contributions are welcome! + +## Overview +Vector database allows user to store and retrieve embeddings. Feast provides general APIs to store and retrieve embeddings. + +## Integration +Below are supported vector databases and implemented features: + +| Vector Database | Retrieval | Indexing | V2 Support* | Online Read | +|-----------------|-----------|----------|-------------|-------------| +| Pgvector | [x] | [ ] | [] | [] | +| Elasticsearch | [x] | [x] | [] | [] | +| Milvus | [x] | [x] | [x] | [x] | +| Faiss | [ ] | [ ] | [] | [] | +| SQLite | [x] | [ ] | [x] | [x] | +| Qdrant | [x] | [x] | [] | [] | + +*Note: V2 Support means the SDK supports retrieval of features along with vector embeddings from vector similarity search. + +Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. + +{% hint style="danger" %} +We will be deprecating the `retrieve_online_documents` method in the SDK in the future. +We recommend using the `retrieve_online_documents_v2` method instead, which offers easier vector index configuration +directly in the Feature View and the ability to retrieve standard features alongside your vector embeddings for richer context injection. + +Long term we will collapse the two methods into one, but for now, we recommend using the `retrieve_online_documents_v2` method. +Beyond that, we will then have `retrieve_online_documents` and `retrieve_online_documents_v2` simply point to `get_online_features` for +backwards compatibility and the adopt industry standard naming conventions. +{% endhint %} + +**Note**: Milvus and SQLite implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. + +## Examples + +- See the v0 [Rag Demo](https://github.com/feast-dev/feast-workshop/blob/rag/module_4_rag) for an example on how to use vector database using the `retrieve_online_documents` method (planning migration and deprecation (planning migration and deprecation). +- See the v1 [Milvus Quickstart](../../examples/rag/milvus-quickstart.ipynb) for a quickstart guide on how to use Feast with Milvus using the `retrieve_online_documents_v2` method. + +### **Prepare offline embedding dataset** +Run the following commands to prepare the embedding dataset: +```shell +python pull_states.py +python batch_score_documents.py +``` +The output will be stored in `data/city_wikipedia_summaries.csv.` + +### **Initialize Feast feature store and materialize the data to the online store** +Use the feature_store.yaml file to initialize the feature store. This will use the data as offline store, and Milvus as online store. + +```yaml +project: local_rag +provider: local +registry: data/registry.db +online_store: + type: milvus + path: data/online_store.db + vector_enabled: true + embedding_dim: 384 + index_type: "IVF_FLAT" + + +offline_store: + type: file +entity_key_serialization_version: 3 +# By default, no_auth for authentication and authorization, other possible values kubernetes and oidc. Refer the documentation for more details. +auth: + type: no_auth +``` +Run the following command in terminal to apply the feature store configuration: + +```shell +feast apply +``` + +Note that when you run `feast apply` you are going to apply the following Feature View that we will use for retrieval later: + +```python +document_embeddings = FeatureView( + name="embedded_documents", + entities=[item, author], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + # Look how easy it is to enable RAG! + vector_index=True, + vector_search_metric="COSINE", + ), + Field(name="item_id", dtype=Int64), + Field(name="author_id", dtype=String), + Field(name="created_timestamp", dtype=UnixTimestamp), + Field(name="sentence_chunks", dtype=String), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), +) +``` + +Let's use the SDK to write a data frame of embeddings to the online store: +```python +store.write_to_online_store(feature_view_name='city_embeddings', df=df) +``` + +### **Prepare a query embedding** +During inference (e.g., during when a user submits a chat message) we need to embed the input text. This can be thought of as a feature transformation of the input data. In this example, we'll do this with a small Sentence Transformer from Hugging Face. + +```python +import torch +import torch.nn.functional as F +from feast import FeatureStore +from pymilvus import MilvusClient, DataType, FieldSchema +from transformers import AutoTokenizer, AutoModel +from example_repo import city_embeddings_feature_view, item + +TOKENIZER = "sentence-transformers/all-MiniLM-L6-v2" +MODEL = "sentence-transformers/all-MiniLM-L6-v2" + +def mean_pooling(model_output, attention_mask): + token_embeddings = model_output[ + 0 + ] # First element of model_output contains all token embeddings + input_mask_expanded = ( + attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + ) + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( + input_mask_expanded.sum(1), min=1e-9 + ) + +def run_model(sentences, tokenizer, model): + encoded_input = tokenizer( + sentences, padding=True, truncation=True, return_tensors="pt" + ) + # Compute token embeddings + with torch.no_grad(): + model_output = model(**encoded_input) + + sentence_embeddings = mean_pooling(model_output, encoded_input["attention_mask"]) + sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1) + return sentence_embeddings + +question = "Which city has the largest population in New York?" + +tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) +model = AutoModel.from_pretrained(MODEL) +query_embedding = run_model(question, tokenizer, model).detach().cpu().numpy().tolist()[0] +``` + +### **Retrieve the top K similar documents** +First create a feature store instance, and use the `retrieve_online_documents_v2` API to retrieve the top 5 similar documents to the specified query. + +```python +context_data = store.retrieve_online_documents_v2( + features=[ + "city_embeddings:vector", + "city_embeddings:item_id", + "city_embeddings:state", + "city_embeddings:sentence_chunks", + "city_embeddings:wiki_summary", + ], + query=query_embedding, + top_k=3, + distance_metric='COSINE', +).to_df() +``` +### **Generate the Response** +Let's assume we have a base prompt and a function that formats the retrieved documents called `format_documents` that we +can then use to generate the response with OpenAI's chat completion API. +```python +FULL_PROMPT = format_documents(rag_context_data, BASE_PROMPT) + +from openai import OpenAI + +client = OpenAI( + api_key=os.environ.get("OPENAI_API_KEY"), +) +response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": FULL_PROMPT}, + {"role": "user", "content": question} + ], +) + +# And this will print the content. Look at the examples/rag/milvus-quickstart.ipynb for an end-to-end example. +print('\n'.join([c.message.content for c in response.choices])) +``` + +### Configuration and Installation + +We offer [Milvus](https://milvus.io/), [PGVector](https://github.com/pgvector/pgvector), [SQLite](https://github.com/asg017/sqlite-vec), [Elasticsearch](https://www.elastic.co) and [Qdrant](https://qdrant.tech/) as Online Store options for Vector Databases. + +Milvus offers a convenient local implementation for vector similarity search. To use Milvus, you can install the Feast package with the Milvus extra. + +#### Installation with Milvus + +```bash +pip install feast[milvus] +``` +#### Installation with Elasticsearch + +```bash +pip install feast[elasticsearch] +``` + +#### Installation with Qdrant + +```bash +pip install feast[qdrant] +``` +#### Installation with SQLite + +If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command: +```bash +PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ + LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" \ + CPPFLAGS="-I/opt/homebrew/opt/sqlite/include" \ + pyenv install 3.10.14 +``` + +And you can the Feast install package via: +```bash +pip install feast[sqlite_vec] +``` diff --git a/ui/public/docs/reference/alpha-web-ui.md b/ui/public/docs/reference/alpha-web-ui.md new file mode 100644 index 00000000000..80c5b824c5a --- /dev/null +++ b/ui/public/docs/reference/alpha-web-ui.md @@ -0,0 +1,143 @@ +# \[Beta] Web UI + +**Warning**: This is an _experimental_ feature. To our knowledge, this is stable, but there are still rough edges in the experience. Contributions are welcome! + +## Overview + +The Feast Web UI allows users to explore their feature repository through a Web UI. It includes functionality such as: + +* Browsing Feast objects (feature views, entities, data sources, feature services, and saved datasets) and their relationships +* Searching and filtering for Feast objects by tags + +![Sample UI](../../ui/sample.png) + +## Usage + +There are several ways to use the Feast Web UI. + +### Feast CLI + +The easiest way to get started is to run the `feast ui` command within a feature repository: + +Output of `feast ui --help`: + +```bash +Usage: feast ui [OPTIONS] + +Shows the Feast UI over the current directory + +Options: +-h, --host TEXT Specify a host for the server [default: 0.0.0.0] +-p, --port INTEGER Specify a port for the server [default: 8888] +-r, --registry_ttl_sec INTEGER Number of seconds after which the registry is refreshed. Default is 5 seconds. +--help Show this message and exit. +``` + +This will spin up a Web UI on localhost which automatically refreshes its view of the registry every `registry_ttl_sec` + +### Importing as a module to integrate with an existing React App + +This is the recommended way to use Feast UI for teams maintaining their own internal UI for their deployment of Feast. + +Start with bootstrapping a React app with `create-react-app` + +``` +npx create-react-app your-feast-ui +``` + +Then, in your app folder, install Feast UI and optionally its peer dependencies. Assuming you use yarn + +``` +yarn add @feast-dev/feast-ui +# For custom UI using the Elastic UI Framework (optional): +yarn add @elastic/eui +# For general custom styling (optional): +yarn add @emotion/react +``` + +Edit `index.js` in the React app to use Feast UI. + +```js +import React from "react"; +import ReactDOM from "react-dom"; +import "./index.css"; + +import FeastUI from "@feast-dev/feast-ui"; +import "@feast-dev/feast-ui/dist/feast-ui.css"; + +ReactDOM.render( + + + , + document.getElementById("root") +); +``` + +When you start the React app, it will look for `projects-list.json` to find a list of your projects. The JSON should look something like this. + +```json +{ + "projects": [ + { + "name": "Credit Score Project", + "description": "Project for credit scoring team and associated models.", + "id": "credit_score_project", + "registryPath": "/registry.json" + } + ] +} +``` + +* **Note** - `registryPath` only supports a file location or a url. + +Then start the React App + +```bash +yarn start +``` + +#### Customization + +The advantage of importing Feast UI as a module is in the ease of customization. The `` component exposes a `feastUIConfigs` prop thorough which you can customize the UI. Currently it supports a few parameters. + +##### Fetching the Project List + +By default, the Feast UI fetches the project list from the app root path. You can use `projectListPromise` to provide a promise that overrides where it's fetched from. + +```jsx + { + return res.json(); + }) + }} +/> +``` + +##### Custom Tabs + +You can add custom tabs for any of the core Feast objects through the `tabsRegistry`. + +```jsx +const tabsRegistry = { + RegularFeatureViewCustomTabs: [ + { + label: "Custom Tab Demo", // Navigation Label for the tab + path: "demo-tab", // Subpath for the tab + Component: RFVDemoCustomTab, // a React Component + }, + ] +} + + +``` + +Examples of custom tabs can be found in the `ui/custom-tabs` folder. diff --git a/ui/public/docs/reference/batch-materialization/README.md b/ui/public/docs/reference/batch-materialization/README.md new file mode 100644 index 00000000000..a05d6d75e5d --- /dev/null +++ b/ui/public/docs/reference/batch-materialization/README.md @@ -0,0 +1,11 @@ +# Batch materialization + +Please see [Batch Materialization Engine](../../getting-started/components/batch-materialization-engine.md) for an explanation of batch materialization engines. + +{% page-ref page="snowflake.md" %} + +{% page-ref page="bytewax.md" %} + +{% page-ref page="lambda.md" %} + +{% page-ref page="spark.md" %} diff --git a/ui/public/docs/reference/batch-materialization/lambda.md b/ui/public/docs/reference/batch-materialization/lambda.md new file mode 100644 index 00000000000..126d07c4103 --- /dev/null +++ b/ui/public/docs/reference/batch-materialization/lambda.md @@ -0,0 +1,24 @@ +# AWS Lambda (alpha) + +## Description + +The AWS Lambda batch materialization engine is considered alpha status. It relies on the offline store to output feature values to S3 via `to_remote_storage`, and then loads them into the online store. + +See [LambdaMaterializationEngineConfig](https://rtd.feast.dev/en/master/index.html?highlight=LambdaMaterializationEngine#feast.infra.materialization.aws_lambda.lambda_engine.LambdaMaterializationEngineConfig) for configuration options. + +See also [Dockerfile](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/materialization/aws_lambda/Dockerfile) for a Dockerfile that can be used below with `materialization_image`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +... +offline_store: + type: snowflake.offline +... +batch_engine: + type: lambda + lambda_role: [your iam role] + materialization_image: [image uri of above Docker image] +``` +{% endcode %} diff --git a/ui/public/docs/reference/batch-materialization/snowflake.md b/ui/public/docs/reference/batch-materialization/snowflake.md new file mode 100644 index 00000000000..c2fa441d6d2 --- /dev/null +++ b/ui/public/docs/reference/batch-materialization/snowflake.md @@ -0,0 +1,28 @@ +# Snowflake + +## Description + +The [Snowflake](https://trial.snowflake.com) batch materialization engine provides a highly scalable and parallel execution engine using a Snowflake Warehouse for batch materializations operations (`materialize` and `materialize-incremental`) when using a `SnowflakeSource`. + +The engine requires no additional configuration other than for you to supply Snowflake's standard login and context details. The engine leverages custom (automatically deployed for you) Python UDFs to do the proper serialization of your offline store data to your online serving tables. + +When using all three options together, `snowflake.offline`, `snowflake.engine`, and `snowflake.online`, you get the most unique experience of unlimited scale and performance + governance and data security. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +... +offline_store: + type: snowflake.offline +... +batch_engine: + type: snowflake.engine + account: snowflake_deployment.us-east-1 + user: user_login + password: user_password + role: sysadmin + warehouse: demo_wh + database: FEAST +``` +{% endcode %} diff --git a/ui/public/docs/reference/batch-materialization/spark.md b/ui/public/docs/reference/batch-materialization/spark.md new file mode 100644 index 00000000000..27a1388c48e --- /dev/null +++ b/ui/public/docs/reference/batch-materialization/spark.md @@ -0,0 +1,55 @@ +# Spark (alpha) + +## Description + +The Spark batch materialization engine is considered alpha status. It relies on the offline store to output feature values to S3 via `to_remote_storage`, and then loads them into the online store. + +See [SparkMaterializationEngine](https://rtd.feast.dev/en/master/index.html?highlight=SparkMaterializationEngine#feast.infra.materialization.spark.spark_materialization_engine.SparkMaterializationEngineConfig) for configuration options. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +... +offline_store: + type: snowflake.offline +... +batch_engine: + type: spark.engine + partitions: [optional num partitions to use to write to online store] +``` +{% endcode %} + +## Example in Python + +{% code title="feature_store.py" %} +```python +from feast import FeatureStore, RepoConfig +from feast.repo_config import RegistryConfig +from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig +from feast.infra.offline_stores.contrib.spark_offline_store.spark import SparkOfflineStoreConfig + +repo_config = RepoConfig( + registry="s3://[YOUR_BUCKET]/feast-registry.db", + project="feast_repo", + provider="aws", + offline_store=SparkOfflineStoreConfig( + spark_conf={ + "spark.ui.enabled": "false", + "spark.eventLog.enabled": "false", + "spark.sql.catalogImplementation": "hive", + "spark.sql.parser.quotedRegexColumnNames": "true", + "spark.sql.session.timeZone": "UTC" + } + ), + batch_engine={ + "type": "spark.engine", + "partitions": 10 + }, + online_store=DynamoDBOnlineStoreConfig(region="us-west-1"), + entity_key_serialization_version=2 +) + +store = FeatureStore(config=repo_config) +``` +{% endcode %} \ No newline at end of file diff --git a/ui/public/docs/reference/beta-on-demand-feature-view.md b/ui/public/docs/reference/beta-on-demand-feature-view.md new file mode 100644 index 00000000000..c58eab1f40a --- /dev/null +++ b/ui/public/docs/reference/beta-on-demand-feature-view.md @@ -0,0 +1,281 @@ +# [Beta] On Demand Feature Views + +**Warning**: This is an experimental feature. While it is stable to our knowledge, there may still be rough edges in the experience. Contributions are welcome! + +## Overview + +On Demand Feature Views (ODFVs) allow data scientists to use existing features and request-time data to transform and +create new features. Users define transformation logic that is executed during both historical and online retrieval. +Additionally, ODFVs provide flexibility in applying transformations either during data ingestion (at write time) or +during feature retrieval (at read time), controlled via the `write_to_online_store` parameter. + +By setting `write_to_online_store=True`, transformations are applied during data ingestion, and the transformed +features are stored in the online store. This can improve online feature retrieval performance by reducing computation +during reads. Conversely, if `write_to_online_store=False` (the default if omitted), transformations are applied during +feature retrieval. + +### Why Use On Demand Feature Views? + +ODFVs enable data scientists to easily impact the online feature retrieval path. For example, a data scientist could: + +1. Call `get_historical_features` to generate a training dataset. +2. Iterate in a notebook and do your feature engineering using Pandas or native Python. +3. Copy transformation logic into ODFVs and commit to a development branch of the feature repository. +4. Verify with `get_historical_features` (on a small dataset) that the transformation gives the expected output over historical data. +5. Decide whether to apply the transformation on writes or on reads by setting the `write_to_online_store` parameter accordingly. +6. Verify with `get_online_features` on the development branch that the transformation correctly outputs online features. +7. Submit a pull request to the staging or production branches, impacting production traffic. + +## Transformation Modes + +When defining an ODFV, you can specify the transformation mode using the `mode` parameter. Feast supports the following modes: + +- **Pandas Mode (`mode="pandas"`)**: The transformation function takes a Pandas DataFrame as input and returns a Pandas DataFrame as output. This mode is useful for batch transformations over multiple rows. +- **Native Python Mode (`mode="python"`)**: The transformation function uses native Python and can operate on inputs as lists of values or as single dictionaries representing a singleton (single row). + +### Singleton Transformations in Native Python Mode + +Native Python mode supports transformations on singleton dictionaries by setting `singleton=True`. This allows you to +write transformation functions that operate on a single row at a time, making the code more intuitive and aligning with +how data scientists typically think about data transformations. + +## Example +See [https://github.com/feast-dev/on-demand-feature-views-demo](https://github.com/feast-dev/on-demand-feature-views-demo) for an example on how to use on demand feature views. + + +## Registering Transformations + +When defining an ODFV, you can control when the transformation is applied using the `write_to_online_store` parameter: + +- `write_to_online_store=True`: The transformation is applied during data ingestion (on write), and the transformed features are stored in the online store. +- `write_to_online_store=False` (default): The transformation is applied during feature retrieval (on read). + +### Examples + +#### Example 1: On Demand Transformation on Read Using Pandas Mode + +```python +from feast import Field, RequestSource, on_demand_feature_view +from feast.types import Float64, Int64 +import pandas as pd + +# Define a request data source for request-time features +input_request = RequestSource( + name="vals_to_add", + schema=[ + Field(name="val_to_add", dtype=Int64), + Field(name="val_to_add_2", dtype=Int64), + ], +) + +# Use input data and feature view features to create new features in Pandas mode +@on_demand_feature_view( + sources=[driver_hourly_stats_view, input_request], + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], + mode="pandas", +) +def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = features_df["conv_rate"] + features_df["val_to_add"] + df["conv_rate_plus_val2"] = features_df["conv_rate"] + features_df["val_to_add_2"] + return df +``` + +#### Example 2: On Demand Transformation on Read Using Native Python Mode (List Inputs) + +```python +from feast import Field, on_demand_feature_view +from feast.types import Float64 +from typing import Any, Dict + +# Use input data and feature view features to create new features in Native Python mode +@on_demand_feature_view( + sources=[driver_hourly_stats_view, input_request], + schema=[ + Field(name="conv_rate_plus_val1_python", dtype=Float64), + Field(name="conv_rate_plus_val2_python", dtype=Float64), + ], + mode="python", +) +def transformed_conv_rate_python(inputs: Dict[str, Any]) -> Dict[str, Any]: + output = { + "conv_rate_plus_val1_python": [ + conv_rate + val_to_add + for conv_rate, val_to_add in zip(inputs["conv_rate"], inputs["val_to_add"]) + ], + "conv_rate_plus_val2_python": [ + conv_rate + val_to_add + for conv_rate, val_to_add in zip( + inputs["conv_rate"], inputs["val_to_add_2"] + ) + ], + } + return output +``` + +#### **New** Example 3: On Demand Transformation on Read Using Native Python Mode (Singleton Input) + +```python +from feast import Field, on_demand_feature_view +from feast.types import Float64 +from typing import Any, Dict + +# Use input data and feature view features to create new features in Native Python mode with singleton input +@on_demand_feature_view( + sources=[driver_hourly_stats_view, input_request], + schema=[ + Field(name="conv_rate_plus_acc_singleton", dtype=Float64), + ], + mode="python", + singleton=True, +) +def transformed_conv_rate_singleton(inputs: Dict[str, Any]) -> Dict[str, Any]: + output = { + "conv_rate_plus_acc_singleton": inputs["conv_rate"] + inputs["acc_rate"] + } + return output +``` + +In this example, `inputs` is a dictionary representing a single row, and the transformation function returns a dictionary of transformed features for that single row. This approach is more intuitive and aligns with how data scientists typically process single data records. + +#### Example 4: On Demand Transformation on Write Using Pandas Mode + +```python +from feast import Field, on_demand_feature_view +from feast.types import Float64 +import pandas as pd + +# Existing Feature View +driver_hourly_stats_view = ... + +# Define an ODFV applying transformation during write time +@on_demand_feature_view( + sources=[driver_hourly_stats_view], + schema=[ + Field(name="conv_rate_adjusted", dtype=Float64), + ], + mode="pandas", + write_to_online_store=True, # Apply transformation during write time +) +def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_adjusted"] = features_df["conv_rate"] * 1.1 # Adjust conv_rate by 10% + return df +``` + +To ingest data with the new feature view, include all input features required for the transformations: + +```python +from feast import FeatureStore +import pandas as pd + +store = FeatureStore(repo_path=".") + +# Data to ingest +data = pd.DataFrame({ + "driver_id": [1001], + "event_timestamp": [pd.Timestamp.now()], + "conv_rate": [0.5], + "acc_rate": [0.8], + "avg_daily_trips": [10], +}) + +# Ingest data to the online store +store.push("driver_hourly_stats_view", data) +``` + +### Feature Retrieval + +{% hint style="info" %} +**Note**: The name of the on demand feature view is the function name (e.g., `transformed_conv_rate`). +{% endhint %} + +#### Offline Features + +Retrieve historical features by referencing individual features or using a feature service: + +```python +training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", + "transformed_conv_rate:conv_rate_plus_val2", + "transformed_conv_rate_singleton:conv_rate_plus_acc_singleton", + ], +).to_df() +``` + +#### Online Features + +Retrieve online features by referencing individual features or using a feature service: + +```python +entity_rows = [ + { + "driver_id": 1001, + "val_to_add": 1, + "val_to_add_2": 2, + } +] + +online_response = store.get_online_features( + entity_rows=entity_rows, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "transformed_conv_rate_python:conv_rate_plus_val1_python", + "transformed_conv_rate_python:conv_rate_plus_val2_python", + "transformed_conv_rate_singleton:conv_rate_plus_acc_singleton", + ], +).to_dict() +``` + +### Materializing Pre-transformed Data + +In some scenarios, you may have already transformed your data in batch (e.g., using Spark or another batch processing framework) and want to directly materialize the pre-transformed features without applying transformations during ingestion. Feast supports this through the `transform_on_write` parameter. + +When using `write_to_online_store=True` with On Demand Feature Views, you can set `transform_on_write=False` to skip transformations during the write operation. This is particularly useful for optimizing performance when working with large pre-transformed datasets. + +```python +from feast import FeatureStore +import pandas as pd + +store = FeatureStore(repo_path=".") + +# Pre-transformed data (transformations already applied) +pre_transformed_data = pd.DataFrame({ + "driver_id": [1001], + "event_timestamp": [pd.Timestamp.now()], + "conv_rate": [0.5], + # Pre-calculated values for the transformed features + "conv_rate_adjusted": [0.55], # Already contains the adjusted value +}) + +# Write to online store, skipping transformations +store.write_to_online_store( + feature_view_name="transformed_conv_rate", + df=pre_transformed_data, + transform_on_write=False # Skip transformation during write +) +``` + +This approach allows for a hybrid workflow where you can: +1. Transform data in batch using powerful distributed processing tools +2. Materialize the pre-transformed data without reapplying transformations +3. Still use the Feature Server to execute transformations during API calls when needed + +Even when features are materialized with transformations skipped (`transform_on_write=False`), the feature server can still apply transformations during API calls for any missing values or for features that require real-time computation. + +## CLI Commands +There are new CLI commands to manage on demand feature views: + +feast on-demand-feature-views list: Lists all registered on demand feature views after feast apply is run. +feast on-demand-feature-views describe [NAME]: Describes the definition of an on demand feature view. + + diff --git a/ui/public/docs/reference/codebase-structure.md b/ui/public/docs/reference/codebase-structure.md new file mode 100644 index 00000000000..7077e48fef3 --- /dev/null +++ b/ui/public/docs/reference/codebase-structure.md @@ -0,0 +1,136 @@ +# Codebase structure + +Let's examine the Feast codebase. +This analysis is accurate as of Feast 0.23. + +``` +$ tree -L 1 -d +. +├── docs +├── examples +├── go +├── infra +├── java +├── protos +├── sdk +└── ui +``` + +## Python SDK + +The Python SDK lives in `sdk/python/feast`. +The majority of Feast logic lives in these Python files: +* The core Feast objects ([entities](../getting-started/concepts/entity.md), [feature views](../getting-started/concepts/feature-view.md), [data sources](../getting-started/concepts/dataset.md), etc.) are defined in their respective Python files, such as `entity.py`, `feature_view.py`, and `data_source.py`. +* The `FeatureStore` class is defined in `feature_store.py` and the associated configuration object (the Python representation of the `feature_store.yaml` file) are defined in `repo_config.py`. +* The CLI and other core feature store logic are defined in `cli.py` and `repo_operations.py`. +* The type system that is used to manage conversion between Feast types and external typing systems is managed in `type_map.py`. +* The Python feature server (the server that is started through the `feast serve` command) is defined in `feature_server.py`. + +There are also several important submodules: +* `infra/` contains all the infrastructure components, such as the provider, offline store, online store, batch materialization engine, and registry. +* `dqm/` covers data quality monitoring, such as the dataset profiler. +* `diff/` covers the logic for determining how to apply infrastructure changes upon feature repo changes (e.g. the output of `feast plan` and `feast apply`). +* `embedded_go/` covers the Go feature server. +* `ui/` contains the embedded Web UI, to be launched on the `feast ui` command. + +Of these submodules, `infra/` is the most important. +It contains the interfaces for the [provider](getting-started/components/provider.md), [offline store](getting-started/components/offline-store.md), [online store](getting-started/components/online-store.md), [batch materialization engine](getting-started/components/batch-materialization-engine.md), and [registry](getting-started/components/registry.md), as well as all of their individual implementations. + +``` +$ tree --dirsfirst -L 1 infra +infra +├── contrib +├── feature_servers +├── materialization +├── offline_stores +├── online_stores +├── registry +├── transformation_servers +├── utils +├── __init__.py +├── aws.py +├── gcp.py +├── infra_object.py +├── key_encoding_utils.py +├── local.py +├── passthrough_provider.py +└── provider.py +``` + +The tests for the Python SDK are contained in `sdk/python/tests`. +For more details, see this [overview](../how-to-guides/adding-or-reusing-tests.md#test-suite-overview) of the test suite. + +### Example flow: `feast apply` + +Let's walk through how `feast apply` works by tracking its execution across the codebase. + +1. All CLI commands are in `cli.py`. + Most of these commands are backed by methods in `repo_operations.py`. + The `feast apply` command triggers `apply_total_command`, which then calls `apply_total` in `repo_operations.py`. +2. With a `FeatureStore` object (from `feature_store.py`) that is initialized based on the `feature_store.yaml` in the current working directory, `apply_total` first parses the feature repo with `parse_repo` and then calls either `FeatureStore.apply` or `FeatureStore._apply_diffs` to apply those changes to the feature store. +3. Let's examine `FeatureStore.apply`. + It splits the objects based on class (e.g. `Entity`, `FeatureView`, etc.) and then calls the appropriate registry method to apply or delete the object. + For example, it might call `self._registry.apply_entity` to apply an entity. + If the default file-based registry is used, this logic can be found in `infra/registry/registry.py`. +4. Then the feature store must update its cloud infrastructure (e.g. online store tables) to match the new feature repo, so it calls `Provider.update_infra`, which can be found in `infra/provider.py`. +5. Assuming the provider is a built-in provider (e.g. one of the local, GCP, or AWS providers), it will call `PassthroughProvider.update_infra` in `infra/passthrough_provider.py`. +6. This delegates to the online store and batch materialization engine. + For example, if the feature store is configured to use the Redis online store then the `update` method from `infra/online_stores/redis.py` will be called. + And if the local materialization engine is configured then the `update` method from `infra/materialization/local_engine.py` will be called. + +At this point, the `feast apply` command is complete. + +### Example flow: `feast materialize` + +Let's walk through how `feast materialize` works by tracking its execution across the codebase. + +1. The `feast materialize` command triggers `materialize_command` in `cli.py`, which then calls `FeatureStore.materialize` from `feature_store.py`. +2. This then calls `Provider.materialize_single_feature_view`, which can be found in `infra/provider.py`. +3. As with `feast apply`, the provider is most likely backed by the passthrough provider, in which case `PassthroughProvider.materialize_single_feature_view` will be called. +4. This delegates to the underlying batch materialization engine. + Assuming that the local engine has been configured, `LocalMaterializationEngine.materialize` from `infra/materialization/local_engine.py` will be called. +5. Since materialization involves reading features from the offline store and writing them to the online store, the local engine will delegate to both the offline store and online store. + Specifically, it will call `OfflineStore.pull_latest_from_table_or_query` and `OnlineStore.online_write_batch`. + These two calls will be routed to the offline store and online store that have been configured. + +### Example flow: `get_historical_features` + +Let's walk through how `get_historical_features` works by tracking its execution across the codebase. + +1. We start with `FeatureStore.get_historical_features` in `feature_store.py`. + This method does some internal preparation, and then delegates the actual execution to the underlying provider by calling `Provider.get_historical_features`, which can be found in `infra/provider.py`. +2. As with `feast apply`, the provider is most likely backed by the passthrough provider, in which case `PassthroughProvider.get_historical_features` will be called. +3. That call simply delegates to `OfflineStore.get_historical_features`. + So if the feature store is configured to use Snowflake as the offline store, `SnowflakeOfflineStore.get_historical_features` will be executed. + +## Java SDK + +The `java/` directory contains the Java serving component. +See [here](https://github.com/feast-dev/feast/blob/master/java/CONTRIBUTING.md) for more details on how the repo is structured. + +## Go feature server + +The `go/` directory contains the Go feature server. +Most of the files here have logic to help with reading features from the online store. +Within `go/`, the `internal/feast/` directory contains most of the core logic: +* `onlineserving/` covers the core serving logic. +* `model/` contains the implementations of the Feast objects (entity, feature view, etc.). + * For example, `entity.go` is the Go equivalent of `entity.py`. It contains a very simple Go implementation of the entity object. +* `registry/` covers the registry. + * Currently only the file-based registry supported (the sql-based registry is unsupported). Additionally, the file-based registry only supports a file-based registry store, not the GCS or S3 registry stores. +* `onlinestore/` covers the online stores (currently only Redis and SQLite are supported). + +## Protobufs + +Feast uses [protobuf](https://github.com/protocolbuffers/protobuf) to store serialized versions of the core Feast objects. +The protobuf definitions are stored in `protos/feast`. + +The [registry](../getting-started/concepts/registry.md) consists of the serialized representations of the Feast objects. + +Typically, changes being made to the Feast objects require changes to their corresponding protobuf representations. +The usual best practices for making changes to protobufs should be followed ensure backwards and forwards compatibility. + +## Web UI + +The `ui/` directory contains the Web UI. +See [here](https://github.com/feast-dev/feast/blob/master/ui/CONTRIBUTING.md) for more details on the structure of the Web UI. diff --git a/ui/public/docs/reference/compute-engine/README.md b/ui/public/docs/reference/compute-engine/README.md new file mode 100644 index 00000000000..75f29890046 --- /dev/null +++ b/ui/public/docs/reference/compute-engine/README.md @@ -0,0 +1,120 @@ +# 🧠 ComputeEngine (WIP) + +The `ComputeEngine` is Feast’s pluggable abstraction for executing feature pipelines — including transformations, aggregations, joins, and materializations/get_historical_features — on a backend of your choice (e.g., Spark, PyArrow, Pandas, Ray). + +It powers both: + +- `materialize()` – for batch and stream generation of features to offline/online stores +- `get_historical_features()` – for point-in-time correct training dataset retrieval + +This system builds and executes DAGs (Directed Acyclic Graphs) of typed operations, enabling modular and scalable workflows. + +--- + +## 🧠 Core Concepts + +| Component | Description | +|--------------------|----------------------------------------------------------------------| +| `ComputeEngine` | Interface for executing materialization and retrieval tasks | +| `FeatureBuilder` | Constructs a DAG from Feature View definition for a specific backend | +| `DAGNode` | Represents a logical operation (read, aggregate, join, etc.) | +| `ExecutionPlan` | Executes nodes in dependency order and stores intermediate outputs | +| `ExecutionContext` | Holds config, registry, stores, entity data, and node outputs | + +--- + +## ✨ Available Engines + +### 🔥 SparkComputeEngine + +- Distributed DAG execution via Apache Spark +- Supports point-in-time joins and large-scale materialization +- Integrates with `SparkOfflineStore` and `SparkMaterializationJob` + +### 🧪 LocalComputeEngine + +- Runs on Arrow + Specified backend (e.g., Pandas, Polars) +- Designed for local dev, testing, or lightweight feature generation +- Supports `LocalMaterializationJob` and `LocalHistoricalRetrievalJob` + +--- + +## 🛠️ Feature Builder Flow +```markdown +SourceReadNode + | + v +JoinNode (Only for get_historical_features with entity df) + | + v +FilterNode (Always included; applies TTL or user-defined filters) + | + v +AggregationNode (If aggregations are defined in FeatureView) + | + v +DeduplicationNode (If no aggregation is defined for get_historical_features) + | + v +TransformationNode (If feature_transformation is defined) + | + v +ValidationNode (If enable_validation = True) + | + v +Output + ├──> RetrievalOutput (For get_historical_features) + └──> OnlineStoreWrite / OfflineStoreWrite (For materialize) +``` + +Each step is implemented as a `DAGNode`. An `ExecutionPlan` executes these nodes in topological order, caching `DAGValue` outputs. + +--- + +## 🧩 Implementing a Custom Compute Engine + +To create your own compute engine: + +1. **Implement the interface** + +```python +from feast.infra.compute_engines.base import ComputeEngine +from feast.infra.materialization.batch_materialization_engine import MaterializationTask, MaterializationJob +from feast.infra.compute_engines.tasks import HistoricalRetrievalTask +class MyComputeEngine(ComputeEngine): + def materialize(self, task: MaterializationTask) -> MaterializationJob: + ... + + def get_historical_features(self, task: HistoricalRetrievalTask) -> RetrievalJob: + ... +``` + +2. Create a FeatureBuilder +```python +from feast.infra.compute_engines.feature_builder import FeatureBuilder + +class CustomFeatureBuilder(FeatureBuilder): + def build_source_node(self): ... + def build_aggregation_node(self, input_node): ... + def build_join_node(self, input_node): ... + def build_filter_node(self, input_node): + def build_dedup_node(self, input_node): + def build_transformation_node(self, input_node): ... + def build_output_nodes(self, input_node): ... +``` + +3. Define DAGNode subclasses + * ReadNode, AggregationNode, JoinNode, WriteNode, etc. + * Each DAGNode.execute(context) -> DAGValue + +4. Return an ExecutionPlan + * ExecutionPlan stores DAG nodes in topological order + * Automatically handles intermediate value caching + +## 🚧 Roadmap +- [x] Modular, backend-agnostic DAG execution framework +- [x] Spark engine with native support for materialization + PIT joins +- [ ] PyArrow + Pandas engine for local compute +- [ ] Native multi-feature-view DAG optimization +- [ ] DAG validation, metrics, and debug output +- [ ] Scalable distributed backend via Ray or Polars diff --git a/ui/public/docs/reference/data-sources/README.md b/ui/public/docs/reference/data-sources/README.md new file mode 100644 index 00000000000..151a948d0af --- /dev/null +++ b/ui/public/docs/reference/data-sources/README.md @@ -0,0 +1,59 @@ +# Data sources + +Please see [Data Source](../../getting-started/concepts/data-ingestion.md) for a conceptual explanation of data sources. + +{% content-ref url="overview.md" %} +[overview.md](overview.md) +{% endcontent-ref %} + +{% content-ref url="file.md" %} +[file.md](file.md) +{% endcontent-ref %} + +{% content-ref url="snowflake.md" %} +[snowflake.md](snowflake.md) +{% endcontent-ref %} + +{% content-ref url="bigquery.md" %} +[bigquery.md](bigquery.md) +{% endcontent-ref %} + +{% content-ref url="redshift.md" %} +[redshift.md](redshift.md) +{% endcontent-ref %} + +{% content-ref url="push.md" %} +[push.md](push.md) +{% endcontent-ref %} + +{% content-ref url="kafka.md" %} +[kafka.md](kafka.md) +{% endcontent-ref %} + +{% content-ref url="kinesis.md" %} +[kinesis.md](kinesis.md) +{% endcontent-ref %} + +{% content-ref url="couchbase.md" %} +[couchbase.md](couchbase.md) +{% endcontent-ref %} + +{% content-ref url="spark.md" %} +[spark.md](spark.md) +{% endcontent-ref %} + +{% content-ref url="postgres.md" %} +[postgres.md](postgres.md) +{% endcontent-ref %} + +{% content-ref url="trino.md" %} +[trino.md](trino.md) +{% endcontent-ref %} + +{% content-ref url="mssql.md" %} +[mssql.md](mssql.md) +{% endcontent-ref %} + +{% content-ref url="clickhouse.md" %} +[clickhouse.md](clickhouse.md) +{% endcontent-ref %} diff --git a/ui/public/docs/reference/data-sources/bigquery.md b/ui/public/docs/reference/data-sources/bigquery.md new file mode 100644 index 00000000000..51c9b19ecdb --- /dev/null +++ b/ui/public/docs/reference/data-sources/bigquery.md @@ -0,0 +1,37 @@ +# BigQuery source + +## Description + +BigQuery data sources are BigQuery tables or views. +These can be specified either by a table reference or a SQL query. +However, no performance guarantees can be provided for SQL query-based sources, so table references are recommended. + +## Examples + +Using a table reference: + +```python +from feast import BigQuerySource + +my_bigquery_source = BigQuerySource( + table_ref="gcp_project:bq_dataset.bq_table", +) +``` + +Using a query: + +```python +from feast import BigQuerySource + +BigQuerySource( + query="SELECT timestamp as ts, created, f1, f2 " + "FROM `my_project.my_dataset.my_features`", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/latest/index.html#feast.infra.offline_stores.bigquery_source.BigQuerySource). + +## Supported Types + +BigQuery data sources support all eight primitive types and their corresponding array types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/clickhouse.md b/ui/public/docs/reference/data-sources/clickhouse.md new file mode 100644 index 00000000000..7630d5dd14a --- /dev/null +++ b/ui/public/docs/reference/data-sources/clickhouse.md @@ -0,0 +1,36 @@ +# Clickhouse source (contrib) + +## Description + +Clickhouse data sources are Clickhouse tables or views. +These can be specified either by a table reference or a SQL query. + +## Disclaimer + +The Clickhouse data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a Clickhouse source: + +```python +from feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source import ( + ClickhouseSource, +) + +driver_stats_source = ClickhouseSource( + name="feast_driver_hourly_stats", + query="SELECT * FROM feast_driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source.ClickhouseSource). + +## Supported Types + +Clickhouse data sources support all eight primitive types and their corresponding array types. +The support for Clickhouse Decimal type is achieved by converting it to double. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/couchbase.md b/ui/public/docs/reference/data-sources/couchbase.md new file mode 100644 index 00000000000..596e33cf50d --- /dev/null +++ b/ui/public/docs/reference/data-sources/couchbase.md @@ -0,0 +1,37 @@ +# Couchbase Columnar source (contrib) + +## Description + +Couchbase Columnar data sources are [Couchbase Capella Columnar](https://docs.couchbase.com/columnar/intro/intro.html) collections that can be used as a source for feature data. **Note that Couchbase Columnar is available through [Couchbase Capella](https://cloud.couchbase.com/).** + +## Disclaimer + +The Couchbase Columnar data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a Couchbase Columnar source: + +```python +from feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source import ( + CouchbaseColumnarSource, +) + +driver_stats_source = CouchbaseColumnarSource( + name="driver_hourly_stats_source", + query="SELECT * FROM Default.Default.`feast_driver_hourly_stats`", + database="Default", + scope="Default", + collection="feast_driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source.CouchbaseColumnarSource). + +## Supported Types + +Couchbase Capella Columnar data sources support `BOOLEAN`, `STRING`, `BIGINT`, and `DOUBLE` primitive types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/file.md b/ui/public/docs/reference/data-sources/file.md new file mode 100644 index 00000000000..d3fd09deca6 --- /dev/null +++ b/ui/public/docs/reference/data-sources/file.md @@ -0,0 +1,25 @@ +# File source + +## Description + +File data sources are files on disk or on S3. +Currently only Parquet and Delta formats are supported. + +## Example + +```python +from feast import FileSource +from feast.data_format import ParquetFormat + +parquet_file_source = FileSource( + file_format=ParquetFormat(), + path="file:///feast/customer.parquet", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/latest/index.html#feast.infra.offline_stores.file_source.FileSource). + +## Supported Types + +File data sources support all eight primitive types and their corresponding array types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/kafka.md b/ui/public/docs/reference/data-sources/kafka.md new file mode 100644 index 00000000000..8794c7a1e81 --- /dev/null +++ b/ui/public/docs/reference/data-sources/kafka.md @@ -0,0 +1,75 @@ +# Kafka source + +**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. + +## Description + +Kafka sources allow users to register Kafka streams as data sources. Feast currently does not launch or monitor jobs to ingest data from Kafka. Users are responsible for launching and monitoring their own ingestion jobs, which should write feature values to the online store through [FeatureStore.write_to_online_store](https://rtd.feast.dev/en/latest/index.html#feast.feature_store.FeatureStore.write_to_online_store). An example of how to launch such a job with Spark can be found [here](https://github.com/feast-dev/feast/tree/master/sdk/python/feast/infra/contrib). Feast also provides functionality to write to the offline store using the `write_to_offline_store` functionality. + +Kafka sources must have a batch source specified. The batch source will be used for retrieving historical features. Thus users are also responsible for writing data from their Kafka streams to a batch data source such as a data warehouse table. When using a Kafka source as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the feature view definition explicitly. + +## Stream sources +Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: + +1. Raw events come in (stream 1) +2. Streaming transformations applied (e.g. generating features like `last_N_purchased_categories`) (stream 2) +3. Write stream 2 values to an offline store as a historical log for training (optional) +4. Write stream 2 values to an online store for low latency feature serving +5. Periodically materialize feature values from the offline store into the online store for decreased training-serving skew and improved model performance + +## Example +### Defining a Kafka source +Note that the Kafka source has a batch source. +```python +from datetime import timedelta + +from feast import Field, FileSource, KafkaSource, stream_feature_view +from feast.data_format import JsonFormat +from feast.types import Float32 + +driver_stats_batch_source = FileSource( + name="driver_stats_source", + path="data/driver_stats.parquet", + timestamp_field="event_timestamp", +) + +driver_stats_stream_source = KafkaSource( + name="driver_stats_stream", + kafka_bootstrap_servers="localhost:9092", + topic="drivers", + timestamp_field="event_timestamp", + batch_source=driver_stats_batch_source, + message_format=JsonFormat( + schema_json="driver_id integer, event_timestamp timestamp, conv_rate double, acc_rate double, created timestamp" + ), + watermark_delay_threshold=timedelta(minutes=5), +) +``` + +### Using the Kafka source in a stream feature view +The Kafka source can be used in a stream feature view. +```python +@stream_feature_view( + entities=[driver], + ttl=timedelta(seconds=8640000000), + mode="spark", + schema=[ + Field(name="conv_percentage", dtype=Float32), + Field(name="acc_percentage", dtype=Float32), + ], + timestamp_field="event_timestamp", + online=True, + source=driver_stats_stream_source, +) +def driver_hourly_stats_stream(df: DataFrame): + from pyspark.sql.functions import col + + return ( + df.withColumn("conv_percentage", col("conv_rate") * 100.0) + .withColumn("acc_percentage", col("acc_rate") * 100.0) + .drop("conv_rate", "acc_rate") + ) +``` + +### Ingesting data +See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. diff --git a/ui/public/docs/reference/data-sources/kinesis.md b/ui/public/docs/reference/data-sources/kinesis.md new file mode 100644 index 00000000000..f2adadfec03 --- /dev/null +++ b/ui/public/docs/reference/data-sources/kinesis.md @@ -0,0 +1,74 @@ +# Kinesis source + +**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. + +## Description + +Kinesis sources allow users to register Kinesis streams as data sources. Feast currently does not launch or monitor jobs to ingest data from Kinesis. Users are responsible for launching and monitoring their own ingestion jobs, which should write feature values to the online store through [FeatureStore.write_to_online_store](https://rtd.feast.dev/en/latest/index.html#feast.feature_store.FeatureStore.write_to_online_store). An example of how to launch such a job with Spark to ingest from Kafka can be found [here](https://github.com/feast-dev/feast/tree/master/sdk/python/feast/infra/contrib); by using a different plugin, the example can be adapted to Kinesis. Feast also provides functionality to write to the offline store using the `write_to_offline_store` functionality. + +Kinesis sources must have a batch source specified. The batch source will be used for retrieving historical features. Thus users are also responsible for writing data from their Kinesis streams to a batch data source such as a data warehouse table. When using a Kinesis source as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the feature view definition explicitly. + +## Stream sources +Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: + +1. Raw events come in (stream 1) +2. Streaming transformations applied (e.g. generating features like `last_N_purchased_categories`) (stream 2) +3. Write stream 2 values to an offline store as a historical log for training (optional) +4. Write stream 2 values to an online store for low latency feature serving +5. Periodically materialize feature values from the offline store into the online store for decreased training-serving skew and improved model performance + +## Example +### Defining a Kinesis source +Note that the Kinesis source has a batch source. +```python +from datetime import timedelta + +from feast import Field, FileSource, KinesisSource, stream_feature_view +from feast.data_format import JsonFormat +from feast.types import Float32 + +driver_stats_batch_source = FileSource( + name="driver_stats_source", + path="data/driver_stats.parquet", + timestamp_field="event_timestamp", +) + +driver_stats_stream_source = KinesisSource( + name="driver_stats_stream", + stream_name="drivers", + timestamp_field="event_timestamp", + batch_source=driver_stats_batch_source, + record_format=JsonFormat( + schema_json="driver_id integer, event_timestamp timestamp, conv_rate double, acc_rate double, created timestamp" + ), + watermark_delay_threshold=timedelta(minutes=5), +) +``` + +### Using the Kinesis source in a stream feature view +The Kinesis source can be used in a stream feature view. +```python +@stream_feature_view( + entities=[driver], + ttl=timedelta(seconds=8640000000), + mode="spark", + schema=[ + Field(name="conv_percentage", dtype=Float32), + Field(name="acc_percentage", dtype=Float32), + ], + timestamp_field="event_timestamp", + online=True, + source=driver_stats_stream_source, +) +def driver_hourly_stats_stream(df: DataFrame): + from pyspark.sql.functions import col + + return ( + df.withColumn("conv_percentage", col("conv_rate") * 100.0) + .withColumn("acc_percentage", col("acc_rate") * 100.0) + .drop("conv_rate", "acc_rate") + ) +``` + +### Ingesting data +See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. The approach used in the tutorial can be easily adapted to work for Kinesis as well. diff --git a/ui/public/docs/reference/data-sources/mssql.md b/ui/public/docs/reference/data-sources/mssql.md new file mode 100644 index 00000000000..8bf1ede6aa8 --- /dev/null +++ b/ui/public/docs/reference/data-sources/mssql.md @@ -0,0 +1,29 @@ +# MsSQL source (contrib) + +## Description + +MsSQL data sources are Microsoft sql table sources. +These can be specified either by a table reference or a SQL query. + +## Disclaimer + +The MsSQL data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a MsSQL source: + +```python +from feast.infra.offline_stores.contrib.mssql_offline_store.mssqlserver_source import ( + MsSqlServerSource, +) + +driver_hourly_table = "driver_hourly" + +driver_source = MsSqlServerSource( + table_ref=driver_hourly_table, + event_timestamp_column="datetime", + created_timestamp_column="created", +) +``` diff --git a/ui/public/docs/reference/data-sources/overview.md b/ui/public/docs/reference/data-sources/overview.md new file mode 100644 index 00000000000..9880d388dde --- /dev/null +++ b/ui/public/docs/reference/data-sources/overview.md @@ -0,0 +1,31 @@ +# Overview + +## Functionality + +In Feast, each batch data source is associated with corresponding offline stores. +For example, a `SnowflakeSource` can only be processed by the Snowflake offline store, while a `FileSource` can be processed by both File and DuckDB offline stores. +Otherwise, the primary difference between batch data sources is the set of supported types. +Feast has an internal type system, and aims to support eight primitive types (`bytes`, `string`, `int32`, `int64`, `float32`, `float64`, `bool`, and `timestamp`) along with the corresponding array types. +However, not every batch data source supports all of these types. + +For more details on the Feast type system, see [here](../type-system.md). + +## Functionality Matrix + +There are currently four core batch data source implementations: `FileSource`, `BigQuerySource`, `SnowflakeSource`, and `RedshiftSource`. +There are several additional implementations contributed by the Feast community (`PostgreSQLSource`, `SparkSource`, and `TrinoSource`), which are not guaranteed to be stable or to match the functionality of the core implementations. +Details for each specific data source can be found [here](README.md). + +Below is a matrix indicating which data sources support which types. + +| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | Couchbase | +| :-------------------------------- | :-- | :-- |:----------| :-- | :-- | :-- | :-- |:----------| +| `bytes` | yes | yes | yes | yes | yes | yes | yes | yes | +| `string` | yes | yes | yes | yes | yes | yes | yes | yes | +| `int32` | yes | yes | yes | yes | yes | yes | yes | yes | +| `int64` | yes | yes | yes | yes | yes | yes | yes | yes | +| `float32` | yes | yes | yes | yes | yes | yes | yes | yes | +| `float64` | yes | yes | yes | yes | yes | yes | yes | yes | +| `bool` | yes | yes | yes | yes | yes | yes | yes | yes | +| `timestamp` | yes | yes | yes | yes | yes | yes | yes | yes | +| array types | yes | yes | yes | no | yes | yes | no | no | diff --git a/ui/public/docs/reference/data-sources/postgres.md b/ui/public/docs/reference/data-sources/postgres.md new file mode 100644 index 00000000000..23d7818a04f --- /dev/null +++ b/ui/public/docs/reference/data-sources/postgres.md @@ -0,0 +1,35 @@ +# PostgreSQL source (contrib) + +## Description + +PostgreSQL data sources are PostgreSQL tables or views. +These can be specified either by a table reference or a SQL query. + +## Disclaimer + +The PostgreSQL data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a Postgres source: + +```python +from feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source import ( + PostgreSQLSource, +) + +driver_stats_source = PostgreSQLSource( + name="feast_driver_hourly_stats", + query="SELECT * FROM feast_driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source.PostgreSQLSource). + +## Supported Types + +PostgreSQL data sources support all eight primitive types and their corresponding array types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/push.md b/ui/public/docs/reference/data-sources/push.md new file mode 100644 index 00000000000..7a7ef96c7ca --- /dev/null +++ b/ui/public/docs/reference/data-sources/push.md @@ -0,0 +1,79 @@ +# Push source + +## Description + +Push sources allow feature values to be pushed to the online store and offline store in real time. This allows fresh feature values to be made available to applications. Push sources supercede the [FeatureStore.write_to_online_store](https://rtd.feast.dev/en/latest/index.html#feast.feature_store.FeatureStore.write_to_online_store). + +Push sources can be used by multiple feature views. When data is pushed to a push source, Feast propagates the feature values to all the consuming feature views. + +Push sources must have a batch source specified. The batch source will be used for retrieving historical features. Thus users are also responsible for pushing data to a batch data source such as a data warehouse table. When using a push source as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the feature view definition explicitly. + +## Stream sources +Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: + +1. Raw events come in (stream 1) +2. Streaming transformations applied (e.g. generating features like `last_N_purchased_categories`) (stream 2) +3. Write stream 2 values to an offline store as a historical log for training (optional) +4. Write stream 2 values to an online store for low latency feature serving +5. Periodically materialize feature values from the offline store into the online store for decreased training-serving skew and improved model performance + +Feast allows users to push features previously registered in a feature view to the online store for fresher features. It also allows users to push batches of stream data to the offline store by specifying that the push be directed to the offline store. This will push the data to the offline store declared in the repository configuration used to initialize the feature store. + +## Example (basic) +### Defining a push source +Note that the push schema needs to also include the entity. + +```python +from feast import Entity, PushSource, ValueType, BigQuerySource, FeatureView, Feature, Field +from feast.types import Int64 + +push_source = PushSource( + name="push_source", + batch_source=BigQuerySource(table="test.test"), +) + +user = Entity(name="user", join_keys=["user_id"]) + +fv = FeatureView( + name="feature view", + entities=[user], + schema=[Field(name="life_time_value", dtype=Int64)], + source=push_source, +) +``` + +### Pushing data +Note that the `to` parameter is optional and defaults to online but we can specify these options: `PushMode.ONLINE`, `PushMode.OFFLINE`, or `PushMode.ONLINE_AND_OFFLINE`. +```python +from feast import FeatureStore +import pandas as pd +from feast.data_source import PushMode + +fs = FeatureStore(...) +feature_data_frame = pd.DataFrame() +fs.push("push_source_name", feature_data_frame, to=PushMode.ONLINE_AND_OFFLINE) +``` + +See also [Python feature server](../feature-servers/python-feature-server.md) for instructions on how to push data to a deployed feature server. + +## Example (Spark Streaming) + +The default option to write features from a stream is to add the Python SDK into your existing PySpark pipeline. + +```python +from feast import FeatureStore + +store = FeatureStore(...) + +spark = SparkSession.builder.getOrCreate() + +streamingDF = spark.readStream.format(...).load() + +def feast_writer(spark_df): + pandas_df = spark_df.to_pandas() + store.push("driver_hourly_stats", pandas_df) + +streamingDF.writeStream.foreachBatch(feast_writer).start() +``` + +This can also be used under the hood by a contrib stream processor (see [Tutorial: Building streaming features](../../tutorials/building-streaming-features.md)) diff --git a/ui/public/docs/reference/data-sources/redshift.md b/ui/public/docs/reference/data-sources/redshift.md new file mode 100644 index 00000000000..2c3c65cc701 --- /dev/null +++ b/ui/public/docs/reference/data-sources/redshift.md @@ -0,0 +1,37 @@ +# Redshift source + +## Description + +Redshift data sources are Redshift tables or views. +These can be specified either by a table reference or a SQL query. +However, no performance guarantees can be provided for SQL query-based sources, so table references are recommended. + +## Examples + +Using a table name: + +```python +from feast import RedshiftSource + +my_redshift_source = RedshiftSource( + table="redshift_table", +) +``` + +Using a query: + +```python +from feast import RedshiftSource + +my_redshift_source = RedshiftSource( + query="SELECT timestamp as ts, created, f1, f2 " + "FROM redshift_table", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.redshift_source.RedshiftSource). + +## Supported Types + +Redshift data sources support all eight primitive types, but currently do not support array types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/snowflake.md b/ui/public/docs/reference/data-sources/snowflake.md new file mode 100644 index 00000000000..98a56e09f87 --- /dev/null +++ b/ui/public/docs/reference/data-sources/snowflake.md @@ -0,0 +1,50 @@ +# Snowflake source + +## Description + +Snowflake data sources are Snowflake tables or views. +These can be specified either by a table reference or a SQL query. + +## Examples + +Using a table reference: + +```python +from feast import SnowflakeSource + +my_snowflake_source = SnowflakeSource( + database="FEAST", + schema="PUBLIC", + table="FEATURE_TABLE", +) +``` + +Using a query: + +```python +from feast import SnowflakeSource + +my_snowflake_source = SnowflakeSource( + query=""" + SELECT + timestamp_column AS "ts", + "created", + "f1", + "f2" + FROM + `FEAST.PUBLIC.FEATURE_TABLE` + """, +) +``` + +{% hint style="warning" %} +Be careful about how Snowflake handles table and column name conventions. +In particular, you can read more about quote identifiers [here](https://docs.snowflake.com/en/sql-reference/identifiers-syntax.html). +{% endhint %} + +The full set of configuration options is available [here](https://rtd.feast.dev/en/latest/index.html#feast.infra.offline_stores.snowflake_source.SnowflakeSource). + +## Supported Types + +Snowflake data sources support all eight primitive types. Array types are also supported but not with type inference. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/spark.md b/ui/public/docs/reference/data-sources/spark.md new file mode 100644 index 00000000000..99d5902667a --- /dev/null +++ b/ui/public/docs/reference/data-sources/spark.md @@ -0,0 +1,59 @@ +# Spark source (contrib) + +## Description + +Spark data sources are tables or files that can be loaded from some Spark store (e.g. Hive or in-memory). They can also be specified by a SQL query. + +## Disclaimer + +The Spark data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Using a table reference from SparkSession (for example, either in-memory or a Hive Metastore): + +```python +from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import ( + SparkSource, +) + +my_spark_source = SparkSource( + table="FEATURE_TABLE", +) +``` + +Using a query: + +```python +from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import ( + SparkSource, +) + +my_spark_source = SparkSource( + query="SELECT timestamp as ts, created, f1, f2 " + "FROM spark_table", +) +``` + +Using a file reference: + +```python +from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import ( + SparkSource, +) + +my_spark_source = SparkSource( + path=f"{CURRENT_DIR}/data/driver_hourly_stats", + file_format="parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SparkSource). + +## Supported Types + +Spark data sources support all eight primitive types and their corresponding array types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/data-sources/trino.md b/ui/public/docs/reference/data-sources/trino.md new file mode 100644 index 00000000000..c74981f47e0 --- /dev/null +++ b/ui/public/docs/reference/data-sources/trino.md @@ -0,0 +1,34 @@ +# Trino source (contrib) + +## Description + +Trino data sources are Trino tables or views. +These can be specified either by a table reference or a SQL query. + +## Disclaimer + +The Trino data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a Trino source: + +```python +from feast.infra.offline_stores.contrib.trino_offline_store.trino_source import ( + TrinoSource, +) + +driver_hourly_stats = TrinoSource( + event_timestamp_column="event_timestamp", + table_ref="feast.driver_stats", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#trino-source). + +## Supported Types + +Trino data sources support all eight primitive types, but currently do not support array types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/denormalized.md b/ui/public/docs/reference/denormalized.md new file mode 100644 index 00000000000..9ac39947f05 --- /dev/null +++ b/ui/public/docs/reference/denormalized.md @@ -0,0 +1,121 @@ +# Streaming feature computation with Denormalized + +Denormalized makes it easy to compute real-time features and write them directly to your Feast online store. This guide will walk you through setting up a streaming pipeline that computes feature aggregations and pushes them to Feast in real-time. + +![Denormalized/Feast integration diagram](../assets/feast-denormalized.png) + +## Prerequisites + +- Python 3.12+ +- Kafka cluster (local or remote) OR docker installed + +For a full working demo, check out the [feast-example](https://github.com/probably-nothing-labs/feast-example) repo. + +## Quick Start + +1. First, create a new Python project or use our template: +```bash +mkdir my-feature-project +cd my-feature-project +python -m venv .venv +source .venv/bin/activate # or `.venv\Scripts\activate` on Windows +pip install denormalized[feast] feast +``` + +2. Set up your Feast feature repository: +```bash +feast init feature_repo +``` + +## Project Structure + +Your project should look something like this: +``` +my-feature-project/ +├── feature_repo/ +│ ├── feature_store.yaml +│ └── sensor_data.py # Feature definitions +├── stream_job.py # Denormalized pipeline +└── main.py # Pipeline runner +``` + +3. Run a test Kafka instance in docker + +`docker run --rm -p 9092:9092 emgeee/kafka_emit_measurements:latest` + +This will spin up a docker container that runs a kafka instance and run a simple script to emit fake data to two topics. + + +## Define Your Features + +In `feature_repo/sensor_data.py`, define your feature view and entity: + +```python +from feast import Entity, FeatureView, PushSource, Field +from feast.types import Float64, String + +# Define your entity +sensor = Entity( + name="sensor", + join_keys=["sensor_name"], +) + +# Create a push source for real-time features +source = PushSource( + name="push_sensor_statistics", + batch_source=your_batch_source # Define your batch source +) + +# Define your feature view +stats_view = FeatureView( + name="sensor_statistics", + entities=[sensor], + schema=ds.get_feast_schema(), # Denormalized handles this for you! + source=source, + online=True, +) +``` + +## Create Your Streaming Pipeline + +In `stream_job.py`, define your streaming computations: + +```python +from denormalized import Context, FeastDataStream +from denormalized.datafusion import col, functions as f +from feast import FeatureStore + +sample_event = { + "occurred_at_ms": 100, + "sensor_name": "foo", + "reading": 0.0, +} + +# Create a stream from your Kafka topic +ds = FeastDataStream(Context().from_topic("temperature", json.dumps(sample_event), "localhost:9092", "occurred_at_ms")) + +# Define your feature computations +ds = ds.window( + [col("sensor_name")], # Group by sensor + [ + f.count(col("reading")).alias("count"), + f.min(col("reading")).alias("min"), + f.max(col("reading")).alias("max"), + f.avg(col("reading")).alias("average"), + ], + 1000, # Window size in ms + None # Slide interval (None = tumbling window) +) + +feature_store = FeatureStore(repo_path="feature_repo/") + +# This single line connects Denormalized to Feast! +ds.write_feast_feature(feature_store, "push_sensor_statistics") +``` + + + +## Need Help? + +- Email us at hello@denormalized.io +- Check out more examples on our [GitHub](https://github.com/probably-nothing-labs/denormalized/tree/main/py-denormalized/python/examples) diff --git a/ui/public/docs/reference/dqm.md b/ui/public/docs/reference/dqm.md new file mode 100644 index 00000000000..5a02413e534 --- /dev/null +++ b/ui/public/docs/reference/dqm.md @@ -0,0 +1,77 @@ +# Data Quality Monitoring + +Data Quality Monitoring (DQM) is a Feast module aimed to help users to validate their data with the user-curated set of rules. +Validation could be applied during: +* Historical retrieval (training dataset generation) +* [planned] Writing features into an online store +* [planned] Reading features from an online store + +Its goal is to address several complex data problems, namely: +* Data consistency - new training datasets can be significantly different from previous datasets. This might require a change in model architecture. +* Issues/bugs in the upstream pipeline - bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. +* Training/serving skew - distribution shift could significantly decrease the performance of the model. + +> To monitor data quality, we check that the characteristics of the tested dataset (aka the tested dataset's profile) are "equivalent" to the characteristics of the reference dataset. +> How exactly profile equivalency should be measured is up to the user. + +### Overview + +The validation process consists of the following steps: +1. User prepares reference dataset (currently only [saved datasets](../getting-started/concepts/dataset.md) from historical retrieval are supported). +2. User defines profiler function, which should produce profile by given dataset (currently only profilers based on [Great Expectations](https://docs.greatexpectations.io) are allowed). +3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. + +### Preparations +Feast with Great Expectations support can be installed via +```shell +pip install 'feast[ge]' +``` + +### Dataset profile +Currently, Feast supports only [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) +as dataset's profile. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). + +Great Expectations supports automatic profiling as well as manually specifying expectations: +```python +from great_expectations.dataset import Dataset +from great_expectations.core.expectation_suite import ExpectationSuite + +from feast.dqm.profilers.ge_profiler import ge_profiler + +@ge_profiler +def automatic_profiler(dataset: Dataset) -> ExpectationSuite: + from great_expectations.profile.user_configurable_profiler import UserConfigurableProfiler + + return UserConfigurableProfiler( + profile_dataset=dataset, + ignored_columns=['conv_rate'], + value_set_threshold='few' + ).build_suite() +``` +However, from our experience capabilities of automatic profiler are quite limited. So we would recommend crafting your own expectations: +```python +@ge_profiler +def manual_profiler(dataset: Dataset) -> ExpectationSuite: + dataset.expect_column_max_to_be_between("column", 1, 2) + return dataset.get_expectation_suite() +``` + + + +### Validating Training Dataset +During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)` of RetrievalJob. +If parameter is provided Feast will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. +Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. + +```python +from feast import FeatureStore + +fs = FeatureStore(".") + +job = fs.get_historical_features(...) +job.to_df( + validation_reference=fs + .get_saved_dataset("my_reference_dataset") + .as_reference(profiler=manual_profiler) +) +``` diff --git a/ui/public/docs/reference/feast-cli-commands.md b/ui/public/docs/reference/feast-cli-commands.md new file mode 100644 index 00000000000..712df18a6b6 --- /dev/null +++ b/ui/public/docs/reference/feast-cli-commands.md @@ -0,0 +1,377 @@ +# Feast CLI reference + +## Overview + +The Feast CLI comes bundled with the Feast Python package. It is immediately available after [installing Feast](../how-to-guides/feast-snowflake-gcp-aws/install-feast.md). + +```text +Usage: feast [OPTIONS] COMMAND [ARGS]... + + Feast CLI + + For more information, see our public docs at https://docs.feast.dev/ + +Options: + -c, --chdir TEXT Switch to a different feature repository directory before + executing the given subcommand. + + --help Show this message and exit. + +Commands: + apply Create or update a feature store deployment + configuration Display Feast configuration + entities Access entities + feature-views Access feature views + init Create a new Feast repository + materialize Run a (non-incremental) materialization job to... + materialize-incremental Run an incremental materialization job to ingest... + permissions Access permissions + registry-dump Print contents of the metadata registry + teardown Tear down deployed feature store infrastructure + version Display Feast SDK version +``` + +## Global Options + +The Feast CLI provides one global top-level option that can be used with other commands + +**chdir \(-c, --chdir\)** + +This command allows users to run Feast CLI commands in a different folder from the current working directory. + +```text +feast -c path/to/my/feature/repo apply +``` + +## Apply + +Creates or updates a feature store deployment + +```bash +feast apply +``` + +**What does Feast apply do?** + +1. Feast will scan Python files in your feature repository and find all Feast object definitions, such as feature views, entities, and data sources. +2. Feast will validate your feature definitions (e.g. for uniqueness of features) +3. Feast will sync the metadata about Feast objects to the registry. If a registry does not exist, then it will be instantiated. The standard registry is a simple protobuf binary file that is stored on disk \(locally or in an object store\). +4. Feast CLI will create all necessary feature store infrastructure. The exact infrastructure that is deployed or configured depends on the `provider` configuration that you have set in `feature_store.yaml`. For example, setting `local` as your provider will result in a `sqlite` online store being created. + +{% hint style="warning" %} +`feast apply` \(when configured to use cloud provider like `gcp` or `aws`\) will create cloud infrastructure. This may incur costs. +{% endhint %} + +## Configuration + +Display the actual configuration being used by Feast, including both user-provided configurations and default configurations applied by Feast. + +```bash +feast configuration +``` + +```yaml +project: foo +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +offline_store: + type: dask +entity_key_serialization_version: 2 +auth: + type: no_auth +``` + +## Entities + +List all registered entities + +```text +feast entities list + +Options: + --tags TEXT Filter by tags (e.g. --tags 'key:value' --tags 'key:value, + key:value, ...'). Items return when ALL tags match. +``` + +```text +NAME DESCRIPTION TYPE +driver_id driver id ValueType.INT64 +``` + +## Feature views + +List all registered feature views + +```text +feast feature-views list + +Options: + --tags TEXT Filter by tags (e.g. --tags 'key:value' --tags 'key:value, + key:value, ...'). Items return when ALL tags match. +``` + +```text +NAME ENTITIES TYPE +driver_hourly_stats {'driver'} FeatureView +``` + +## Init + +Creates a new feature repository + +```text +feast init my_repo_name +``` + +```text +Creating a new Feast repository in /projects/my_repo_name. +``` + +```text +. +├── data +│ └── driver_stats.parquet +├── example.py +└── feature_store.yaml +``` + +It's also possible to use other templates + +```text +feast init -t gcp my_feature_repo +``` + +or to set the name of the new project + +```text +feast init -t gcp my_feature_repo +``` + +## Materialize + +Load data from feature views into the online store between two dates + +```bash +feast materialize 2020-01-01T00:00:00 2022-01-01T00:00:00 +``` + +Load data for specific feature views into the online store between two dates + +```text +feast materialize -v driver_hourly_stats 2020-01-01T00:00:00 2022-01-01T00:00:00 +``` + +```text +Materializing 1 feature views from 2020-01-01 to 2022-01-01 + +driver_hourly_stats: +100%|██████████████████████████| 5/5 [00:00<00:00, 5949.37it/s] +``` + +## Materialize incremental + +Load data from feature views into the online store, beginning from either the previous `materialize` or `materialize-incremental` end date, or the beginning of time. + +```text +feast materialize-incremental 2022-01-01T00:00:00 +``` + +## Permissions + +### List permissions +List all registered permission + +```text +feast permissions list + +Options: + --tags TEXT Filter by tags (e.g. --tags 'key:value' --tags 'key:value, + key:value, ...'). Items return when ALL tags match. + -v, --verbose Print the resources matching each configured permission +``` + +```text ++-----------------------+-------------+-----------------------+-----------+----------------+-------------------------+ +| NAME | TYPES | NAME_PATTERNS | ACTIONS | ROLES | REQUIRED_TAGS | ++=======================+=============+=======================+===========+================+================+========+ +| reader_permission1234 | FeatureView | transformed_conv_rate | DESCRIBE | reader | - | +| | | driver_hourly_stats | DESCRIBE | reader | - | ++-----------------------+-------------+-----------------------+-----------+----------------+-------------------------+ +| writer_permission1234 | FeatureView | transformed_conv_rate | CREATE | writer | - | ++-----------------------+-------------+-----------------------+-----------+----------------+-------------------------+ +| special | FeatureView | special.* | DESCRIBE | admin | test-key2 : test-value2 | +| | | | UPDATE | special-reader | test-key : test-value | ++-----------------------+-------------+-----------------------+-----------+----------------+-------------------------+ +``` + +`verbose` option describes the resources matching each configured permission: + +```text +feast permissions list -v +``` + +```text +Permissions: + +permissions +├── reader_permission1234 ['reader'] +│ └── FeatureView: none +└── writer_permission1234 ['writer'] + ├── FeatureView: none + │── OnDemandFeatureView: ['transformed_conv_rate_fresh', 'transformed_conv_rate'] + └── BatchFeatureView: ['driver_hourly_stats', 'driver_hourly_stats_fresh'] +``` + +### Describe a permission +Describes the provided permission + +```text +feast permissions describe permission-name +name: permission-name +types: +- FEATURE_VIEW +namePattern: transformed_conv_rate +requiredTags: + required1: required-value1 + required2: required-value2 +actions: +- DESCRIBE +policy: + roleBasedPolicy: + roles: + - reader +tags: + key1: value1 + key2: value2 + +``` +### Permission check +The `permissions check` command is used to identify resources that lack the appropriate permissions based on their type, name, or tags. + +This command is particularly useful for administrators when roles, actions, or permissions have been modified or newly configured. By running this command, administrators can easily verify which resources and actions are not protected by any permission configuration, ensuring that proper security measures are in place. + +```text +> feast permissions check + + +The following resources are not secured by any permission configuration: +NAME TYPE +driver Entity +driver_hourly_stats_fresh FeatureView +The following actions are not secured by any permission configuration (Note: this might not be a security concern, depending on the used APIs): +NAME TYPE UNSECURED ACTIONS +driver Entity CREATE + DESCRIBE + UPDATE + DELETE + READ_ONLINE + READ_OFFLINE + WRITE_ONLINE + WRITE_OFFLINE +driver_hourly_stats_fresh FeatureView CREATE + DESCRIBE + UPDATE + DELETE + READ_ONLINE + READ_OFFLINE + WRITE_ONLINE + WRITE_OFFLINE + +Based on the above results, the administrator can reassess the permissions configuration and make any necessary adjustments to meet their security requirements. + +If no resources are accessible publicly, the permissions check command will return the following response: +> feast permissions check +The following resources are not secured by any permission configuration: +NAME TYPE +The following actions are not secured by any permission configuration (Note: this might not be a security concern, depending on the used APIs): +NAME TYPE UNSECURED ACTIONS +``` + + +### List of the configured roles +List all the configured roles + +```text +feast permissions list-roles + +Options: + --verbose Print the resources and actions permitted to each configured + role +``` + +```text +ROLE NAME +admin +reader +writer +``` + +`verbose` option describes the resources and actions permitted to each managed role: + +```text +feast permissions list-roles -v +``` + +```text +ROLE NAME RESOURCE NAME RESOURCE TYPE PERMITTED ACTIONS +admin driver_hourly_stats_source FileSource CREATE + DELETE + QUERY_OFFLINE + QUERY_ONLINE + DESCRIBE + UPDATE +admin vals_to_add RequestSource CREATE + DELETE + QUERY_OFFLINE + QUERY_ONLINE + DESCRIBE + UPDATE +admin driver_stats_push_source PushSource CREATE + DELETE + QUERY_OFFLINE + QUERY_ONLINE + DESCRIBE + UPDATE +admin driver_hourly_stats_source FileSource CREATE + DELETE + QUERY_OFFLINE + QUERY_ONLINE + DESCRIBE + UPDATE +admin vals_to_add RequestSource CREATE + DELETE + QUERY_OFFLINE + QUERY_ONLINE + DESCRIBE + UPDATE +admin driver_stats_push_source PushSource CREATE + DELETE + QUERY_OFFLINE + QUERY_ONLINE + DESCRIBE + UPDATE +reader driver_hourly_stats FeatureView DESCRIBE +reader driver_hourly_stats_fresh FeatureView DESCRIBE +... +``` + + +## Teardown + +Tear down deployed feature store infrastructure + +```text +feast teardown +``` + +## Version + +Print the current Feast version + +```text +feast version +``` + diff --git a/ui/public/docs/reference/feast-ignore.md b/ui/public/docs/reference/feast-ignore.md new file mode 100644 index 00000000000..23b2f24afe6 --- /dev/null +++ b/ui/public/docs/reference/feast-ignore.md @@ -0,0 +1,33 @@ +# .feastignore + +## Overview + +`.feastignore` is a file that is placed at the root of the [Feature Repository](feature-repository.md). This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: + +{% code title=".feastignore" %} +```text +# Ignore virtual environment +venv + +# Ignore a specific Python file +scripts/foo.py + +# Ignore all Python files directly under scripts directory +scripts/*.py + +# Ignore all "foo.py" anywhere under scripts directory +scripts/**/foo.py +``` +{% endcode %} + +`.feastignore` file is optional. If the file can not be found, every Python in the feature repo directory will be parsed by `feast apply`. + +## Feast Ignore Patterns + +| Pattern | Example matches | Explanation | +| :--- | :--- | :--- | +| venv | venv/foo.py venv/a/foo.py | You can specify a path to a specific directory. Everything in that directory will be ignored. | +| scripts/foo.py | scripts/foo.py | You can specify a path to a specific file. Only that file will be ignored. | +| scripts/\*.py | scripts/foo.py scripts/bar.py | You can specify asterisk \(\*\) anywhere in the expression. An asterisk matches zero or more characters, except "/". | +| scripts/\*\*/foo.py | scripts/foo.py scripts/a/foo.py scripts/a/b/foo.py | You can specify double asterisk \(\*\*\) anywhere in the expression. A double asterisk matches zero or more directories. | + diff --git a/ui/public/docs/reference/feature-repository.md b/ui/public/docs/reference/feature-repository.md new file mode 100644 index 00000000000..f9c91de3505 --- /dev/null +++ b/ui/public/docs/reference/feature-repository.md @@ -0,0 +1,126 @@ +# Feature repository + +Feast manages two important sets of configuration: feature definitions, and configuration about how to run the feature store. With Feast, this configuration can be written declaratively and stored as code in a central location. This central location is called a feature repository, and it's essentially just a directory that contains some code files. + +The feature repository is the declarative source of truth for what the desired state of a feature store should be. The Feast CLI uses the feature repository to configure your infrastructure, e.g., migrate tables. + +## What is a feature repository? + +A feature repository consists of: + +* A collection of Python files containing feature declarations. +* A `feature_store.yaml` file containing infrastructural configuration. +* A `.feastignore` file containing paths in the feature repository to ignore. + +{% hint style="info" %} +Typically, users store their feature repositories in a Git repository, especially when working in teams. However, using Git is not a requirement. +{% endhint %} + +## Structure of a feature repository + +The structure of a feature repository is as follows: + +* The root of the repository should contain a `feature_store.yaml` file and may contain a `.feastignore` file. +* The repository should contain Python files that contain feature definitions. +* The repository can contain other files as well, including documentation and potentially data files. + +An example structure of a feature repository is shown below: + +```text +$ tree -a +. +├── data +│ └── driver_stats.parquet +├── driver_features.py +├── feature_store.yaml +└── .feastignore + +1 directory, 4 files +``` + +A couple of things to note about the feature repository: + +* Feast reads _all_ Python files recursively when `feast apply` is ran, including subdirectories, even if they don't contain feature definitions. +* It's recommended to add `.feastignore` and add paths to all imperative scripts if you need to store them inside the feature registry. + +## The feature\_store.yaml configuration file + +The configuration for a feature store is stored in a file named `feature_store.yaml` , which must be located at the root of a feature repository. An example `feature_store.yaml` file is shown below: + +{% code title="feature\_store.yaml" %} +```yaml +project: my_feature_repo_1 +registry: data/metadata.db +provider: local +online_store: + path: data/online_store.db +``` +{% endcode %} + +The `feature_store.yaml` file configures how the feature store should run. See [feature\_store.yaml](feature-store-yaml.md) for more details. + +## The .feastignore file + +This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: + +{% code title=".feastignore" %} +```text +# Ignore virtual environment +venv + +# Ignore a specific Python file +scripts/foo.py + +# Ignore all Python files directly under scripts directory +scripts/*.py + +# Ignore all "foo.py" anywhere under scripts directory +scripts/**/foo.py +``` +{% endcode %} + +See [.feastignore](feast-ignore.md) for more details. + +## Feature definitions + +A feature repository can also contain one or more Python files that contain feature definitions. An example feature definition file is shown below: + +{% code title="driver\_features.py" %} +```python +from datetime import timedelta + +from feast import BigQuerySource, Entity, Feature, FeatureView, Field +from feast.types import Float32, Int64, String + +driver_locations_source = BigQuerySource( + table="rh_prod.ride_hailing_co.drivers", + timestamp_field="event_timestamp", + created_timestamp_column="created_timestamp", +) + +driver = Entity( + name="driver", + description="driver id", +) + +driver_locations = FeatureView( + name="driver_locations", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="lat", dtype=Float32), + Field(name="lon", dtype=String), + Field(name="driver", dtype=Int64), + ], + source=driver_locations_source, +) +``` +{% endcode %} + +To declare new feature definitions, just add code to the feature repository, either in existing files or in a new file. For more information on how to define features, see [Feature Views](../getting-started/concepts/feature-view.md). + +### Next steps + +* See [Create a feature repository](../how-to-guides/feast-snowflake-gcp-aws/README.md) to get started with an example feature repository. +* See [feature\_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md) or [Feature Views](../getting-started/concepts/feature-view.md) for more information on the configuration files that live in a feature registry. + diff --git a/ui/public/docs/reference/feature-repository/README.md b/ui/public/docs/reference/feature-repository/README.md new file mode 100644 index 00000000000..2c1b112a783 --- /dev/null +++ b/ui/public/docs/reference/feature-repository/README.md @@ -0,0 +1,130 @@ +# Feature repository + +Feast users use Feast to manage two important sets of configuration: + +* Configuration about how to run Feast on your infrastructure +* Feature definitions + +With Feast, the above configuration can be written declaratively and stored as code in a central location. This central location is called a feature repository. The feature repository is the declarative source of truth for what the desired state of a feature store should be. + +The Feast CLI uses the feature repository to configure, deploy, and manage your feature store. + +## What is a feature repository? + +A feature repository consists of: + +* A collection of Python files containing feature declarations. +* A `feature_store.yaml` file containing infrastructural configuration. +* A `.feastignore` file containing paths in the feature repository to ignore. + +{% hint style="info" %} +Typically, users store their feature repositories in a Git repository, especially when working in teams. However, using Git is not a requirement. +{% endhint %} + +## Structure of a feature repository + +The structure of a feature repository is as follows: + +* The root of the repository should contain a `feature_store.yaml` file and may contain a `.feastignore` file. +* The repository should contain Python files that contain feature definitions. +* The repository can contain other files as well, including documentation and potentially data files. + +An example structure of a feature repository is shown below: + +``` +$ tree -a +. +├── data +│ └── driver_stats.parquet +├── driver_features.py +├── feature_store.yaml +└── .feastignore + +1 directory, 4 files +``` + +A couple of things to note about the feature repository: + +* Feast reads _all_ Python files recursively when `feast apply` is ran, including subdirectories, even if they don't contain feature definitions. +* It's recommended to add `.feastignore` and add paths to all imperative scripts if you need to store them inside the feature registry. + +## The feature_store.yaml configuration file + +The configuration for a feature store is stored in a file named `feature_store.yaml` , which must be located at the root of a feature repository. An example `feature_store.yaml` file is shown below: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo_1 +registry: data/metadata.db +provider: local +online_store: + path: data/online_store.db +``` +{% endcode %} + +The `feature_store.yaml` file configures how the feature store should run. See [feature_store.yaml](feature-store-yaml.md) for more details. + +## The .feastignore file + +This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: + +{% code title=".feastignore" %} +``` +# Ignore virtual environment +venv + +# Ignore a specific Python file +scripts/foo.py + +# Ignore all Python files directly under scripts directory +scripts/*.py + +# Ignore all "foo.py" anywhere under scripts directory +scripts/**/foo.py +``` +{% endcode %} + +See [.feastignore](feast-ignore.md) for more details. + +## Feature definitions + +A feature repository can also contain one or more Python files that contain feature definitions. An example feature definition file is shown below: + +{% code title="driver_features.py" %} +```python +from datetime import timedelta + +from feast import BigQuerySource, Entity, Feature, FeatureView, Field +from feast.types import Float32, Int64, String + +driver_locations_source = BigQuerySource( + table_ref="rh_prod.ride_hailing_co.drivers", + timestamp_field="event_timestamp", + created_timestamp_column="created_timestamp", +) + +driver = Entity( + name="driver", + description="driver id", +) + +driver_locations = FeatureView( + name="driver_locations", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="lat", dtype=Float32), + Field(name="lon", dtype=String), + Field(name="driver", dtype=Int64), + ], + source=driver_locations_source, +) +``` +{% endcode %} + +To declare new feature definitions, just add code to the feature repository, either in existing files or in a new file. For more information on how to define features, see [Feature Views](../../getting-started/concepts/feature-view.md). + +### Next steps + +* See [Create a feature repository](../../how-to-guides/feast-snowflake-gcp-aws/create-a-feature-repository.md) to get started with an example feature repository. +* See [feature_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md), or [Feature Views](../../getting-started/concepts/feature-view.md) for more information on the configuration files that live in a feature registry. diff --git a/ui/public/docs/reference/feature-repository/feast-ignore.md b/ui/public/docs/reference/feature-repository/feast-ignore.md new file mode 100644 index 00000000000..3c23ebe88bc --- /dev/null +++ b/ui/public/docs/reference/feature-repository/feast-ignore.md @@ -0,0 +1,32 @@ +# .feastignore + +## Overview + +`.feastignore` is a file that is placed at the root of the [Feature Repository](./). This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: + +{% code title=".feastignore" %} +``` +# Ignore virtual environment +venv + +# Ignore a specific Python file +scripts/foo.py + +# Ignore all Python files directly under scripts directory +scripts/*.py + +# Ignore all "foo.py" anywhere under scripts directory +scripts/**/foo.py +``` +{% endcode %} + +`.feastignore` file is optional. If the file can not be found, every Python file in the feature repo directory will be parsed by `feast apply`. + +## Feast Ignore Patterns + +| Pattern | Example matches | Explanation | +| ------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| venv |

venv/foo.py
venv/a/foo.py

| You can specify a path to a specific directory. Everything in that directory will be ignored. | +| scripts/foo.py | scripts/foo.py | You can specify a path to a specific file. Only that file will be ignored. | +| scripts/\*.py |

scripts/foo.py
scripts/bar.py

| You can specify an asterisk (\*) anywhere in the expression. An asterisk matches zero or more characters, except "/". | +| scripts/\*\*/foo.py |

scripts/foo.py
scripts/a/foo.py
scripts/a/b/foo.py

| You can specify a double asterisk (\*\*) anywhere in the expression. A double asterisk matches zero or more directories. | diff --git a/ui/public/docs/reference/feature-repository/feature-store-yaml.md b/ui/public/docs/reference/feature-repository/feature-store-yaml.md new file mode 100644 index 00000000000..a87e09ba43e --- /dev/null +++ b/ui/public/docs/reference/feature-repository/feature-store-yaml.md @@ -0,0 +1,29 @@ +# feature_store.yaml + +## Overview + +`feature_store.yaml` is used to configure a feature store. The file must be located at the root of a [feature repository](./). An example `feature_store.yaml` is shown below: + +{% code title="feature_store.yaml" %} +```yaml +project: loyal_spider +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +``` +{% endcode %} + +## Options + +The following top-level configuration options exist in the `feature_store.yaml` file. + +* **provider** — Configures the environment in which Feast will deploy and operate. +* **registry** — Configures the location of the feature registry. +* **online_store** — Configures the online store. +* **offline_store** — Configures the offline store. +* **project** — Defines a namespace for the entire feature store. Can be used to isolate multiple deployments in a single installation of Feast. Should only contain letters, numbers, and underscores. +* **engine** - Configures the batch materialization engine. + +Please see the [RepoConfig](https://rtd.feast.dev/en/latest/#feast.repo_config.RepoConfig) API reference for the full list of configuration options. diff --git a/ui/public/docs/reference/feature-repository/registration-inferencing.md b/ui/public/docs/reference/feature-repository/registration-inferencing.md new file mode 100644 index 00000000000..84faf949e11 --- /dev/null +++ b/ui/public/docs/reference/feature-repository/registration-inferencing.md @@ -0,0 +1,7 @@ +# Registration Inferencing + +## Overview + +* FeatureView - When the `features` parameter is left out of the feature view definition, upon a `feast apply` call, Feast will automatically consider every column in the data source as a feature to be registered other than the specific timestamp columns associated with the underlying data source definition (e.g. timestamp_field) and the columns associated with the feature view's entities. +* DataSource - When the `timestamp_field` parameter is left out of the data source definition, upon a 'feast apply' call, Feast will automatically find the sole timestamp column in the table underlying the data source and use that as the `timestamp_field`. If there are no columns of timestamp type or multiple columns of timestamp type, `feast apply` will throw an exception. +* Entity - When the `value_type` parameter is left out of the entity definition, upon a `feast apply` call, Feast will automatically find the column corresponding with the entity's `join_key` and take that column's data type to be the `value_type`. If the column doesn't exist, `feast apply` will throw an exception. diff --git a/ui/public/docs/reference/feature-servers/README.md b/ui/public/docs/reference/feature-servers/README.md new file mode 100644 index 00000000000..156e60c7431 --- /dev/null +++ b/ui/public/docs/reference/feature-servers/README.md @@ -0,0 +1,19 @@ +# Feast servers + +Feast users can choose to retrieve features from a feature server, as opposed to through the Python SDK. + +{% content-ref url="python-feature-server.md" %} +[python-feature-server.md](python-feature-server.md) +{% endcontent-ref %} + +{% content-ref url="go-feature-server.md" %} +[go-feature-server.md](go-feature-server.md) +{% endcontent-ref %} + +{% content-ref url="offline-feature-server.md" %} +[offline-feature-server.md](offline-feature-server.md) +{% endcontent-ref %} + +{% content-ref url="registry-server.md" %} +[registry-server.md](registry-server.md) +{% endcontent-ref %} \ No newline at end of file diff --git a/ui/public/docs/reference/feature-servers/offline-feature-server.md b/ui/public/docs/reference/feature-servers/offline-feature-server.md new file mode 100644 index 00000000000..9fd429794f7 --- /dev/null +++ b/ui/public/docs/reference/feature-servers/offline-feature-server.md @@ -0,0 +1,60 @@ +# Offline feature server + +## Description + +The Offline feature server is an Apache Arrow Flight Server that uses the gRPC communication protocol to exchange data. +This server wraps calls to existing offline store implementations and exposes interfaces as Arrow Flight endpoints. + +## How to configure the server + +## CLI + +There is a CLI command that starts the Offline feature server: `feast serve_offline`. By default, remote offline server uses port 8815, the port can be overridden with a `--port` flag. + +## Deploying as a service on Kubernetes + +See [this](../../how-to-guides/running-feast-in-production.md#id-4.2.-deploy-feast-feature-servers-on-kubernetes) for an example on how to run Feast on Kubernetes using the Operator. + +The Offline feature server can be deployed with a slight modification of the FeatureStore CR - +```yaml +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-offline-server +spec: + feastProject: my_project + services: + offlineStore: + server: {} +``` +> _More advanced FeatureStore CR examples can be found in the feast-operator [samples directory](../../../infra/feast-operator/config/samples)._ + +## Server Example + +The complete example can be found under [remote-offline-store-example](../../../examples/remote-offline-store) + +## How to configure the client + +Please see the detail how to configure offline store client [remote-offline-store.md](../offline-stores/remote-offline-store.md) + +## Functionality Matrix + +The set of functionalities supported by remote offline stores is the same as those supported by offline stores with the SDK, which are described in detail [here](../offline-stores/overview.md#functionality). + +# Offline Feature Server Permissions and Access Control + +## API Endpoints and Permissions + +| Endpoint | Resource Type | Permission | Description | +| ------------------------------------- |------------------|---------------|---------------------------------------------------| +| offline_write_batch | FeatureView | Write Offline | Write a batch of data to the offline store | +| write_logged_features | FeatureService | Write Offline | Write logged features to the offline store | +| persist | DataSource | Write Offline | Persist the result of a read in the offline store | +| get_historical_features | FeatureView | Read Offline | Retrieve historical features | +| pull_all_from_table_or_query | DataSource | Read Offline | Pull all data from a table or read it | +| pull_latest_from_table_or_query | DataSource | Read Offline | Pull the latest data from a table or read it | + + +## How to configure Authentication and Authorization ? + +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. \ No newline at end of file diff --git a/ui/public/docs/reference/feature-servers/python-feature-server.md b/ui/public/docs/reference/feature-servers/python-feature-server.md new file mode 100644 index 00000000000..7d251671636 --- /dev/null +++ b/ui/public/docs/reference/feature-servers/python-feature-server.md @@ -0,0 +1,240 @@ +# Python feature server + +## Overview + +The Python feature server is an HTTP endpoint that serves features with JSON I/O. This enables users to write and read features from the online store using any programming language that can make HTTP requests. + +## CLI + +There is a CLI command that starts the server: `feast serve`. By default, Feast uses port 6566; the port be overridden with a `--port` flag. + +## Deploying as a service + +See [this](../../how-to-guides/running-feast-in-production.md#id-4.2.-deploy-feast-feature-servers-on-kubernetes) for an example on how to run Feast on Kubernetes using the Operator. + +## Example + +### Initializing a feature server + +Here's an example of how to start the Python feature server with a local feature repo: + +```bash +$ feast init feature_repo +Creating a new Feast repository in /home/tsotne/feast/feature_repo. + +$ cd feature_repo + +$ feast apply +Created entity driver +Created feature view driver_hourly_stats +Created feature service driver_activity + +Created sqlite table feature_repo_driver_hourly_stats + +$ feast materialize-incremental $(date +%Y-%m-%d) +Materializing 1 feature views to 2021-09-09 17:00:00-07:00 into the sqlite online store. + +driver_hourly_stats from 2021-09-09 16:51:08-07:00 to 2021-09-09 17:00:00-07:00: +100%|████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 295.24it/s] + +$ feast serve +09/10/2021 10:42:11 AM INFO:Started server process [8889] +INFO: Waiting for application startup. +09/10/2021 10:42:11 AM INFO:Waiting for application startup. +INFO: Application startup complete. +09/10/2021 10:42:11 AM INFO:Application startup complete. +INFO: Uvicorn running on http://127.0.0.1:6566 (Press CTRL+C to quit) +09/10/2021 10:42:11 AM INFO:Uvicorn running on http://127.0.0.1:6566 (Press CTRL+C to quit) +``` + +### Retrieving features + +After the server starts, we can execute cURL commands from another terminal tab: + +```bash +$ curl -X POST \ + "http://localhost:6566/get-online-features" \ + -d '{ + "features": [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips" + ], + "entities": { + "driver_id": [1001, 1002, 1003] + } + }' | jq +{ + "metadata": { + "feature_names": [ + "driver_id", + "conv_rate", + "avg_daily_trips", + "acc_rate" + ] + }, + "results": [ + { + "values": [ + 1001, + 0.7037263512611389, + 308, + 0.8724706768989563 + ], + "statuses": [ + "PRESENT", + "PRESENT", + "PRESENT", + "PRESENT" + ], + "event_timestamps": [ + "1970-01-01T00:00:00Z", + "2021-12-31T23:00:00Z", + "2021-12-31T23:00:00Z", + "2021-12-31T23:00:00Z" + ] + }, + { + "values": [ + 1002, + 0.038169607520103455, + 332, + 0.48534533381462097 + ], + "statuses": [ + "PRESENT", + "PRESENT", + "PRESENT", + "PRESENT" + ], + "event_timestamps": [ + "1970-01-01T00:00:00Z", + "2021-12-31T23:00:00Z", + "2021-12-31T23:00:00Z", + "2021-12-31T23:00:00Z" + ] + }, + { + "values": [ + 1003, + 0.9665873050689697, + 779, + 0.7793770432472229 + ], + "statuses": [ + "PRESENT", + "PRESENT", + "PRESENT", + "PRESENT" + ], + "event_timestamps": [ + "1970-01-01T00:00:00Z", + "2021-12-31T23:00:00Z", + "2021-12-31T23:00:00Z", + "2021-12-31T23:00:00Z" + ] + } + ] +} +``` + +It's also possible to specify a feature service name instead of the list of features: + +``` +curl -X POST \ + "http://localhost:6566/get-online-features" \ + -d '{ + "feature_service": , + "entities": { + "driver_id": [1001, 1002, 1003] + } + }' | jq +``` + +### Pushing features to the online and offline stores + +The Python feature server also exposes an endpoint for [push sources](../data-sources/push.md). This endpoint allows you to push data to the online and/or offline store. + +The request definition for `PushMode` is a string parameter `to` where the options are: \[`"online"`, `"offline"`, `"online_and_offline"`]. + +**Note:** timestamps need to be strings, and might need to be timezone aware (matching the schema of the offline store) + +``` +curl -X POST "http://localhost:6566/push" -d '{ + "push_source_name": "driver_stats_push_source", + "df": { + "driver_id": [1001], + "event_timestamp": ["2022-05-13 10:59:42+00:00"], + "created": ["2022-05-13 10:59:42"], + "conv_rate": [1.0], + "acc_rate": [1.0], + "avg_daily_trips": [1000] + }, + "to": "online_and_offline" + }' | jq +``` + +or equivalently from Python: + +```python +import json +import requests +from datetime import datetime + +event_dict = { + "driver_id": [1001], + "event_timestamp": [str(datetime(2021, 5, 13, 10, 59, 42))], + "created": [str(datetime(2021, 5, 13, 10, 59, 42))], + "conv_rate": [1.0], + "acc_rate": [1.0], + "avg_daily_trips": [1000], + "string_feature": "test2", +} +push_data = { + "push_source_name":"driver_stats_push_source", + "df":event_dict, + "to":"online", +} +requests.post( + "http://localhost:6566/push", + data=json.dumps(push_data)) +``` + +## Starting the feature server in TLS(SSL) mode + +Enabling TLS mode ensures that data between the Feast client and server is transmitted securely. For an ideal production environment, it is recommended to start the feature server in TLS mode. + +### Obtaining a self-signed TLS certificate and key +In development mode we can generate a self-signed certificate for testing. In an actual production environment it is always recommended to get it from a trusted TLS certificate provider. + +```shell +openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -days 365 -nodes +``` + +The above command will generate two files +* `key.pem` : certificate private key +* `cert.pem`: certificate public key + +### Starting the Online Server in TLS(SSL) Mode +To start the feature server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments with the `feast serve` command. + +```shell +feast serve --key /path/to/key.pem --cert /path/to/cert.pem +``` + +# Online Feature Server Permissions and Access Control + +## API Endpoints and Permissions + +| Endpoint | Resource Type | Permission | Description | +|----------------------------|---------------------------------|-------------------------------------------------------|----------------------------------------------------------------| +| /get-online-features | FeatureView,OnDemandFeatureView | Read Online | Get online features from the feature store | +| /retrieve-online-documents | FeatureView | Read Online | Retrieve online documents from the feature store for RAG | +| /push | FeatureView | Write Online, Write Offline, Write Online and Offline | Push features to the feature store (online, offline, or both) | +| /write-to-online-store | FeatureView | Write Online | Write features to the online store | +| /materialize | FeatureView | Write Online | Materialize features within a specified time range | +| /materialize-incremental | FeatureView | Write Online | Incrementally materialize features up to a specified timestamp | + +## How to configure Authentication and Authorization ? + +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. \ No newline at end of file diff --git a/ui/public/docs/reference/feature-servers/registry-server.md b/ui/public/docs/reference/feature-servers/registry-server.md new file mode 100644 index 00000000000..9707a597035 --- /dev/null +++ b/ui/public/docs/reference/feature-servers/registry-server.md @@ -0,0 +1,26 @@ +# Registry server + +## Description + +The Registry server uses the gRPC communication protocol to exchange data. +This enables users to communicate with the server using any programming language that can make gRPC requests. + +## How to configure the server + +## CLI + +There is a CLI command that starts the Registry server: `feast serve_registry`. By default, remote Registry Server uses port 6570, the port can be overridden with a `--port` flag. +To start the Registry Server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments. +More info about TLS mode can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#starting-feast-registry-server-in-tls-mode) + +## How to configure the client + +Please see the detail how to configure Remote Registry client [remote.md](../registries/remote.md) + +# Registry Server Permissions and Access Control + +Please refer the [page](./../registry/registry-permissions.md) for more details on API Endpoints and Permissions. + +## How to configure Authentication and Authorization ? + +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. \ No newline at end of file diff --git a/ui/public/docs/reference/feature-store-yaml.md b/ui/public/docs/reference/feature-store-yaml.md new file mode 100644 index 00000000000..01f586c047b --- /dev/null +++ b/ui/public/docs/reference/feature-store-yaml.md @@ -0,0 +1,132 @@ +# feature\_store.yaml + +## Overview + +`feature_store.yaml` is a file that is placed at the root of the [Feature Repository](feature-repository.md). This file contains configuration about how the feature store runs. An example `feature_store.yaml` is shown below: + +{% code title="feature\_store.yaml" %} +```yaml +project: loyal_spider +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +``` +{% endcode %} + +## Fields in feature\_store.yaml + +* **provider** \("local" or "gcp"\) — Defines the environment in which Feast will execute data flows. +* **registry** \(a local or GCS filepath\) — Defines the location of the feature registry. +* **online\_store** — Configures the online store. This field will have various subfields depending on the type of online store: + * **type** \("sqlite" or "datastore"\) — Defines the type of online store. + * **path** \(a local filepath\) — Parameter for the sqlite online store. Defines the path to the SQLite database file. + * **project\_id** — Optional parameter for the datastore online store. Sets the GCP project id used by Feast, if not set Feast will use the default GCP project id in the local environment. +* **project** — Defines a namespace for the entire feature store. Can be used to isolate multiple deployments in a single installation of Feast. + +## Providers + +The `provider` field defines the environment in which Feast will execute data flows. As a result, it also determines the default values for other fields. + +### Local + +When using the local provider: + +* Feast can read from **local Parquet data sources.** +* Feast performs historical feature retrieval \(point-in-time joins\) using **pandas.** +* Feast performs online feature serving from a **SQLite database.** + +### **GCP** + +When using the GCP provider: + +* Feast can read data from **BigQuery data sources.** +* Feast performs historical feature retrieval \(point-in-time joins\) in **BigQuery.** +* Feast performs online feature serving from **Google Cloud Datastore.** + +**Permissions** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Command + ComponentPermissionsRecommended Role
Apply + BigQuery (source) +

bigquery.jobs.create

+

bigquery.readsessions.create

+

bigquery.readsessions.getData

+ 		roles/bigquery.user
Apply + Datastore (destination) +

datastore.entities.allocateIds

+

datastore.entities.create

+

datastore.entities.delete

+

datastore.entities.get

+

datastore.entities.list

+

datastore.entities.update

+ 		roles/datastore.owner
Materialize + BigQuery (source)bigquery.jobs.createroles/bigquery.user
Materialize + Datastore (destination) +

datastore.entities.allocateIds

+

datastore.entities.create

+

datastore.entities.delete

+

datastore.entities.get

+

datastore.entities.list

+

datastore.entities.update

+

datastore.databases.get

+ 		roles/datastore.owner
Get Online Features + Datastoredatastore.entities.getroles/datastore.user
Get Historical Features + BigQuery (source) +

bigquery.datasets.get

+

bigquery.tables.get

+

bigquery.tables.create

+

bigquery.tables.updateData

+

bigquery.tables.update

+

bigquery.tables.delete

+

bigquery.tables.getData

+ 		roles/bigquery.dataEditor
+ diff --git a/ui/public/docs/reference/offline-stores/README.md b/ui/public/docs/reference/offline-stores/README.md new file mode 100644 index 00000000000..ab25fe9a276 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/README.md @@ -0,0 +1,47 @@ +# Offline stores + +Please see [Offline Store](../../getting-started/components/offline-store.md) for a conceptual explanation of offline stores. + +{% content-ref url="overview.md" %} +[overview.md](overview.md) +{% endcontent-ref %} + +{% content-ref url="dask.md" %} +[dask.md](dask.md) +{% endcontent-ref %} + +{% content-ref url="snowflake.md" %} +[snowflake.md](snowflake.md) +{% endcontent-ref %} + +{% content-ref url="bigquery.md" %} +[bigquery.md](bigquery.md) +{% endcontent-ref %} + +{% content-ref url="redshift.md" %} +[redshift.md](redshift.md) +{% endcontent-ref %} + +{% content-ref url="duckdb.md" %} +[duckdb.md](duckdb.md) +{% endcontent-ref %} + +{% content-ref url="couchbase.md" %} +[couchbase.md](couchbase.md) +{% endcontent-ref %} + +{% content-ref url="spark.md" %} +[spark.md](spark.md) +{% endcontent-ref %} + +{% content-ref url="postgres.md" %} +[postgres.md](postgres.md) +{% endcontent-ref %} + +{% content-ref url="trino.md" %} +[trino.md](trino.md) +{% endcontent-ref %} + +{% content-ref url="mssql.md" %} +[mssql.md](mssql.md) +{% endcontent-ref %} diff --git a/ui/public/docs/reference/offline-stores/bigquery.md b/ui/public/docs/reference/offline-stores/bigquery.md new file mode 100644 index 00000000000..b7607abf595 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/bigquery.md @@ -0,0 +1,60 @@ +# BigQuery offline store + +## Description + +The BigQuery offline store provides support for reading [BigQuerySources](../data-sources/bigquery.md). + +* All joins happen within BigQuery. +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to BigQuery as a table (marked for expiration) in order to complete join operations. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[gcp]'`. You can get started by then running `feast init -t gcp`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: gs://my-bucket/data/registry.db +provider: gcp +offline_store: + type: bigquery + dataset: feast_bq_dataset +``` +{% endcode %} + +The full set of configuration options is available in [BigQueryOfflineStoreConfig](https://rtd.feast.dev/en/latest/index.html#feast.infra.offline_stores.bigquery.BigQueryOfflineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the BigQuery offline store. + +| | BigQuery | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | + +Below is a matrix indicating which functionality is supported by `BigQueryRetrievalJob`. + +| | BigQuery | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data* | partial | + +*See [GitHub issue](https://github.com/feast-dev/feast/issues/2530) for details on proposed solutions for enabling the BigQuery offline store to understand tables that use `_PARTITIONTIME` as the partition column. + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/clickhouse.md b/ui/public/docs/reference/offline-stores/clickhouse.md new file mode 100644 index 00000000000..317d6e23e1e --- /dev/null +++ b/ui/public/docs/reference/offline-stores/clickhouse.md @@ -0,0 +1,69 @@ +# Clickhouse offline store (contrib) + +## Description + +The Clickhouse offline store provides support for reading [ClickhouseSource](../data-sources/clickhouse.md). +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Clickhouse as a table (temporary table by default) in order to complete join operations. + +## Disclaimer + +The Clickhouse offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[clickhouse]'`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD + use_temporary_tables_for_entity_df: true +online_store: + path: data/online_store.db +``` +{% endcode %} + +Note that `use_temporary_tables_for_entity_df` is an optional parameter. +The full set of configuration options is available in [ClickhouseOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse.ClickhouseOfflineStore). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Clickhouse offline store. + +| | Clickhouse | +| :----------------------------------------------------------------- |:-----------| +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | no | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `ClickhouseRetrievalJob`. + +| | Clickhouse | +| ----------------------------------------------------- |------------| +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/couchbase.md b/ui/public/docs/reference/offline-stores/couchbase.md new file mode 100644 index 00000000000..3ae0f68d4c2 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/couchbase.md @@ -0,0 +1,79 @@ +# Couchbase Columnar offline store (contrib) + +## Description + +The Couchbase Columnar offline store provides support for reading [CouchbaseColumnarSources](../data-sources/couchbase.md). **Note that Couchbase Columnar is available through [Couchbase Capella](https://cloud.couchbase.com/).** +* Entity dataframes can be provided as a SQL++ query or can be provided as a Pandas dataframe. A Pandas dataframe will be uploaded to Couchbase Capella Columnar as a collection. + +## Disclaimer + +The Couchbase Columnar offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started + +In order to use this offline store, you'll need to run `pip install 'feast[couchbase]'`. You can get started by then running `feast init -t couchbase`. + +To get started with Couchbase Capella Columnar: +1. Sign up for a [Couchbase Capella](https://cloud.couchbase.com/) account +2. [Deploy a Columnar cluster](https://docs.couchbase.com/columnar/admin/prepare-project.html) +3. [Create an Access Control Account](https://docs.couchbase.com/columnar/admin/auth/auth-data.html) + - This account should be able to read and write. + - For testing purposes, it is recommended to assign all roles to avoid any permission issues. +4. [Configure allowed IP addresses](https://docs.couchbase.com/columnar/admin/ip-allowed-list.html) + - You must allow the IP address of the machine running Feast. + + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: couchbase.offline + connection_string: COUCHBASE_COLUMNAR_CONNECTION_STRING # Copied from Settings > Connection String page in Capella Columnar console, starts with couchbases:// + user: COUCHBASE_COLUMNAR_USER # Couchbase cluster access name from Settings > Access Control page in Capella Columnar console + password: COUCHBASE_COLUMNAR_PASSWORD # Couchbase password from Settings > Access Control page in Capella Columnar console + timeout: 120 # Timeout in seconds for Columnar operations, optional +online_store: + path: data/online_store.db +``` +{% endcode %} + +Note that `timeout`is an optional parameter. +The full set of configuration options is available in [CouchbaseColumnarOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase.CouchbaseColumnarOfflineStoreConfig). + + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Couchbase Columnar offline store. + +| | Couchbase Columnar | +| :----------------------------------------------------------------- |:-------------------| +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `CouchbaseColumnarRetrievalJob`. + +| | Couchbase Columnar | +| ----------------------------------------------------- |--------------------| +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/dask.md b/ui/public/docs/reference/offline-stores/dask.md new file mode 100644 index 00000000000..d8698ba544b --- /dev/null +++ b/ui/public/docs/reference/offline-stores/dask.md @@ -0,0 +1,55 @@ +# Dask offline store + +## Description + +The Dask offline store provides support for reading [FileSources](../data-sources/file.md). + +{% hint style="warning" %} +All data is downloaded and joined using Python and therefore may not scale to production workloads. +{% endhint %} + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +offline_store: + type: dask +``` +{% endcode %} + +The full set of configuration options is available in [DaskOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.dask.DaskOfflineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the dask offline store. + +| | Dask | +| :-------------------------------- | :-- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | + +Below is a matrix indicating which functionality is supported by `DaskRetrievalJob`. + +| | Dask | +| --------------------------------- | --- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/duckdb.md b/ui/public/docs/reference/offline-stores/duckdb.md new file mode 100644 index 00000000000..da3c3cd0c77 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/duckdb.md @@ -0,0 +1,56 @@ +# DuckDB offline store + +## Description + +The duckdb offline store provides support for reading [FileSources](../data-sources/file.md). It can read both Parquet and Delta formats. DuckDB offline store uses [ibis](https://ibis-project.org/) under the hood to convert offline store operations to DuckDB queries. + +* Entity dataframes can be provided as a Pandas dataframe. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[duckdb]'`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: duckdb +online_store: + path: data/online_store.db +``` +{% endcode %} + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the DuckDB offline store. + +| | DuckdDB | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | + +Below is a matrix indicating which functionality is supported by `IbisRetrievalJob`. + +| | DuckDB| +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | no | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/mssql.md b/ui/public/docs/reference/offline-stores/mssql.md new file mode 100644 index 00000000000..e352b3dd2aa --- /dev/null +++ b/ui/public/docs/reference/offline-stores/mssql.md @@ -0,0 +1,62 @@ +# MsSQL/Synapse offline store (contrib) + +## Description + +The MsSQL offline store provides support for reading [MsSQL Sources](../data-sources/mssql.md). Specifically, it is developed to read from [Synapse SQL](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/overview-features) on Microsoft Azure + +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[azure]'`. You can get started by then following this [tutorial](https://github.com/feast-dev/feast/blob/master/docs/tutorials/azure/README.md). + +## Disclaimer + +The MsSQL offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +registry: + registry_store_type: AzureRegistryStore + path: ${REGISTRY_PATH} # Environment Variable +project: production +provider: azure +online_store: + type: redis + connection_string: ${REDIS_CONN} # Environment Variable +offline_store: + type: mssql + connection_string: ${SQL_CONN} # Environment Variable +``` +{% endcode %} + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Spark offline store. + +| | MsSql | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `MsSqlServerRetrievalJob`. + +| | MsSql | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| local execution of Python-based on-demand transforms | no | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/overview.md b/ui/public/docs/reference/offline-stores/overview.md new file mode 100644 index 00000000000..191ccd21a64 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/overview.md @@ -0,0 +1,58 @@ +# Overview + +## Functionality + +Here are the methods exposed by the `OfflineStore` interface, along with the core functionality supported by the method: +* `get_historical_features`: point-in-time correct join to retrieve historical features +* `pull_latest_from_table_or_query`: retrieve latest feature values for materialization into the online store +* `pull_all_from_table_or_query`: retrieve a saved dataset +* `offline_write_batch`: persist dataframes to the offline store, primarily for push sources +* `write_logged_features`: persist logged features to the offline store, for feature logging + +The first three of these methods all return a `RetrievalJob` specific to an offline store, such as a `SnowflakeRetrievalJob`. Here is a list of functionality supported by `RetrievalJob`s: +* export to dataframe +* export to arrow table +* export to arrow batches (to handle large datasets in memory) +* export to SQL +* export to data lake (S3, GCS, etc.) +* export to data warehouse +* export as Spark dataframe +* local execution of Python-based on-demand transforms +* remote execution of Python-based on-demand transforms +* persist results in the offline store +* preview the query plan before execution (`RetrievalJob`s are lazily executed) +* read partitioned data + +## Functionality Matrix + +There are currently four core offline store implementations: `DaskOfflineStore`, `BigQueryOfflineStore`, `SnowflakeOfflineStore`, and `RedshiftOfflineStore`. +There are several additional implementations contributed by the Feast community (`PostgreSQLOfflineStore`, `SparkOfflineStore`, and `TrinoOfflineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +Details for each specific offline store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). + +Below is a matrix indicating which offline stores support which methods. + +| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | Couchbase | +| :-------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | +| `get_historical_features` | yes | yes | yes | yes | yes | yes | yes | yes | +| `pull_latest_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | yes | +| `pull_all_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | yes | +| `offline_write_batch` | yes | yes | yes | yes | no | no | no | no | +| `write_logged_features` | yes | yes | yes | yes | no | no | no | no | + + +Below is a matrix indicating which `RetrievalJob`s support what functionality. + +| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | DuckDB | Couchbase | +| --------------------------------- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| export to dataframe | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| export to arrow table | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| export to arrow batches | no | no | no | yes | no | no | no | no | no | +| export to SQL | no | yes | yes | yes | yes | no | yes | no | yes | +| export to data lake (S3, GCS, etc.) | no | no | yes | no | yes | no | no | no | yes | +| export to data warehouse | no | yes | yes | yes | yes | no | no | no | yes | +| export as Spark dataframe | no | no | yes | no | no | yes | no | no | no | +| local execution of Python-based on-demand transforms | yes | yes | yes | yes | yes | no | yes | yes | yes | +| remote execution of Python-based on-demand transforms | no | no | no | no | no | no | no | no | no | +| persist results in the offline store | yes | yes | yes | yes | yes | yes | no | yes | yes | +| preview the query plan before execution | yes | yes | yes | yes | yes | yes | yes | no | yes | +| read partitioned data | yes | yes | yes | yes | yes | yes | yes | yes | yes | diff --git a/ui/public/docs/reference/offline-stores/postgres.md b/ui/public/docs/reference/offline-stores/postgres.md new file mode 100644 index 00000000000..321ddcf25e7 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/postgres.md @@ -0,0 +1,76 @@ +# PostgreSQL offline store (contrib) + +## Description + +The PostgreSQL offline store provides support for reading [PostgreSQLSources](../data-sources/postgres.md). +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Postgres as a table in order to complete join operations. + +## Disclaimer + +The PostgreSQL offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[postgres]'`. You can get started by then running `feast init -t postgres`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: postgres + host: DB_HOST + port: DB_PORT + database: DB_NAME + db_schema: DB_SCHEMA + user: DB_USERNAME + password: DB_PASSWORD + sslmode: verify-ca + sslkey_path: /path/to/client-key.pem + sslcert_path: /path/to/client-cert.pem + sslrootcert_path: /path/to/server-ca.pem + entity_select_mode: temp_table +online_store: + path: data/online_store.db +``` +{% endcode %} + +Note that `sslmode`, `sslkey_path`, `sslcert_path`, and `sslrootcert_path` are optional parameters. +The full set of configuration options is available in [PostgreSQLOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLOfflineStoreConfig). + +Additionally, a new optional parameter `entity_select_mode` was added to tell how Postgres should load the entity data. By default(`temp_table`), a temporary table is created and the entity data frame or sql is loaded into that table. A new value of `embed_query` was added to allow directly loading the SQL query into a CTE, providing improved performance and skipping the need to CREATE and DROP the temporary table. + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the PostgreSQL offline store. + +| | Postgres | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `PostgreSQLRetrievalJob`. + +| | Postgres | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/redshift.md b/ui/public/docs/reference/offline-stores/redshift.md new file mode 100644 index 00000000000..e33a1856cb2 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/redshift.md @@ -0,0 +1,179 @@ +# Redshift offline store + +## Description + +The Redshift offline store provides support for reading [RedshiftSources](../data-sources/redshift.md). + +* All joins happen within Redshift. +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Redshift temporarily in order to complete join operations. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[aws]'`. You can get started by then running `feast init -t aws`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: aws +offline_store: + type: redshift + region: us-west-2 + cluster_id: feast-cluster + database: feast-database + user: redshift-user + s3_staging_location: s3://feast-bucket/redshift + iam_role: arn:aws:iam::123456789012:role/redshift_s3_access_role +``` +{% endcode %} + +The full set of configuration options is available in [RedshiftOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.redshift.RedshiftOfflineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redshift offline store. + +| | Redshift | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | + +Below is a matrix indicating which functionality is supported by `RedshiftRetrievalJob`. + +| | Redshift | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | yes | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Permissions + +Feast requires the following permissions in order to execute commands for Redshift offline store: + +| **Command** | Permissions | Resources | +| --------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Apply** |

redshift-data:DescribeTable

redshift:GetClusterCredentials

|

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

| +| **Materialize** | redshift-data:ExecuteStatement | arn:aws:redshift:\:\:cluster:\ | +| **Materialize** | redshift-data:DescribeStatement | \* | +| **Materialize** |

s3:ListBucket

s3:GetObject

s3:DeleteObject

|

arn:aws:s3:::<bucket_name>

arn:aws:s3:::<bucket_name>/*

| +| **Get Historical Features** |

redshift-data:ExecuteStatement

redshift:GetClusterCredentials

|

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

| +| **Get Historical Features** | redshift-data:DescribeStatement | \* | +| **Get Historical Features** |

s3:ListBucket

s3:GetObject

s3:PutObject

s3:DeleteObject

|

arn:aws:s3:::<bucket_name>

arn:aws:s3:::<bucket_name>/*

| + +The following inline policy can be used to grant Feast the necessary permissions: + +```javascript +{ + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::/*", + "arn:aws:s3:::" + ] + }, + { + "Action": [ + "redshift-data:DescribeTable", + "redshift:GetClusterCredentials", + "redshift-data:ExecuteStatement" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:redshift:::dbuser:/", + "arn:aws:redshift:::dbname:/", + "arn:aws:redshift:::cluster:" + ] + }, + { + "Action": [ + "redshift-data:DescribeStatement" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} +``` + +In addition to this, Redshift offline store requires an IAM role that will be used by Redshift itself to interact with S3. More concretely, Redshift has to use this IAM role to run [UNLOAD](https://docs.aws.amazon.com/redshift/latest/dg/r_UNLOAD.html) and [COPY](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) commands. Once created, this IAM role needs to be configured in `feature_store.yaml` file as `offline_store: iam_role`. + +The following inline policy can be used to grant Redshift necessary permissions to access S3: + +```javascript +{ + "Statement": [ + { + "Action": "s3:*", + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::feast-int-bucket", + "arn:aws:s3:::feast-int-bucket/*" + ] + } + ], + "Version": "2012-10-17" +} +``` + +While the following trust relationship is necessary to make sure that Redshift, and only Redshift can assume this role: + +```javascript +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "redshift.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + + +## Redshift Serverless + +In order to use [AWS Redshift Serverless](https://aws.amazon.com/redshift/redshift-serverless/), specify a workgroup instead of a cluster_id and user. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: aws +offline_store: + type: redshift + region: us-west-2 + workgroup: feast-workgroup + database: feast-database + s3_staging_location: s3://feast-bucket/redshift + iam_role: arn:aws:iam::123456789012:role/redshift_s3_access_role +``` +{% endcode %} + +Please note that the IAM policies above will need the [redshift-serverless](https://aws.permissions.cloud/iam/redshift-serverless) version, rather than the standard [redshift](https://aws.permissions.cloud/iam/redshift). \ No newline at end of file diff --git a/ui/public/docs/reference/offline-stores/remote-offline-store.md b/ui/public/docs/reference/offline-stores/remote-offline-store.md new file mode 100644 index 00000000000..8057ae32849 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/remote-offline-store.md @@ -0,0 +1,31 @@ +# Remote Offline Store + +## Description + +The Remote Offline Store is an Arrow Flight client for the offline store that implements the `RemoteOfflineStore` class using the existing `OfflineStore` interface. +The client implements various methods, including `get_historical_features`, `pull_latest_from_table_or_query`, `write_logged_features`, and `offline_write_batch`. + +## How to configure the client + +User needs to create client side `feature_store.yaml` file and set the `offline_store` type `remote` and provide the server connection configuration +including adding the host and specifying the port (default is 8815) required by the Arrow Flight client to connect with the Arrow Flight server. + +{% code title="feature_store.yaml" %} +```yaml +offline_store: + type: remote + host: localhost + port: 8815 +``` +{% endcode %} + +## Client Example + +The complete example can be find under [remote-offline-store-example](../../../examples/remote-offline-store) + +## How to configure the server + +Please see the detail how to configure offline feature server [offline-feature-server.md](../feature-servers/offline-feature-server.md) + +## How to configure Authentication and Authorization +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. diff --git a/ui/public/docs/reference/offline-stores/snowflake.md b/ui/public/docs/reference/offline-stores/snowflake.md new file mode 100644 index 00000000000..39bbe3f8a03 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/snowflake.md @@ -0,0 +1,82 @@ +# Snowflake offline store + +## Description + +The [Snowflake](https://trial.snowflake.com) offline store provides support for reading [SnowflakeSources](../data-sources/snowflake.md). +* All joins happen within Snowflake. +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Snowflake as a temporary table in order to complete join operations. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[snowflake]'`. + +If you're using a file based registry, then you'll also need to install the relevant cloud extra (`pip install 'feast[snowflake, CLOUD]'` where `CLOUD` is one of `aws`, `gcp`, `azure`) + +You can get started by then running `feast init -t snowflake`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +offline_store: + type: snowflake.offline + account: snowflake_deployment.us-east-1 + user: user_login + password: user_password + role: SYSADMIN + warehouse: COMPUTE_WH + database: FEAST + schema: PUBLIC +``` +{% endcode %} + +The full set of configuration options is available in [SnowflakeOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.snowflake.SnowflakeOfflineStoreConfig). + + +## Limitation +Please be aware that here is a restriction/limitation for using SQL query string in Feast with Snowflake. Try to avoid the usage of single quote in SQL query string. For example, the following query string will fail: +``` +SELECT + some_column +FROM + some_table +WHERE + other_column = 'value' +``` +That 'value' will fail in Snowflake. Instead, please use pairs of dollar signs like `$$value$$` as [mentioned in Snowflake document](https://docs.snowflake.com/en/sql-reference/data-types-text#label-dollar-quoted-string-constants). + + + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Snowflake offline store. + +| | Snowflake | +| :----------------------------------------------------------------- | :-------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | + +Below is a matrix indicating which functionality is supported by `SnowflakeRetrievalJob`. + +| | Snowflake | +| ----------------------------------------------------- | --------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | yes | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | yes | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/spark.md b/ui/public/docs/reference/offline-stores/spark.md new file mode 100644 index 00000000000..2e2facba64a --- /dev/null +++ b/ui/public/docs/reference/offline-stores/spark.md @@ -0,0 +1,72 @@ +# Spark offline store (contrib) + +## Description + +The Spark offline store provides support for reading [SparkSources](../data-sources/spark.md). + +* Entity dataframes can be provided as a SQL query, Pandas dataframe or can be provided as a Pyspark dataframe. A Pandas dataframes will be converted to a Spark dataframe and processed as a temporary view. + +## Disclaimer + +The Spark offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[spark]'`. You can get started by then running `feast init -t spark`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: spark + spark_conf: + spark.master: "local[*]" + spark.ui.enabled: "false" + spark.eventLog.enabled: "false" + spark.sql.catalogImplementation: "hive" + spark.sql.parser.quotedRegexColumnNames: "true" + spark.sql.session.timeZone: "UTC" + spark.sql.execution.arrow.fallback.enabled: "true" + spark.sql.execution.arrow.pyspark.enabled: "true" +online_store: + path: data/online_store.db +``` +{% endcode %} + +The full set of configuration options is available in [SparkOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.spark_offline_store.spark.SparkOfflineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Spark offline store. + +| | Spark | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `SparkRetrievalJob`. + +| | Spark | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | yes | +| local execution of Python-based on-demand transforms | no | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/offline-stores/trino.md b/ui/public/docs/reference/offline-stores/trino.md new file mode 100644 index 00000000000..fd437a7aa67 --- /dev/null +++ b/ui/public/docs/reference/offline-stores/trino.md @@ -0,0 +1,108 @@ +# Trino offline store (contrib) + +## Description + +The Trino offline store provides support for reading [TrinoSources](../data-sources/trino.md). +* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Trino as a table in order to complete join operations. + +## Disclaimer + +The Trino offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[trino]'`. You can then run `feast init`, then swap out `feature_store.yaml` with the below example to connect to Trino. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: feature_repo +registry: data/registry.db +provider: local +offline_store: + type: feast_trino.trino.TrinoOfflineStore + host: localhost + port: 8080 + catalog: memory + connector: + type: memory + user: trino + source: feast-trino-offline-store + http-scheme: https + ssl-verify: false + x-trino-extra-credential-header: foo=bar, baz=qux + + # enables authentication in Trino connections, pick the one you need + # if you don't need authentication, you can safely remove the whole auth block + auth: + # Basic Auth + type: basic + config: + username: foo + password: $FOO + + # Certificate + type: certificate + config: + cert-file: /path/to/cert/file + key-file: /path/to/key/file + + # JWT + type: jwt + config: + token: $JWT_TOKEN + + # OAuth2 (no config required) + type: oauth2 + + # Kerberos + type: kerberos + config: + config-file: /path/to/kerberos/config/file + service-name: foo + mutual-authentication: true + force-preemptive: true + hostname-override: custom-hostname + sanitize-mutual-error-response: true + principal: principal-name + delegate: true + ca_bundle: /path/to/ca/bundle/file +online_store: + path: data/online_store.db +``` +{% endcode %} + +The full set of configuration options is available in [TrinoOfflineStoreConfig](https://rtd.feast.dev/en/master/#trino-offline-store). + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Trino offline store. + +| | Trino | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `TrinoRetrievalJob`. + +| | Trino | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | no | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/README.md b/ui/public/docs/reference/online-stores/README.md new file mode 100644 index 00000000000..5df4710434c --- /dev/null +++ b/ui/public/docs/reference/online-stores/README.md @@ -0,0 +1,71 @@ +# Online stores + +Please see [Online Store](../../getting-started/components/online-store.md) for an explanation of online stores. + +{% content-ref url="overview.md" %} +[overview.md](overview.md) +{% endcontent-ref %} + +{% content-ref url="sqlite.md" %} +[sqlite.md](sqlite.md) +{% endcontent-ref %} + +{% content-ref url="snowflake.md" %} +[snowflake.md](snowflake.md) +{% endcontent-ref %} + +{% content-ref url="redis.md" %} +[redis.md](redis.md) +{% endcontent-ref %} + +{% content-ref url="dragonfly.md" %} +[dragonfly.md](dragonfly.md) +{% endcontent-ref %} + +{% content-ref url="ikv.md" %} +[ikv.md](ikv.md) +{% endcontent-ref %} + +{% content-ref url="datastore.md" %} +[datastore.md](datastore.md) +{% endcontent-ref %} + +{% content-ref url="dynamodb.md" %} +[dynamodb.md](dynamodb.md) +{% endcontent-ref %} + +{% content-ref url="bigtable.md" %} +[bigtable.md](mysql.md) +{% endcontent-ref %} + +{% content-ref url="postgres.md" %} +[postgres.md](postgres.md) +{% endcontent-ref %} + +{% content-ref url="cassandra.md" %} +[cassandra.md](cassandra.md) +{% endcontent-ref %} + +{% content-ref url="couchbase.md" %} +[couchbase.md](couchbase.md) +{% endcontent-ref %} + +{% content-ref url="mysql.md" %} +[mysql.md](mysql.md) +{% endcontent-ref %} + +{% content-ref url="hazelcast.md" %} +[hazelcast.md](hazelcast.md) +{% endcontent-ref %} + +{% content-ref url="scylladb.md" %} +[scylladb.md](scylladb.md) +{% endcontent-ref %} + +{% content-ref url="remote.md" %} +[remote.md](remote.md) +{% endcontent-ref %} + +{% content-ref url="singlestore.md" %} +[singlestore.md](singlestore.md) +{% endcontent-ref %} diff --git a/ui/public/docs/reference/online-stores/bigtable.md b/ui/public/docs/reference/online-stores/bigtable.md new file mode 100644 index 00000000000..0d6e7cfb13b --- /dev/null +++ b/ui/public/docs/reference/online-stores/bigtable.md @@ -0,0 +1,56 @@ +# Bigtable online store + +## Description + +The [Bigtable](https://cloud.google.com/bigtable) online store provides support for +materializing feature values into Cloud Bigtable. The data model used to store feature +values in Bigtable is described in more detail +[here](../../specs/online_store_format.md#google-bigtable-online-store-format). + +## Getting started + +In order to use this online store, you'll need to run `pip install 'feast[gcp]'`. You +can then get started with the command `feast init REPO_NAME -t gcp`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: gcp +online_store: + type: bigtable + project_id: my_gcp_project + instance: my_bigtable_instance +``` +{% endcode %} + +The full set of configuration options is available in +[BigtableOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.bigtable.BigtableOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Bigtable online store. + +| | Bigtable | +|-----------------------------------------------------------|----------| +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/cassandra.md b/ui/public/docs/reference/online-stores/cassandra.md new file mode 100644 index 00000000000..198f15ca47f --- /dev/null +++ b/ui/public/docs/reference/online-stores/cassandra.md @@ -0,0 +1,92 @@ +# Cassandra + Astra DB online store + +## Description + +The [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] online store provides support for materializing feature values into an Apache Cassandra / Astra DB database for online features. + +* The whole project is contained within a Cassandra keyspace +* Each feature view is mapped one-to-one to a specific Cassandra table +* This implementation inherits all strengths of Cassandra such as high availability, fault-tolerance, and data distribution + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[cassandra]'`. You can then get started with the command `feast init REPO_NAME -t cassandra`. + +### Example (Cassandra) + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - 192.168.1.1 + - 192.168.1.2 + - 192.168.1.3 + keyspace: KeyspaceName + port: 9042 # optional + username: user # optional + password: secret # optional + protocol_version: 5 # optional + load_balancing: # optional + local_dc: 'datacenter1' # optional + load_balancing_policy: 'TokenAwarePolicy(DCAwareRoundRobinPolicy)' # optional + read_concurrency: 100 # optional + write_concurrency: 100 # optional +``` +{% endcode %} + +### Example (Astra DB) + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + secure_bundle_path: /path/to/secure/bundle.zip + keyspace: KeyspaceName + username: Client_ID + password: Client_Secret + protocol_version: 4 # optional + load_balancing: # optional + local_dc: 'eu-central-1' # optional + load_balancing_policy: 'TokenAwarePolicy(DCAwareRoundRobinPolicy)' # optional + read_concurrency: 100 # optional + write_concurrency: 100 # optional +``` +{% endcode %} + +The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). +For a full explanation of configuration options please look at file +`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. + +Storage specifications can be found at `docs/specs/online_store_format.md`. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Cassandra online store. + +| | Cassandra | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/couchbase.md b/ui/public/docs/reference/online-stores/couchbase.md new file mode 100644 index 00000000000..2878deb97ee --- /dev/null +++ b/ui/public/docs/reference/online-stores/couchbase.md @@ -0,0 +1,78 @@ +# Couchbase Online Store +> NOTE: +> This is a community-contributed online store that is in alpha development. It is not officially supported by the Feast project. + +## Description +The [Couchbase](https://www.couchbase.com/) online store provides support for materializing feature values into a Couchbase Operational cluster for serving online features in real-time. + +* Only the latest feature values are persisted +* Features are stored in a document-oriented format + +The data model for using Couchbase as an online store follows a document format: +* Document ID: `{project}:{table_name}:{entity_key_hex}:{feature_name}` +* Document Content: + * `metadata`: + * `event_ts` (ISO formatted timestamp) + * `created_ts` (ISO formatted timestamp) + * `feature_name` (String) + * `value` (Base64 encoded protobuf binary) + + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[couchbase]'`. You can then get started with the command `feast init REPO_NAME -t couchbase`. + +To get started with Couchbase Capella Operational: +1. [Sign up for a Couchbase Capella account](https://docs.couchbase.com/cloud/get-started/create-account.html#sign-up-free-tier) +2. [Deploy an Operational cluster](https://docs.couchbase.com/cloud/get-started/create-account.html#getting-started) +3. [Create a bucket](https://docs.couchbase.com/cloud/clusters/data-service/manage-buckets.html#add-bucket) + - This can be named anything, but must correspond to the bucket described in the `feature_store.yaml` configuration file. +4. [Create cluster access credentials](https://docs.couchbase.com/cloud/clusters/manage-database-users.html#create-database-credentials) + - These credentials should have full access to the bucket created in step 3. +5. [Configure allowed IP addresses](https://docs.couchbase.com/cloud/clusters/allow-ip-address.html) + - You must allow the IP address of the machine running Feast. + +## Example +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: couchbase.online + connection_string: couchbase://127.0.0.1 # Couchbase connection string, copied from 'Connect' page in Couchbase Capella console + user: Administrator # Couchbase username from access credentials + password: password # Couchbase password from access credentials + bucket_name: feast # Couchbase bucket name, defaults to feast + kv_port: 11210 # Couchbase key-value port, defaults to 11210. Required if custom ports are used. +entity_key_serialization_version: 2 +``` +{% endcode %} + +The full set of configuration options is available in `CouchbaseOnlineStoreConfig`. + + +## Functionality Matrix +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Couchbase online store. + +| | Couchbase | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + diff --git a/ui/public/docs/reference/online-stores/datastore.md b/ui/public/docs/reference/online-stores/datastore.md new file mode 100644 index 00000000000..761d246ba7e --- /dev/null +++ b/ui/public/docs/reference/online-stores/datastore.md @@ -0,0 +1,50 @@ +# Datastore online store + +## Description + +The [Datastore](https://cloud.google.com/datastore) online store provides support for materializing feature values into Cloud Datastore. The data model used to store feature values in Datastore is described in more detail [here](../../specs/online_store_format.md#google-datastore-online-store-format). + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[gcp]'`. You can then get started with the command `feast init REPO_NAME -t gcp`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: gcp +online_store: + type: datastore + project_id: my_gcp_project + namespace: my_datastore_namespace +``` +{% endcode %} + +The full set of configuration options is available in [DatastoreOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.datastore.DatastoreOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Datastore online store. + +| | Datastore | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/dragonfly.md b/ui/public/docs/reference/online-stores/dragonfly.md new file mode 100644 index 00000000000..bcd814ecc45 --- /dev/null +++ b/ui/public/docs/reference/online-stores/dragonfly.md @@ -0,0 +1,90 @@ +# Dragonfly online store + +## Description + +[Dragonfly](https://github.com/dragonflydb/dragonfly) is a modern in-memory datastore that implements novel algorithms and data structures on top of a multi-threaded, share-nothing architecture. Thanks to its API compatibility, Dragonfly can act as a drop-in replacement for Redis. Due to Dragonfly's hardware efficiency, you can run a single node instance on a small 8GB instance or scale vertically to large 768GB machines with 64 cores. This greatly reduces infrastructure costs as well as architectural complexity. + +Similar to Redis, Dragonfly can be used as an online feature store for Feast. + +## Using Dragonfly as a drop-in Feast online store instead of Redis + +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI + +`pip install feast` + +In order to use Dragonfly as the online store, you'll need to install the redis extra: + +`pip install 'feast[redis]'` + +### 1. Create a feature repository + +Bootstrap a new feature repository: + +``` +feast init feast_dragonfly +cd feast_dragonfly/feature_repo +``` + +Update `feature_repo/feature_store.yaml` with the below contents: + +``` +project: feast_dragonfly +registry: data/registry.db +provider: local +online_store: +type: redis +connection_string: "localhost:6379" +``` + +### 2. Start Dragonfly + +There are several options available to get Dragonfly up and running quickly. We will be using Docker for this tutorial. + +`docker run --network=host --ulimit memlock=-1 docker.dragonflydb.io/dragonflydb/dragonfly` + +### 3. Register feature definitions and deploy your feature store + +`feast apply` + +The `apply` command scans python files in the current directory (`example_repo.py` in this case) for feature view/entity definitions, registers the objects, and deploys infrastructure. +You should see the following output: + +``` +.... +Created entity driver +Created feature view driver_hourly_stats_fresh +Created feature view driver_hourly_stats +Created on demand feature view transformed_conv_rate +Created on demand feature view transformed_conv_rate_fresh +Created feature service driver_activity_v1 +Created feature service driver_activity_v3 +Created feature service driver_activity_v2 +``` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redis online store. + +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/dynamodb.md b/ui/public/docs/reference/online-stores/dynamodb.md new file mode 100644 index 00000000000..344caccac1d --- /dev/null +++ b/ui/public/docs/reference/online-stores/dynamodb.md @@ -0,0 +1,84 @@ +# DynamoDB online store + +## Description + +The [DynamoDB](https://aws.amazon.com/dynamodb/) online store provides support for materializing feature values into AWS DynamoDB. + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[aws]'`. You can then get started with the command `feast init REPO_NAME -t aws`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: aws +online_store: + type: dynamodb + region: us-west-2 +``` +{% endcode %} + +The full set of configuration options is available in [DynamoDBOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.dynamodb.DynamoDBOnlineStoreConfig). + +## Permissions + +Feast requires the following permissions in order to execute commands for DynamoDB online store: + +| **Command** | Permissions | Resources | +| ----------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------- | +| **Apply** |

dynamodb:CreateTable

dynamodb:DescribeTable

dynamodb:DeleteTable

| arn:aws:dynamodb:\:\:table/\* | +| **Materialize** | dynamodb.BatchWriteItem | arn:aws:dynamodb:\:\:table/\* | +| **Get Online Features** | dynamodb.BatchGetItem | arn:aws:dynamodb:\:\:table/\* | + +The following inline policy can be used to grant Feast the necessary permissions: + +```javascript +{ + "Statement": [ + { + "Action": [ + "dynamodb:CreateTable", + "dynamodb:DescribeTable", + "dynamodb:DeleteTable", + "dynamodb:BatchWriteItem", + "dynamodb:BatchGetItem" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:dynamodb:::table/*" + ] + } + ], + "Version": "2012-10-17" +} +``` + +Lastly, this IAM role needs to be associated with the desired Redshift cluster. Please follow the official AWS guide for the necessary steps [here](https://docs.aws.amazon.com/redshift/latest/dg/c-getting-started-using-spectrum-add-role.html). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the DynamoDB online store. + +| | DynamoDB | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/elasticsearch.md b/ui/public/docs/reference/online-stores/elasticsearch.md new file mode 100644 index 00000000000..683fbf6ae00 --- /dev/null +++ b/ui/public/docs/reference/online-stores/elasticsearch.md @@ -0,0 +1,139 @@ +# ElasticSearch online store + +## Description + +The ElasticSearch online store provides support for materializing tabular feature values, as well as embedding feature vectors, into an ElasticSearch index for serving online features. \ +The embedding feature vectors are stored as dense vectors, and can be used for similarity search. More information on dense vectors can be found [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html). + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[elasticsearch]'`. You can get started by then running `feast init -t elasticsearch`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: elasticsearch + host: ES_HOST + port: ES_PORT + user: ES_USERNAME + password: ES_PASSWORD + write_batch_size: 1000 +``` +{% endcode %} + +The full set of configuration options is available in [ElasticsearchOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.elasticsearch_online_store.ElasticsearchOnlineStoreConfig). + +## Functionality Matrix + + +| | Postgres | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Retrieving online document vectors + +The ElasticSearch online store supports retrieving document vectors for a given list of entity keys. The document vectors are returned as a dictionary where the key is the entity key and the value is the document vector. The document vector is a dense vector of floats. + +{% code title="python" %} +```python +from feast import FeatureStore + +feature_store = FeatureStore(repo_path="feature_store.yaml") + +query_vector = [1.0, 2.0, 3.0, 4.0, 5.0] +top_k = 5 + +# Retrieve the top k closest features to the query vector + +feature_values = feature_store.retrieve_online_documents_v2( + features=["my_feature"], + query=query_vector, + top_k=top_k, +) +``` +{% endcode %} + +## Indexing +Currently, the indexing mapping in the ElasticSearch online store is configured as: + +{% code title="indexing_mapping" %} +```json +{ + "dynamic_templates": [ + { + "feature_objects": { + "match_mapping_type": "object", + "match": "*", + "mapping": { + "type": "object", + "properties": { + "feature_value": {"type": "binary"}, + "value_text": {"type": "text"}, + "vector_value": { + "type": "dense_vector", + "dims": vector_field_length, + "index": True, + "similarity": config.online_store.similarity, + }, + }, + }, + } + } + ], + "properties": { + "entity_key": {"type": "keyword"}, + "timestamp": {"type": "date"}, + "created_ts": {"type": "date"}, + }, +} +``` +{% endcode %} +And the online_read API mapping is configured as: + +{% code title="online_read_mapping" %} +```json +"query": { + "bool": { + "must": [ + {"terms": {"entity_key": entity_keys}}, + {"terms": {"feature_name": requested_features}}, + ] + } +}, +``` +{% endcode %} + +And the similarity search API mapping is configured as: + +{% code title="similarity_search_mapping" %} +```json +{ + "field": "vector_value", + "query_vector": embedding_vector, + "k": top_k, +} +``` +{% endcode %} + +These APIs are subject to change in future versions of Feast to improve performance and usability. \ No newline at end of file diff --git a/ui/public/docs/reference/online-stores/hazelcast.md b/ui/public/docs/reference/online-stores/hazelcast.md new file mode 100644 index 00000000000..c2fb2d898a8 --- /dev/null +++ b/ui/public/docs/reference/online-stores/hazelcast.md @@ -0,0 +1,58 @@ +# Hazelcast online store + +## Description + +The [Hazelcast](htpps://hazelcast.com) online store provides support for materializing feature values into a Hazelcast cluster for serving online features in real-time. +In order to use Hazelcast as an online store, you need to have a running Hazelcast cluster. See this [getting started](https://hazelcast.com/get-started/) page for more details. + +* Each feature view is mapped one-to-one to a specific Hazelcast IMap +* This implementation inherits all strengths of Hazelcast such as high availability, fault-tolerance, and data distribution. +* Secure TSL/SSL connection is supported by Hazelcast online store. +* You can set TTL (Time-To-Live) setting for your features in Hazelcast cluster. + +Each feature view corresponds to an IMap in Hazelcast cluster and the entries in that IMap correspond to features of entities. +Each feature value stored separately and can be retrieved individually. + +## Getting started + +In order to use Hazelcast online store, you'll need to run `pip install 'feast[hazelcast]'`. You can then get started with the command `feast init REPO_NAME -t hazelcast`. + + +## Examples + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: hazelcast + cluster_name: dev + cluster_members: ["localhost:5701"] + key_ttl_seconds: 36000 +``` +{% endcode %} + +## Functionality Matrix + +| | Hazelcast | +| :-------------------------------------------------------- |:----------| +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + diff --git a/ui/public/docs/reference/online-stores/ikv.md b/ui/public/docs/reference/online-stores/ikv.md new file mode 100644 index 00000000000..79f21d17797 --- /dev/null +++ b/ui/public/docs/reference/online-stores/ikv.md @@ -0,0 +1,69 @@ +# IKV (Inlined Key-Value Store) online store + +## Description + +[IKV](https://github.com/inlinedio/ikv-store) is a fully-managed embedded key-value store, primarily designed for storing ML features. Most key-value stores (think Redis or Cassandra) need a remote database cluster, whereas IKV allows you to utilize your existing application infrastructure to store data (cost efficient) and access it without any network calls (better performance). See detailed performance benchmarks and cost comparison with Redis on [https://inlined.io](https://inlined.io). IKV can be used as an online-store in Feast, the rest of this guide goes over the setup. + +## Getting started +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI: `pip install feast` + +In order to use this online store, you'll need to install the IKV extra (along with the dependency needed for the offline store of choice). E.g. +- `pip install 'feast[gcp, ikv]'` +- `pip install 'feast[snowflake, ikv]'` +- `pip install 'feast[aws, ikv]'` +- `pip install 'feast[azure, ikv]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in IKV as the online store as seen below in the examples. + +### 1. Provision an IKV store +Go to [https://inlined.io](https://inlined.io) or email onboarding[at]inlined.io + +### 2. Configure + +Update `my_feature_repo/feature_store.yaml` with the below contents: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: ikv + account_id: secret + account_passkey: secret + store_name: your-store-name + mount_directory: /absolute/path/on/disk/for/ikv/embedded/index +``` +{% endcode %} + +After provisioning an IKV account/store, you should have an account id, passkey and store-name. Additionally you must specify a mount-directory - where IKV will pull/update (maintain) a copy of the index for online reads (IKV is an embedded database). It can be skipped only if you don't plan to read any data from this container. The mount directory path usually points to a location on local/remote disk. + +The full set of configuration options is available in IKVOnlineStoreConfig at `sdk/python/feast/infra/online_stores/contrib/ikv_online_store/ikv.py` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the IKV online store. + +| | IKV | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/milvus.md b/ui/public/docs/reference/online-stores/milvus.md new file mode 100644 index 00000000000..014c7bd68a5 --- /dev/null +++ b/ui/public/docs/reference/online-stores/milvus.md @@ -0,0 +1,65 @@ +# Redis online store + +## Description + +The [Milvus](https://milvus.io/) online store provides support for materializing feature values into Milvus. + +* The data model used to store feature values in Milvus is described in more detail [here](../../specs/online\_store\_format.md). + +## Getting started +In order to use this online store, you'll need to install the Milvus extra (along with the dependency needed for the offline store of choice). E.g. + +`pip install 'feast[milvus]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. + +## Examples + +Connecting to a local MilvusDB instance: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: milvus + path: "data/online_store.db" + connection_string: "localhost:6379" + embedding_dim: 128 + index_type: "FLAT" + metric_type: "COSINE" + username: "username" + password: "password" +``` +{% endcode %} + + +The full set of configuration options is available in [MilvusOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.milvus.MilvusOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Milvus online store. + +| | Milvus | +|:----------------------------------------------------------|:-------| +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | no | +| vector similarity search | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/mysql.md b/ui/public/docs/reference/online-stores/mysql.md new file mode 100644 index 00000000000..8868e64279d --- /dev/null +++ b/ui/public/docs/reference/online-stores/mysql.md @@ -0,0 +1,55 @@ +# MySQL online store + +## Description + +The MySQL online store provides support for materializing feature values into a MySQL database for serving online features. + +* Only the latest feature values are persisted + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[mysql]'`. You can get started by then running `feast init` and then setting the `feature_store.yaml` as described below. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD +``` +{% endcode %} + +The full set of configuration options is available in [MySQLOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.mysql_online_store.MySQLOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Mys online store. + +| | Mys | +| :-------------------------------------------------------- | :--- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/overview.md b/ui/public/docs/reference/online-stores/overview.md new file mode 100644 index 00000000000..b54329ad613 --- /dev/null +++ b/ui/public/docs/reference/online-stores/overview.md @@ -0,0 +1,54 @@ +# Overview + +## Functionality + +Here are the methods exposed by the `OnlineStore` interface, along with the core functionality supported by the method: +* `online_write_batch`: write feature values to the online store +* `online_read`: read feature values from the online store +* `update`: update infrastructure (e.g. tables) in the online store +* `teardown`: teardown infrastructure (e.g. tables) in the online store +* `plan`: generate a plan of infrastructure changes based on feature repo changes + +There is also additional functionality not properly captured by these interface methods: +* support for on-demand transforms +* readable by Python SDK +* readable by Java +* readable by Go +* support for entityless feature views +* support for concurrent writing to the same key +* support for ttl (time to live) at retrieval +* support for deleting expired data + +Finally, there are multiple data models for storing the features in the online store. For example, features could be: +* collocated by feature view +* collocated by feature service +* collocated by entity key + +See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussion around the tradeoffs of each of these data models. + +## Functionality Matrix + +There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, `CassandraOnlineStore` and `IKVOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). + +Below is a matrix indicating which online stores support what functionality. + +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | [IKV](https://inlined.io) | Milvus | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:-------| +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | no | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | no | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | no | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | no | +| support for deleting expired data | no | yes | no | no | no | no | no | no | no | no | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | no | +| collocated by feature service | no | no | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | yes | no | diff --git a/ui/public/docs/reference/online-stores/postgres.md b/ui/public/docs/reference/online-stores/postgres.md new file mode 100644 index 00000000000..fb2253d043e --- /dev/null +++ b/ui/public/docs/reference/online-stores/postgres.md @@ -0,0 +1,95 @@ +# PostgreSQL online store + +## Description + +The PostgreSQL online store provides support for materializing feature values into a PostgreSQL database for serving online features. + +* Only the latest feature values are persisted + +* sslmode, sslkey_path, sslcert_path, and sslrootcert_path are optional + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[postgres]'`. You can get started by then running `feast init -t postgres`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: postgres + host: DB_HOST + port: DB_PORT + database: DB_NAME + db_schema: DB_SCHEMA + user: DB_USERNAME + password: DB_PASSWORD + sslmode: verify-ca + sslkey_path: /path/to/client-key.pem + sslcert_path: /path/to/client-cert.pem + sslrootcert_path: /path/to/server-ca.pem + vector_enabled: false +``` +{% endcode %} + +The full set of configuration options is available in [PostgreSQLOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.postgres_online_store.PostgreSQLOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Postgres online store. + +| | Postgres | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## PGVector +The Postgres online store supports the use of [PGVector](https://github.com/pgvector/pgvector) for storing feature values. +To enable PGVector, set `vector_enabled: true` in the online store configuration. + +The `vector_length` parameter can be used to specify the length of the vector in the Field. + +Please make sure to follow the instructions in the repository, which, as the time of this writing, requires you to +run `CREATE EXTENSION vector;` in the database. + + +Then you can use `retrieve_online_documents` to retrieve the top k closest vectors to a query vector. +For the Retrieval Augmented Generation (RAG) use-case, you have to embed the query prior to passing the query vector. + +{% code title="python" %} +```python +from feast import FeatureStore +from feast.infra.online_stores.postgres_online_store import retrieve_online_documents + +feature_store = FeatureStore(repo_path=".") + +query_vector = [0.1, 0.2, 0.3, 0.4, 0.5] +top_k = 5 + +feature_values = retrieve_online_documents( + feature_store=feature_store, + feature_view_name="document_fv:embedding_float", + query_vector=query_vector, + top_k=top_k, +) +``` +{% endcode %} diff --git a/ui/public/docs/reference/online-stores/qdrant.md b/ui/public/docs/reference/online-stores/qdrant.md new file mode 100644 index 00000000000..54c4c80517c --- /dev/null +++ b/ui/public/docs/reference/online-stores/qdrant.md @@ -0,0 +1,80 @@ +# Qdrant online store + +## Description + +[Qdrant](http://qdrant.tech) is a vector similarity search engine. It provides a production-ready service with a convenient API to store, search, and manage vectors with additional payload and extended filtering support. It makes it useful for all sorts of neural network or semantic-based matching, faceted search, and other applications. + +## Getting started + +In order to use this online store, you'll need to run `pip install 'feast[qdrant]'`. + +## Example + +{% code title="feature_store.yaml" %} + +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: qdrant + host: localhost + port: 6333 + write_batch_size: 100 +``` + +{% endcode %} + +The full set of configuration options is available in [QdrantOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.qdrant_online_store.QdrantOnlineStoreConfig). + +## Functionality Matrix + +| | Qdrant | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Retrieving online document vectors + +The Qdrant online store supports retrieving document vectors for a given list of entity keys. The document vectors are returned as a dictionary where the key is the entity key and the value is the document vector. The document vector is a dense vector of floats. + +{% code title="python" %} + +```python +from feast import FeatureStore + +feature_store = FeatureStore(repo_path="feature_store.yaml") + +query_vector = [1.0, 2.0, 3.0, 4.0, 5.0] +top_k = 5 + +# Retrieve the top k closest features to the query vector +# Since Qdrant supports multiple vectors per entry, +# the vector to use can be specified in the repo config. +# Reference: https://qdrant.tech/documentation/concepts/vectors/#named-vectors +feature_values = feature_store.retrieve_online_documents( + features=["my_feature"], + query=query_vector, + top_k=top_k +) +``` + +{% endcode %} + +These APIs are subject to change in future versions of Feast to improve performance and usability. diff --git a/ui/public/docs/reference/online-stores/redis.md b/ui/public/docs/reference/online-stores/redis.md new file mode 100644 index 00000000000..ae7f8b4c5ca --- /dev/null +++ b/ui/public/docs/reference/online-stores/redis.md @@ -0,0 +1,106 @@ +# Redis online store + +## Description + +The [Redis](https://redis.io) online store provides support for materializing feature values into Redis. + +* Both Redis and Redis Cluster are supported. +* The data model used to store feature values in Redis is described in more detail [here](../../specs/online\_store\_format.md). + +## Getting started +In order to use this online store, you'll need to install the redis extra (along with the dependency needed for the offline store of choice). E.g. +- `pip install 'feast[gcp, redis]'` +- `pip install 'feast[snowflake, redis]'` +- `pip install 'feast[aws, redis]'` +- `pip install 'feast[azure, redis]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. + +## Examples + +Connecting to a single Redis instance: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: "localhost:6379" +``` +{% endcode %} + +Connecting to a Redis Cluster with SSL enabled and password authentication: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + redis_type: redis_cluster + connection_string: "redis1:6379,redis2:6379,ssl=true,password=my_password" +``` +{% endcode %} + +Connecting to a Redis Sentinel with SSL enabled and password authentication: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + redis_type: redis_sentinel + sentinel_master: mymaster + connection_string: "redis1:26379,ssl=true,password=my_password" +``` +{% endcode %} + +Additionally, the redis online store also supports automatically deleting data via a TTL mechanism. +The TTL is applied at the entity level, so feature values from any associated feature views for an entity are removed together. +This TTL can be set in the `feature_store.yaml`, using the `key_ttl_seconds` field in the online store. For example: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + key_ttl_seconds: 604800 + connection_string: "localhost:6379" +``` +{% endcode %} + + +The full set of configuration options is available in [RedisOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.redis.RedisOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redis online store. + +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/remote.md b/ui/public/docs/reference/online-stores/remote.md new file mode 100644 index 00000000000..aa97a495baa --- /dev/null +++ b/ui/public/docs/reference/online-stores/remote.md @@ -0,0 +1,30 @@ +# Remote online store + +## Description + +This remote online store will let you interact with remote feature server. At this moment this only supports the read operation. You can use this online store and able retrieve online features `store.get_online_features` from remote feature server. + +## Examples + +The registry is pointing to registry of remote feature store. If it is not accessible then should be configured to use remote registry. + +{% code title="feature_store.yaml" %} +```yaml +project: my-local-project +registry: /remote/data/registry.db +provider: local +online_store: + path: http://localhost:6566 + type: remote + cert: /path/to/cert.pem +entity_key_serialization_version: 2 +auth: + type: no_auth +``` +{% endcode %} + +`cert` is an optional configuration to the public certificate path when the online server starts in TLS(SSL) mode. This may be needed if the online server is started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`. + +## How to configure Authentication and Authorization +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. + diff --git a/ui/public/docs/reference/online-stores/scylladb.md b/ui/public/docs/reference/online-stores/scylladb.md new file mode 100644 index 00000000000..c8583ac101a --- /dev/null +++ b/ui/public/docs/reference/online-stores/scylladb.md @@ -0,0 +1,94 @@ +# ScyllaDB Cloud online store + +## Description + +ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast. + +The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time. + +## Getting started + +Install Feast with Cassandra support: +```bash +pip install "feast[cassandra]" +``` + +Create a new Feast project: +```bash +feast init REPO_NAME -t cassandra +``` + +### Example (ScyllaDB) + +{% code title="feature_store.yaml" %} +```yaml +project: scylla_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - 172.17.0.2 + keyspace: feast + username: scylla + password: password +``` +{% endcode %} + +### Example (ScyllaDB Cloud) + +{% code title="feature_store.yaml" %} +```yaml +project: scylla_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + keyspace: feast + username: scylla + password: password +``` +{% endcode %} + + +The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). +For a full explanation of configuration options please look at file +`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. + +Storage specifications can be found at `docs/specs/online_store_format.md`. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Cassandra plugin. + +| | Cassandra | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Resources + +* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/) +* [ScyllaDB website](https://www.scylladb.com/) +* [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/) diff --git a/ui/public/docs/reference/online-stores/singlestore.md b/ui/public/docs/reference/online-stores/singlestore.md new file mode 100644 index 00000000000..7272233b9dc --- /dev/null +++ b/ui/public/docs/reference/online-stores/singlestore.md @@ -0,0 +1,51 @@ +# SingleStore online store + +## Description + +The SingleStore online store provides support for materializing feature values into a SingleStore database for serving online features. + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[singlestore]'`. You can get started by then running `feast init` and then setting the `feature_store.yaml` as described below. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: singlestore + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD +``` +{% endcode %} + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the SingleStore online store. + +| | SingleStore | +| :-------------------------------------------------------- | :----------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/snowflake.md b/ui/public/docs/reference/online-stores/snowflake.md new file mode 100644 index 00000000000..6b6d107285c --- /dev/null +++ b/ui/public/docs/reference/online-stores/snowflake.md @@ -0,0 +1,78 @@ +# Snowflake online store + +## Description + +The [Snowflake](https://trial.snowflake.com) online store provides support for materializing feature values into a Snowflake Transient Table for serving online features. + +* Only the latest feature values are persisted + +The data model for using a Snowflake Transient Table as an online store follows a tall format (one row per feature)): +* "entity_feature_key" (BINARY) -- unique key used when reading specific feature_view x entity combination +* "entity_key" (BINARY) -- repeated key currently unused for reading entity_combination +* "feature_name" (VARCHAR) +* "value" (BINARY) +* "event_ts" (TIMESTAMP) +* "created_ts" (TIMESTAMP) + + (This model may be subject to change when Snowflake Hybrid Tables are released) + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[snowflake]'`. You can then get started with the command `feast init REPO_NAME -t snowflake`. + +## Example +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: snowflake.online + account: SNOWFLAKE_DEPLOYMENT_URL + user: SNOWFLAKE_USER + password: SNOWFLAKE_PASSWORD + role: SNOWFLAKE_ROLE + warehouse: SNOWFLAKE_WAREHOUSE + database: SNOWFLAKE_DATABASE +``` +{% endcode %} + +## Tags KWARGs Actions: + +"snowflake-online-store/online_path": Adding the "snowflake-online-store/online_path" key to a FeatureView tags parameter allows you to choose the online table path for the online serving table (ex. "{database}"."{schema}"). + +{% code title="example_config.py" %} +```python +driver_stats_fv = FeatureView( + ... + tags={"snowflake-online-store/online_path": '"FEAST"."ONLINE"'}, +) +``` +{% endcode %} + +The full set of configuration options is available in [SnowflakeOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.snowflake.SnowflakeOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Snowflake online store. + +| | Snowflake | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/online-stores/sqlite.md b/ui/public/docs/reference/online-stores/sqlite.md new file mode 100644 index 00000000000..859702c07f1 --- /dev/null +++ b/ui/public/docs/reference/online-stores/sqlite.md @@ -0,0 +1,49 @@ +# SQLite online store + +## Description + +The [SQLite](https://www.sqlite.org/index.html) online store provides support for materializing feature values into an SQLite database for serving online features. + +* All feature values are stored in an on-disk SQLite database +* Only the latest feature values are persisted + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +``` +{% endcode %} + +The full set of configuration options is available in [SqliteOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.sqlite.SqliteOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Sqlite online store. + +| | Sqlite | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/ui/public/docs/reference/providers/README.md b/ui/public/docs/reference/providers/README.md new file mode 100644 index 00000000000..925ae8ebc1f --- /dev/null +++ b/ui/public/docs/reference/providers/README.md @@ -0,0 +1,11 @@ +# Providers + +Please see [Provider](../../getting-started/components/provider.md) for an explanation of providers. + +{% page-ref page="local.md" %} + +{% page-ref page="google-cloud-platform.md" %} + +{% page-ref page="amazon-web-services.md" %} + +{% page-ref page="azure.md" %} diff --git a/ui/public/docs/reference/providers/amazon-web-services.md b/ui/public/docs/reference/providers/amazon-web-services.md new file mode 100644 index 00000000000..68956a1be93 --- /dev/null +++ b/ui/public/docs/reference/providers/amazon-web-services.md @@ -0,0 +1,32 @@ +# Amazon Web Services + +## Description + +* Offline Store: Uses the **Redshift** offline store by default. Also supports File as the offline store. +* Online Store: Uses the **DynamoDB** online store by default. Also supports Sqlite as an online store. + +## Getting started +In order to use this offline store, you'll need to run (Snowflake) `pip install 'feast[aws, snowflake]'` or (Redshift) `pip install 'feast[aws]'`. + +You can get started by then running `feast init -t snowflake` or `feast init -t aws`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: aws +online_store: + type: dynamodb + region: us-west-2 +offline_store: + type: redshift + region: us-west-2 + cluster_id: feast-cluster + database: feast-database + user: redshift-user + s3_staging_location: s3://feast-bucket/redshift + iam_role: arn:aws:iam::123456789012:role/redshift_s3_access_role +``` +{% endcode %} diff --git a/ui/public/docs/reference/providers/azure.md b/ui/public/docs/reference/providers/azure.md new file mode 100644 index 00000000000..0e7206f076e --- /dev/null +++ b/ui/public/docs/reference/providers/azure.md @@ -0,0 +1,29 @@ +# Azure (contrib) + +## Description + +* Offline Store: Uses the **MsSql** offline store by default. Also supports File as the offline store. +* Online Store: Uses the **Redis** online store by default. Also supports Sqlite as an online store. + +## Disclaimer + +The Azure provider does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[azure]'`. You can get started by then following this [tutorial](https://github.com/feast-dev/feast/blob/master/docs/tutorials/azure/README.md). + +## Example + +{% code title="feature_store.yaml" %} +```yaml +registry: + registry_store_type: AzureRegistryStore + path: ${REGISTRY_PATH} # Environment Variable +project: production +provider: azure +online_store: + type: redis + connection_string: ${REDIS_CONN} # Environment Variable +``` +{% endcode %} \ No newline at end of file diff --git a/ui/public/docs/reference/providers/google-cloud-platform.md b/ui/public/docs/reference/providers/google-cloud-platform.md new file mode 100644 index 00000000000..96af3b6b2ff --- /dev/null +++ b/ui/public/docs/reference/providers/google-cloud-platform.md @@ -0,0 +1,30 @@ +# Google Cloud Platform + +## Description + +* Offline Store: Uses the **BigQuery** offline store by default. Also supports File as the offline store. +* Online Store: Uses the **Datastore** online store by default. Also supports Sqlite as an online store. + +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[gcp]'`. You can get started by then running `feast init -t gcp`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: gs://my-bucket/data/registry.db +provider: gcp +``` +{% endcode %} + +## **Permissions** + +| **Command** | Component | Permissions | Recommended Role | +| --------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | +| **Apply** | BigQuery (source) |

bigquery.jobs.create

bigquery.readsessions.create

bigquery.readsessions.getData

| roles/bigquery.user | +| **Apply** | Datastore (destination) |

datastore.entities.allocateIds

datastore.entities.create

datastore.entities.delete

datastore.entities.get

datastore.entities.list

datastore.entities.update

| roles/datastore.owner | +| **Materialize** | BigQuery (source) | bigquery.jobs.create | roles/bigquery.user | +| **Materialize** | Datastore (destination) |

datastore.entities.allocateIds

datastore.entities.create

datastore.entities.delete

datastore.entities.get

datastore.entities.list

datastore.entities.update

datastore.databases.get

| roles/datastore.owner | +| **Get Online Features** | Datastore | datastore.entities.get | roles/datastore.user | +| **Get Historical Features** | BigQuery (source) |

bigquery.datasets.get

bigquery.tables.get

bigquery.tables.create

bigquery.tables.updateData

bigquery.tables.update

bigquery.tables.delete

bigquery.tables.getData

| roles/bigquery.dataEditor | diff --git a/ui/public/docs/reference/providers/local.md b/ui/public/docs/reference/providers/local.md new file mode 100644 index 00000000000..a93a3b8b2d1 --- /dev/null +++ b/ui/public/docs/reference/providers/local.md @@ -0,0 +1,16 @@ +# Local + +## Description + +* Offline Store: Uses the File offline store by default. Also supports BigQuery as the offline store. +* Online Store: Uses the Sqlite online store by default. Also supports Redis and Datastore as online stores. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +``` +{% endcode %} diff --git a/ui/public/docs/reference/registries/README.md b/ui/public/docs/reference/registries/README.md new file mode 100644 index 00000000000..8068c0ff150 --- /dev/null +++ b/ui/public/docs/reference/registries/README.md @@ -0,0 +1,23 @@ +# Registies + +Please see [Registry](../../getting-started/components/registry.md) for a conceptual explanation of registries. + +{% content-ref url="local.md" %} +[local.md](local.md) +{% endcontent-ref %} + +{% content-ref url="s3.md" %} +[s3.md](s3.md) +{% endcontent-ref %} + +{% content-ref url="gcs.md" %} +[gcs.md](gcs.md) +{% endcontent-ref %} + +{% content-ref url="sql.md" %} +[sql.md](sql.md) +{% endcontent-ref %} + +{% content-ref url="snowflake.md" %} +[snowflake.md](snowflake.md) +{% endcontent-ref %} diff --git a/ui/public/docs/reference/registries/gcs.md b/ui/public/docs/reference/registries/gcs.md new file mode 100644 index 00000000000..13c9657aa13 --- /dev/null +++ b/ui/public/docs/reference/registries/gcs.md @@ -0,0 +1,23 @@ +# GCS Registry + +## Description + +GCS registry provides support for storing the protobuf representation of your feature store objects (data sources, feature views, feature services, etc.) uing Google Cloud Storage. + +While it can be used in production, there are still inherent limitations with a file-based registries, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +An example of how to configure this would be: + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: feast_gcp +registry: + path: gs://[YOUR BUCKET YOU CREATED]/registry.pb + cache_ttl_seconds: 60 +online_store: null +offline_store: + type: dask +``` +{% endcode %} \ No newline at end of file diff --git a/ui/public/docs/reference/registries/local.md b/ui/public/docs/reference/registries/local.md new file mode 100644 index 00000000000..ad1d98cea99 --- /dev/null +++ b/ui/public/docs/reference/registries/local.md @@ -0,0 +1,23 @@ +# Local Registry + +## Description + +Local registry provides support for storing the protobuf representation of your feature store objects (data sources, feature views, feature services, etc.) in local file system. It is only intended to be used for experimentation with Feast and should not be used in production. + +There are inherent limitations with a file-based registries, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +An example of how to configure this would be: + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: feast_local +registry: + path: registry.pb + cache_ttl_seconds: 60 +online_store: null +offline_store: + type: dask +``` +{% endcode %} \ No newline at end of file diff --git a/ui/public/docs/reference/registries/remote.md b/ui/public/docs/reference/registries/remote.md new file mode 100644 index 00000000000..a03e30ac85f --- /dev/null +++ b/ui/public/docs/reference/registries/remote.md @@ -0,0 +1,28 @@ +# Remote Registry + +## Description + +The Remote Registry is a gRPC client for the registry that implements the `RemoteRegistry` class using the existing `BaseRegistry` interface. + +## How to configure the client + +User needs to create a client side `feature_store.yaml` file, set the `registry_type` to `remote` and provide the server connection configuration. +The `path` parameter is a URL with a port (default is 6570) used by the client to connect with the Remote Registry server. + +{% code title="feature_store.yaml" %} +```yaml +registry: + registry_type: remote + path: localhost:6570 +``` +{% endcode %} + +The optional `cert` parameter can be configured as well, it should point to the public certificate path when the Registry Server starts in SSL mode. This may be needed if the Registry Server is started with a self-signed certificate, typically this file ends with *.crt, *.cer, or *.pem. +More info about the `cert` parameter can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#feast-client-connecting-to-remote-registry-sever-started-in-tls-mode) + +## How to configure the server + +Please see the detail how to configure registry server [registry-server.md](../feature-servers/registry-server.md) + +## How to configure Authentication and Authorization +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. diff --git a/ui/public/docs/reference/registries/s3.md b/ui/public/docs/reference/registries/s3.md new file mode 100644 index 00000000000..65069c415c5 --- /dev/null +++ b/ui/public/docs/reference/registries/s3.md @@ -0,0 +1,23 @@ +# S3 Registry + +## Description + +S3 registry provides support for storing the protobuf representation of your feature store objects (data sources, feature views, feature services, etc.) in S3 file system. + +While it can be used in production, there are still inherent limitations with a file-based registries, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +An example of how to configure this would be: + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: feast_aws_s3 +registry: + path: s3://[YOUR BUCKET YOU CREATED]/registry.pb + cache_ttl_seconds: 60 +online_store: null +offline_store: + type: dask +``` +{% endcode %} \ No newline at end of file diff --git a/ui/public/docs/reference/registries/snowflake.md b/ui/public/docs/reference/registries/snowflake.md new file mode 100644 index 00000000000..00d87b19775 --- /dev/null +++ b/ui/public/docs/reference/registries/snowflake.md @@ -0,0 +1,30 @@ +# Snowflake Registry + +## Description + +The [Snowflake](https://trial.snowflake.com) registry provides support for storing the protobuf representation of your feature store objects (data sources, feature views, feature services, etc.) Because Snowflake is an ACID compliant database, this allows for changes to individual objects atomically. + +An example of how to configure this would be: + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +provider: local +registry: + registry_type: snowflake.registry + account: snowflake_deployment.us-east-1 + user: user_login + password: user_password + role: SYSADMIN + warehouse: COMPUTE_WH + database: FEAST + schema: PUBLIC + cache_ttl_seconds: 60 +offline_store: + ... +``` +{% endcode %} + +The full set of configuration options is available in [SnowflakeRegistryConfig](https://rtd.feast.dev/en/latest/#feast.infra.registry.snowflake.SnowflakeRegistryConfig). diff --git a/ui/public/docs/reference/registries/sql.md b/ui/public/docs/reference/registries/sql.md new file mode 100644 index 00000000000..ef9993c8753 --- /dev/null +++ b/ui/public/docs/reference/registries/sql.md @@ -0,0 +1,89 @@ +# SQL Registry + +## Overview + +By default, the registry Feast uses a file-based registry implementation, which stores the protobuf representation of the registry as a serialized file. This registry file can be stored in a local file system, or in cloud storage (in, say, S3 or GCS). + +However, there's inherent limitations with a file-based registry, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +An alternative to the file-based registry is the [SQLRegistry](https://rtd.feast.dev/en/latest/feast.infra.registry_stores.html#feast.infra.registry_stores.sql.SqlRegistry) which ships with Feast. This implementation stores the registry in a relational database, and allows for changes to individual objects atomically. +Under the hood, the SQL Registry implementation uses [SQLAlchemy](https://docs.sqlalchemy.org/en/14/) to abstract over the different databases. Consequently, any [database supported](https://docs.sqlalchemy.org/en/14/core/engines.html#supported-databases) by SQLAlchemy can be used by the SQL Registry. +The following databases are supported and tested out of the box: +- PostgreSQL +- MySQL +- Sqlite + +Feast can use the SQL Registry via a config change in the feature_store.yaml file. An example of how to configure this would be: + +```yaml +project: +provider: +online_store: redis +offline_store: file +registry: + registry_type: sql + path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true +``` + +Specifically, the registry_type needs to be set to sql in the registry config block. On doing so, the path should refer to the [Database URL](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) for the database to be used, as expected by SQLAlchemy. No other additional commands are currently needed to configure this registry. + +Should you choose to use a database technology that is compatible with one of +Feast's supported registry backends, but which speaks a different dialect (e.g. +`cockroachdb`, which is compatible with `postgres`) then some further +intervention may be required on your part. + +`SQLAlchemy`, used by the registry, may not be able to detect your database +version without first updating your DSN scheme to the appropriate +[DBAPI/dialect combination](https://docs.sqlalchemy.org/en/14/glossary.html#term-DBAPI). +When this happens, your database is likely using what is referred to as an +[external dialect](https://docs.sqlalchemy.org/en/14/dialects/#external-dialects) +in `SQLAlchemy` terminology. See your database's documentation for examples on +how to set its scheme in the Database URL. + +`Psycopg`, which is the database library leveraged by the online and offline +stores, is not impacted by the need to speak a particular dialect, and so the +following only applies to the registry. + +If you are not running Feast in a container, to accomodate `SQLAlchemy`'s need +to speak an external dialect, install additional Python modules like we do as +follows using `cockroachdb` for example: + +```shell +pip install sqlalchemy-cockroachdb +``` + +If you are running Feast in a container, you will need to create a custom image +like we do as follows, again using `cockroachdb` as an example: + +```shell +cat <<'EOF' >Dockerfile +ARG QUAY_IO_FEASTDEV_FEATURE_SERVER +FROM quay.io/feastdev/feature-server:${QUAY_IO_FEASTDEV_FEATURE_SERVER} +ARG PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB +RUN pip install -I --no-cache-dir \ + sqlalchemy-cockroachdb==${PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB} +EOF + +export QUAY_IO_FEASTDEV_FEATURE_SERVER=0.27.1 +export PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB=1.4.4 + +docker build \ + --build-arg QUAY_IO_FEASTDEV_FEATURE_SERVER \ + --build-arg PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB \ + --tag ${MY_REGISTRY}/feastdev/feature-server:${QUAY_IO_FEASTDEV_FEATURE_SERVER} . +``` + +If you are running Feast in Kubernetes, set the `image.repository` and +`imagePullSecrets` Helm values accordingly to utilize your custom image. + +There are some things to note about how the SQL registry works: +- Once instantiated, the Registry ensures the tables needed to store data exist, and creates them if they do not. +- Upon tearing down the feast project, the registry ensures that the tables are dropped from the database. +- The schema for how data is laid out in tables can be found . It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. + +## Example Usage: Concurrent materialization +The SQL Registry should be used when materializing feature views concurrently to ensure correctness of data in the registry. This can be achieved by simply running feast materialize or feature_store.materialize multiple times using a correctly configured feature_store.yaml. This will make each materialization process talk to the registry database concurrently, and ensure the metadata updates are serialized. diff --git a/ui/public/docs/reference/registry/registry-permissions.md b/ui/public/docs/reference/registry/registry-permissions.md new file mode 100644 index 00000000000..65508ef5b25 --- /dev/null +++ b/ui/public/docs/reference/registry/registry-permissions.md @@ -0,0 +1,45 @@ +# Registry Permissions and Access Control + + +## API Endpoints and Permissions + +| Endpoint | Resource Type | Permission | Description | +| ------------------------ |---------------------|------------------------| -------------------------------------------------------------- | +| ApplyEntity | Entity | Create, Update, Delete | Apply an entity to the registry | +| GetEntity | Entity | Read | Get an entity from the registry | +| ListEntities | Entity | Read | List entities in the registry | +| DeleteEntity | Entity | Delete | Delete an entity from the registry | +| ApplyDataSource | DataSource | Create, Update, Delete | Apply a data source to the registry | +| GetDataSource | DataSource | Read | Get a data source from the registry | +| ListDataSources | DataSource | Read | List data sources in the registry | +| DeleteDataSource | DataSource | Delete | Delete a data source from the registry | +| ApplyFeatureView | FeatureView | Create, Update, Delete | Apply a feature view to the registry | +| GetFeatureView | FeatureView | Read | Get a feature view from the registry | +| ListFeatureViews | FeatureView | Read | List feature views in the registry | +| DeleteFeatureView | FeatureView | Delete | Delete a feature view from the registry | +| GetStreamFeatureView | StreamFeatureView | Read | Get a stream feature view from the registry | +| ListStreamFeatureViews | StreamFeatureView | Read | List stream feature views in the registry | +| GetOnDemandFeatureView | OnDemandFeatureView | Read | Get an on-demand feature view from the registry | +| ListOnDemandFeatureViews | OnDemandFeatureView | Read | List on-demand feature views in the registry | +| ApplyFeatureService | FeatureService | Create, Update, Delete | Apply a feature service to the registry | +| GetFeatureService | FeatureService | Read | Get a feature service from the registry | +| ListFeatureServices | FeatureService | Read | List feature services in the registry | +| DeleteFeatureService | FeatureService | Delete | Delete a feature service from the registry | +| ApplySavedDataset | SavedDataset | Create, Update, Delete | Apply a saved dataset to the registry | +| GetSavedDataset | SavedDataset | Read | Get a saved dataset from the registry | +| ListSavedDatasets | SavedDataset | Read | List saved datasets in the registry | +| DeleteSavedDataset | SavedDataset | Delete | Delete a saved dataset from the registry | +| ApplyValidationReference | ValidationReference | Create, Update, Delete | Apply a validation reference to the registry | +| GetValidationReference | ValidationReference | Read | Get a validation reference from the registry | +| ListValidationReferences | ValidationReference | Read | List validation references in the registry | +| DeleteValidationReference| ValidationReference | Delete | Delete a validation reference from the registry | +| ApplyPermission | Permission | Create, Update, Delete | Apply a permission to the registry | +| GetPermission | Permission | Read | Get a permission from the registry | +| ListPermissions | Permission | Read | List permissions in the registry | +| DeletePermission | Permission | Delete | Delete a permission from the registry | +| Commit | | None | Commit changes to the registry | +| Refresh | | None | Refresh the registry | +| Proto | | None | Get the proto representation of the registry | + +## How to configure Authentication and Authorization +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. diff --git a/ui/public/docs/reference/type-system.md b/ui/public/docs/reference/type-system.md new file mode 100644 index 00000000000..affe394f570 --- /dev/null +++ b/ui/public/docs/reference/type-system.md @@ -0,0 +1,41 @@ +# Type System + +## Motivation + +Feast uses an internal type system to provide guarantees on training and serving data. +Feast currently supports eight primitive types - `INT32`, `INT64`, `FLOAT32`, `FLOAT64`, `STRING`, `BYTES`, `BOOL`, and `UNIX_TIMESTAMP` - and the corresponding array types. +Null types are not supported, although the `UNIX_TIMESTAMP` type is nullable. +The type system is controlled by [`Value.proto`](https://github.com/feast-dev/feast/blob/master/protos/feast/types/Value.proto) in protobuf and by [`types.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/types.py) in Python. +Type conversion logic can be found in [`type_map.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/type_map.py). + +## Examples + +### Feature inference + +During `feast apply`, Feast runs schema inference on the data sources underlying feature views. +For example, if the `schema` parameter is not specified for a feature view, Feast will examine the schema of the underlying data source to determine the event timestamp column, feature columns, and entity columns. +Each of these columns must be associated with a Feast type, which requires conversion from the data source type system to the Feast type system. +* The feature inference logic calls `_infer_features_and_entities`. +* `_infer_features_and_entities` calls `source_datatype_to_feast_value_type`. +* `source_datatype_to_feast_value_type` cals the appropriate method in `type_map.py`. For example, if a `SnowflakeSource` is being examined, `snowflake_python_type_to_feast_value_type` from `type_map.py` will be called. + +### Materialization + +Feast serves feature values as [`Value`](https://github.com/feast-dev/feast/blob/master/protos/feast/types/Value.proto) proto objects, which have a type corresponding to Feast types. +Thus Feast must materialize feature values into the online store as `Value` proto objects. +* The local materialization engine first pulls the latest historical features and converts it to pyarrow. +* Then it calls `_convert_arrow_to_proto` to convert the pyarrow table to proto format. +* This calls `python_values_to_proto_values` in `type_map.py` to perform the type conversion. + +### Historical feature retrieval + +The Feast type system is typically not necessary when retrieving historical features. +A call to `get_historical_features` will return a `RetrievalJob` object, which allows the user to export the results to one of several possible locations: a Pandas dataframe, a pyarrow table, a data lake (e.g. S3 or GCS), or the offline store (e.g. a Snowflake table). +In all of these cases, the type conversion is handled natively by the offline store. +For example, a BigQuery query exposes a `to_dataframe` method that will automatically convert the result to a dataframe, without requiring any conversions within Feast. + +### Feature serving + +As mentioned above in the section on [materialization](#materialization), Feast persists feature values into the online store as `Value` proto objects. +A call to `get_online_features` will return an `OnlineResponse` object, which essentially wraps a bunch of `Value` protos with some metadata. +The `OnlineResponse` object can then be converted into a Python dictionary, which calls `feast_value_type_to_python_type` from `type_map.py`, a utility that converts the Feast internal types to Python native types. diff --git a/ui/public/docs/reference/ui.png b/ui/public/docs/reference/ui.png new file mode 100644 index 00000000000..360d57186d3 Binary files /dev/null and b/ui/public/docs/reference/ui.png differ diff --git a/ui/rollup.config.js b/ui/rollup.config.js index 93e17345fe6..c0a435389b0 100644 --- a/ui/rollup.config.js +++ b/ui/rollup.config.js @@ -23,55 +23,27 @@ const banner = ` `; const rollupConfig = [ - // ES { input: inputFileName, output: [ { - file: pkg.module, + dir: "dist", + entryFileNames: path.basename(pkg.module), format: "es", sourcemap: "inline", banner, exports: "named", + inlineDynamicImports: true, }, - ], - external: [ - ...Object.keys(pkg.dependencies || {}), - ...Object.keys(pkg.devDependencies || {}), - ], - plugins: [ - pluginTypescript(), - pluginCommonjs({ - extensions: [".js", ".ts"], - }), - babel({ - babelHelpers: "bundled", - configFile: path.resolve(__dirname, ".babelrc.js"), - }), - pluginNodeResolve({ - browser: false, - }), - css({ - output: "feast-ui.css", - }), - svg(), - json(), - copy({ - targets: [{ src: "src/assets/**/*", dest: "dist/assets/" }], - }), - ], - }, - - // CommonJS - { - input: inputFileName, - output: [ + // CommonJS { - file: pkg.main, + dir: "dist", + entryFileNames: path.basename(pkg.main), format: "cjs", sourcemap: "inline", banner, exports: "default", + inlineDynamicImports: true, }, ], external: [ @@ -92,9 +64,14 @@ const rollupConfig = [ }), css({ output: "feast-ui.css", + minify: true, + inject: false, }), svg(), json(), + copy({ + targets: [{ src: "src/assets/**/*", dest: "dist/assets/" }], + }), ], }, ]; diff --git a/ui/src/FeastUISansProviders.tsx b/ui/src/FeastUISansProviders.tsx index 3a15d9bf083..907917081af 100644 --- a/ui/src/FeastUISansProviders.tsx +++ b/ui/src/FeastUISansProviders.tsx @@ -24,6 +24,7 @@ import RootProjectSelectionPage from "./pages/RootProjectSelectionPage"; import DatasetInstance from "./pages/saved-data-sets/DatasetInstance"; import PermissionsIndex from "./pages/permissions/Index"; import LineageIndex from "./pages/lineage/Index"; +import DocumentationIndex from "./pages/documentation/Index"; import NoProjectGuard from "./components/NoProjectGuard"; import TabsRegistryContext, { @@ -147,6 +148,14 @@ const FeastUISansProvidersInner = ({ /> } /> } /> + } + /> + } + /> } /> diff --git a/ui/src/pages/Sidebar.tsx b/ui/src/pages/Sidebar.tsx index 57599139a44..777487a5fad 100644 --- a/ui/src/pages/Sidebar.tsx +++ b/ui/src/pages/Sidebar.tsx @@ -140,6 +140,15 @@ const SideNav = () => { ), isSelected: useMatchSubpath(`${baseUrl}/permissions`), }, + { + name: "Documentation", + id: htmlIdGenerator("documentation")(), + icon: , + renderItem: (props) => ( + + ), + isSelected: useMatchSubpath(`${baseUrl}/documentation`), + }, ], }, ]; diff --git a/ui/src/pages/documentation/APIDocumentation.tsx b/ui/src/pages/documentation/APIDocumentation.tsx new file mode 100644 index 00000000000..69287c5288b --- /dev/null +++ b/ui/src/pages/documentation/APIDocumentation.tsx @@ -0,0 +1,56 @@ +import React, { useEffect, useState } from "react"; +import ReactMarkdown from "react-markdown"; +import { EuiPanel, EuiLoadingSpinner, EuiText } from "@elastic/eui"; +import DocumentationService from "../../services/DocumentationService"; + +const APIDocumentation = () => { + const [content, setContent] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const loadDocumentation = async () => { + try { + setIsLoading(true); + const markdown = await DocumentationService.fetchAPIDocumentation(); + setContent(markdown); + setError(null); + } catch (err) { + setError("Failed to load API documentation"); + console.error(err); + } finally { + setIsLoading(false); + } + }; + + loadDocumentation(); + }, []); + + if (isLoading) { + return ( + + + + ); + } + + if (error) { + return ( + + +

{error}

+ + + ); + } + + return ( + + + {content} + + + ); +}; + +export default APIDocumentation; diff --git a/ui/src/pages/documentation/CLIDocumentation.tsx b/ui/src/pages/documentation/CLIDocumentation.tsx new file mode 100644 index 00000000000..55cfdfbe222 --- /dev/null +++ b/ui/src/pages/documentation/CLIDocumentation.tsx @@ -0,0 +1,56 @@ +import React, { useEffect, useState } from "react"; +import ReactMarkdown from "react-markdown"; +import { EuiPanel, EuiLoadingSpinner, EuiText } from "@elastic/eui"; +import DocumentationService from "../../services/DocumentationService"; + +const CLIDocumentation = () => { + const [content, setContent] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const loadDocumentation = async () => { + try { + setIsLoading(true); + const markdown = await DocumentationService.fetchCLIDocumentation(); + setContent(markdown); + setError(null); + } catch (err) { + setError("Failed to load CLI documentation"); + console.error(err); + } finally { + setIsLoading(false); + } + }; + + loadDocumentation(); + }, []); + + if (isLoading) { + return ( + + + + ); + } + + if (error) { + return ( + + +

{error}

+ + + ); + } + + return ( + + + {content} + + + ); +}; + +export default CLIDocumentation; diff --git a/ui/src/pages/documentation/Index.tsx b/ui/src/pages/documentation/Index.tsx new file mode 100644 index 00000000000..875c4bc87d0 --- /dev/null +++ b/ui/src/pages/documentation/Index.tsx @@ -0,0 +1,85 @@ +import React, { useEffect, useState } from "react"; +import { + EuiPageTemplate, + EuiTabs, + EuiTab, + EuiSpacer, + EuiTitle, + EuiText, + EuiSkeletonText, +} from "@elastic/eui"; +import { useParams, useNavigate } from "react-router-dom"; +import { useDocumentTitle } from "../../hooks/useDocumentTitle"; +import "./styles.css"; + +const DocumentationIndex = () => { + useDocumentTitle("Feast Documentation"); + const { projectName, tab } = useParams(); + const navigate = useNavigate(); + const [activeTab, setActiveTab] = useState("cli"); + + useEffect(() => { + if (tab && ["cli", "sdk", "api"].includes(tab)) { + setActiveTab(tab); + } + }, [tab]); + + const tabs = [ + { + id: "cli", + name: "CLI Reference", + content: React.lazy(() => import("./CLIDocumentation")), + }, + { + id: "sdk", + name: "SDK Reference", + content: React.lazy(() => import("./SDKDocumentation")), + }, + { + id: "api", + name: "API Reference", + content: React.lazy(() => import("./APIDocumentation")), + }, + ]; + + const selectedTabConfig = tabs.find((t) => t.id === activeTab) || tabs[0]; + const TabContent = selectedTabConfig.content; + + const onSelectedTabChanged = (id: string) => { + setActiveTab(id); + navigate(`/p/${projectName}/documentation/${id}`); + }; + + const renderTabs = () => { + return tabs.map((tabItem, index) => ( + onSelectedTabChanged(tabItem.id)} + isSelected={tabItem.id === activeTab} + > + {tabItem.name} + + )); + }; + + return ( + + + +

Feast Documentation

+ + +

Documentation for the Feast SDK, REST API, and CLI.

+ + + {renderTabs()} + + }> + + + + + ); +}; + +export default DocumentationIndex; diff --git a/ui/src/pages/documentation/SDKDocumentation.tsx b/ui/src/pages/documentation/SDKDocumentation.tsx new file mode 100644 index 00000000000..cff8a2ca44b --- /dev/null +++ b/ui/src/pages/documentation/SDKDocumentation.tsx @@ -0,0 +1,56 @@ +import React, { useEffect, useState } from "react"; +import ReactMarkdown from "react-markdown"; +import { EuiPanel, EuiLoadingSpinner, EuiText } from "@elastic/eui"; +import DocumentationService from "../../services/DocumentationService"; + +const SDKDocumentation = () => { + const [content, setContent] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const loadDocumentation = async () => { + try { + setIsLoading(true); + const markdown = await DocumentationService.fetchSDKDocumentation(); + setContent(markdown); + setError(null); + } catch (err) { + setError("Failed to load SDK documentation"); + console.error(err); + } finally { + setIsLoading(false); + } + }; + + loadDocumentation(); + }, []); + + if (isLoading) { + return ( + + + + ); + } + + if (error) { + return ( + + +

{error}

+ + + ); + } + + return ( + + + {content} + + + ); +}; + +export default SDKDocumentation; diff --git a/ui/src/pages/documentation/styles.css b/ui/src/pages/documentation/styles.css new file mode 100644 index 00000000000..b5387a94efb --- /dev/null +++ b/ui/src/pages/documentation/styles.css @@ -0,0 +1,61 @@ +.documentation-content { + line-height: 1.6; + font-size: 16px; +} + +.documentation-content h1, +.documentation-content h2, +.documentation-content h3, +.documentation-content h4, +.documentation-content h5, +.documentation-content h6 { + margin-top: 24px; + margin-bottom: 16px; + font-weight: 600; +} + +.documentation-content h1 { + font-size: 32px; +} + +.documentation-content h2 { + font-size: 24px; +} + +.documentation-content h3 { + font-size: 20px; +} + +.documentation-content code { + padding: 0.2em 0.4em; + background-color: rgba(0, 0, 0, 0.05); + border-radius: 3px; +} + +.documentation-content pre { + padding: 16px; + background-color: rgba(0, 0, 0, 0.05); + border-radius: 3px; + overflow: auto; +} + +.documentation-content pre code { + background-color: transparent; + padding: 0; +} + +.documentation-content table { + border-collapse: collapse; + width: 100%; + margin: 16px 0; +} + +.documentation-content th, +.documentation-content td { + border: 1px solid #ddd; + padding: 8px; +} + +.documentation-content th { + background-color: rgba(0, 0, 0, 0.05); +} diff --git a/ui/src/services/DocumentationService.ts b/ui/src/services/DocumentationService.ts new file mode 100644 index 00000000000..164f20cbb66 --- /dev/null +++ b/ui/src/services/DocumentationService.ts @@ -0,0 +1,223 @@ +const DocumentationService = { + async fetchCLIDocumentation(): Promise { + try { + return `# Feast CLI Reference + +## feast apply + +Apply changes to a feature store. This command should be run after modifying +feature definitions. + +**Usage:** + +\`\`\` +feast apply +\`\`\` + +**Options:** + +- \`--skip-source-validation\`: Skip validation of data sources. + +--- + +## feast materialize + +Materialize features from an offline store into an online store. + +**Usage:** + +\`\`\` +feast materialize START_TS END_TS +\`\`\` + +**Options:** + +- \`--views\`: Feature views to materialize. + +--- + +## feast registry-dump + +Dump registry contents to local file. + +**Usage:** + +\`\`\` +feast registry-dump REGISTRY_PATH +\`\`\` + +--- + +## feast serve + +Start a feature server. + +**Usage:** + +\`\`\` +feast serve +\`\`\` + +**Options:** + +- \`--host\`: Specify host for the server. +- \`--port\`: Specify port for the server. +`; + } catch (error) { + console.error("Error fetching CLI documentation:", error); + return "# Error\nFailed to load CLI documentation."; + } + }, + + async fetchSDKDocumentation(): Promise { + try { + return `# Feast SDK Reference + +## FeatureStore + +The main entry point for interacting with Feast. + +\`\`\`python +from feast import FeatureStore + +fs = FeatureStore(repo_path="path/to/feature_repo") +\`\`\` + +### Methods + +- \`apply()\`: Register feature definitions to the feature store. +- \`get_historical_features()\`: Retrieve historical feature values. +- \`get_online_features()\`: Retrieve the latest feature values. +- \`materialize()\`: Materialize features from offline to online store. + +## FeatureView + +Define a group of features that share the same data source and entities. + +\`\`\`python +from feast import FeatureView, Feature, Entity, ValueType + +driver = Entity(name="driver_id", value_type=ValueType.INT64) + +driver_stats_view = FeatureView( + name="driver_stats", + entities=[driver], + ttl=timedelta(days=1), + features=[ + Feature(name="conv_rate", dtype=ValueType.FLOAT), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + ], + batch_source=FileSource(path="path/to/data.parquet"), +) +\`\`\` + +## Entity + +Define an entity to join feature values with. + +\`\`\`python +from feast import Entity, ValueType + +driver = Entity( + name="driver_id", + value_type=ValueType.INT64, + description="Driver ID", +) +\`\`\` +`; + } catch (error) { + console.error("Error fetching SDK documentation:", error); + return "# Error\nFailed to load SDK documentation."; + } + }, + + async fetchAPIDocumentation(): Promise { + try { + return `# Feast REST API Reference + +## Feature Server Endpoints + +### GET /health + +Health check endpoint. + +**Response:** + +\`\`\`json +{ + "status": "ok" +} +\`\`\` + +### POST /get-online-features + +Retrieve the latest feature values. + +**Request:** + +\`\`\`json +{ + "features": [ + "driver_stats:conv_rate", + "driver_stats:acc_rate" + ], + "entities": { + "driver_id": [1001, 1002] + } +} +\`\`\` + +**Response:** + +\`\`\`json +{ + "metadata": { + "feature_names": ["driver_stats:conv_rate", "driver_stats:acc_rate"] + }, + "results": [ + { + "values": [0.95, 0.79], + "statuses": ["PRESENT", "PRESENT"] + }, + { + "values": [0.83, 0.85], + "statuses": ["PRESENT", "PRESENT"] + } + ] +} +\`\`\` + +### POST /push + +Push feature values to the online store. + +**Request:** + +\`\`\`json +{ + "push_source_name": "driver_stats_push_source", + "df": { + "driver_id": [1001, 1002], + "conv_rate": [0.95, 0.83], + "acc_rate": [0.79, 0.85], + "event_timestamp": ["2022-01-01T00:00:00Z", "2022-01-01T00:00:00Z"] + } +} +\`\`\` + +**Response:** + +\`\`\`json +{ + "status": "success" +} +\`\`\` +`; + } catch (error) { + console.error("Error fetching API documentation:", error); + return "# Error\nFailed to load API documentation."; + } + }, +}; + +export default DocumentationService; diff --git a/ui/yarn.lock b/ui/yarn.lock index 640dd5a0c05..b3d1266d194 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -2566,6 +2566,13 @@ resolved "https://registry.yarnpkg.com/@types/dagre/-/dagre-0.7.52.tgz#edbf0bca6922cd0ad1936a7486f9d03523d7565a" integrity sha512-XKJdy+OClLk3hketHi9Qg6gTfe1F3y+UFnHxKA2rn9Dw+oXa4Gb378Ztz9HlMgZKSxpPmn4BNVh9wgkpvrK1uw== +"@types/debug@^4.0.0": + version "4.1.12" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.12.tgz#a155f21690871953410df4b6b6f53187f0500917" + integrity sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ== + dependencies: + "@types/ms" "*" + "@types/eslint-scope@^3.7.7": version "3.7.7" resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.7.tgz#3108bd5f18b0cdb277c867b3dd449c9ed7079ac5" @@ -2590,6 +2597,13 @@ "@types/estree" "*" "@types/json-schema" "*" +"@types/estree-jsx@^1.0.0": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@types/estree-jsx/-/estree-jsx-1.0.5.tgz#858a88ea20f34fe65111f005a689fa1ebf70dc18" + integrity sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg== + dependencies: + "@types/estree" "*" + "@types/estree@*", "@types/estree@^1.0.0", "@types/estree@^1.0.6": version "1.0.7" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.7.tgz#4158d3105276773d5b7695cd4834b1722e4f37a8" @@ -2673,6 +2687,13 @@ dependencies: "@types/unist" "^2" +"@types/hast@^3.0.0": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/hast/-/hast-3.0.4.tgz#1d6b39993b82cea6ad783945b0508c25903e15aa" + integrity sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ== + dependencies: + "@types/unist" "*" + "@types/hoist-non-react-statics@^3.3.1": version "3.3.6" resolved "https://registry.yarnpkg.com/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.6.tgz#6bba74383cdab98e8db4e20ce5b4a6b98caed010" @@ -2769,6 +2790,13 @@ dependencies: "@types/unist" "^2" +"@types/mdast@^4.0.0": + version "4.0.4" + resolved "https://registry.yarnpkg.com/@types/mdast/-/mdast-4.0.4.tgz#7ccf72edd2f1aa7dd3437e180c64373585804dd6" + integrity sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA== + dependencies: + "@types/unist" "*" + "@types/mdurl@^2": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-2.0.0.tgz#d43878b5b20222682163ae6f897b20447233bdfd" @@ -2784,6 +2812,11 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-5.1.2.tgz#07508b45797cb81ec3f273011b054cd0755eddca" integrity sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA== +"@types/ms@*": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-2.1.0.tgz#052aa67a48eccc4309d7f0191b7e41434b90bb78" + integrity sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA== + "@types/node-forge@^1.3.0": version "1.3.11" resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-1.3.11.tgz#0972ea538ddb0f4d9c2fa0ec5db5724773a604da" @@ -2940,6 +2973,11 @@ resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.7.tgz#baccb07a970b91707df3a3e8ba6896c57ead2d11" integrity sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw== +"@types/unist@*", "@types/unist@^3.0.0": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/unist/-/unist-3.0.3.tgz#acaab0f919ce69cce629c2d4ed2eb4adc1b6c20c" + integrity sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q== + "@types/unist@^2", "@types/unist@^2.0.0", "@types/unist@^2.0.2", "@types/unist@^2.0.3": version "2.0.11" resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.11.tgz#11af57b127e32487774841f7a4e54eab166d03c4" @@ -3067,7 +3105,7 @@ "@typescript-eslint/types" "5.62.0" eslint-visitor-keys "^3.3.0" -"@ungap/structured-clone@^1.2.0": +"@ungap/structured-clone@^1.0.0", "@ungap/structured-clone@^1.2.0": version "1.3.0" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.0.tgz#d06bbb384ebcf6c505fde1c3d0ed4ddffe0aaff8" integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g== @@ -3722,6 +3760,11 @@ bail@^1.0.0: resolved "https://registry.yarnpkg.com/bail/-/bail-1.0.5.tgz#b6fa133404a392cbc1f8c4bf63f5953351e7a776" integrity sha512-xFbRxM1tahm08yHBP16MMjVUAvDaBMD38zsM9EMAUN61omwLmKlOpB/Zku5QkjZ8TZ4vn53pj+t518cH0S03RQ== +bail@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/bail/-/bail-2.0.2.tgz#d26f5cd8fe5d6f832a31517b9f7c356040ba6d5d" + integrity sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw== + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" @@ -3948,6 +3991,11 @@ ccount@^1.0.0: resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" integrity sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg== +ccount@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/ccount/-/ccount-2.0.1.tgz#17a3bf82302e0870d6da43a01311a8bc02a3ecf5" + integrity sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg== + chalk@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" @@ -3984,21 +4032,41 @@ character-entities-html4@^1.0.0: resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-1.1.4.tgz#0e64b0a3753ddbf1fdc044c5fd01d0199a02e125" integrity sha512-HRcDxZuZqMx3/a+qrzxdBKBPUpxWEq9xw2OPZ3a/174ihfrQKVsFhqtthBInFy1zZ9GgZyFXOatNujm8M+El3g== +character-entities-html4@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-2.1.0.tgz#1f1adb940c971a4b22ba39ddca6b618dc6e56b2b" + integrity sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA== + character-entities-legacy@^1.0.0: version "1.1.4" resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz#94bc1845dce70a5bb9d2ecc748725661293d8fc1" integrity sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA== +character-entities-legacy@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz#76bc83a90738901d7bc223a9e93759fdd560125b" + integrity sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ== + character-entities@^1.0.0: version "1.2.4" resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-1.2.4.tgz#e12c3939b7eaf4e5b15e7ad4c5e28e1d48c5b16b" integrity sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw== +character-entities@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-2.0.2.tgz#2d09c2e72cd9523076ccb21157dff66ad43fcc22" + integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ== + character-reference-invalid@^1.0.0: version "1.1.4" resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz#083329cda0eae272ab3dbbf37e9a382c13af1560" integrity sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg== +character-reference-invalid@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz#85c66b041e43b47210faf401278abf808ac45cb9" + integrity sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw== + check-types@^11.2.3: version "11.2.3" resolved "https://registry.yarnpkg.com/check-types/-/check-types-11.2.3.tgz#1ffdf68faae4e941fce252840b1787b8edc93b71" @@ -4124,6 +4192,11 @@ comma-separated-tokens@^1.0.0: resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== +comma-separated-tokens@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" + integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== + commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -4642,7 +4715,7 @@ debug@2.6.9, debug@^2.6.0: dependencies: ms "2.0.0" -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: +debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: version "4.4.0" resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.0.tgz#2b3f2aea2ffeb776477460267377dc8710faba8a" integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA== @@ -4661,6 +4734,13 @@ decimal.js@^10.4.2: resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.5.0.tgz#0f371c7cf6c4898ce0afb09836db73cd82010f22" integrity sha512-8vDa8Qxvr/+d94hSh5P3IJwI5t8/c0KsMp+g8bNw9cY2icONa5aPfvKeieW1WlG0WQYwwhJ7mjui2xtiePQSXw== +decode-named-character-reference@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/decode-named-character-reference/-/decode-named-character-reference-1.1.0.tgz#5d6ce68792808901210dac42a8e9853511e2b8bf" + integrity sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w== + dependencies: + character-entities "^2.0.0" + decode-uri-component@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.2.tgz#e69dbe25d37941171dd540e024c444cd5188e1e9" @@ -4726,7 +4806,7 @@ depd@~1.1.2: resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== -dequal@^2.0.3: +dequal@^2.0.0, dequal@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be" integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA== @@ -4759,6 +4839,13 @@ detect-port-alt@^1.1.6: address "^1.0.1" debug "^2.6.0" +devlop@^1.0.0, devlop@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/devlop/-/devlop-1.1.0.tgz#4db7c2ca4dc6e0e834c30be70c94bbc976dc7018" + integrity sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA== + dependencies: + dequal "^2.0.0" + diff-sequences@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" @@ -5424,6 +5511,11 @@ estraverse@^5.1.0, estraverse@^5.2.0, estraverse@^5.3.0: resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== +estree-util-is-identifier-name@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz#0b5ef4c4ff13508b34dcd01ecfa945f61fce5dbd" + integrity sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg== + estree-walker@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.2.1.tgz#bdafe8095383d8414d5dc2ecf4c9173b6db9412e" @@ -6164,6 +6256,27 @@ hast-util-to-html@^7.1.1: unist-util-is "^4.0.0" xtend "^4.0.0" +hast-util-to-jsx-runtime@^2.0.0: + version "2.3.6" + resolved "https://registry.yarnpkg.com/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz#ff31897aae59f62232e21594eac7ef6b63333e98" + integrity sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg== + dependencies: + "@types/estree" "^1.0.0" + "@types/hast" "^3.0.0" + "@types/unist" "^3.0.0" + comma-separated-tokens "^2.0.0" + devlop "^1.0.0" + estree-util-is-identifier-name "^3.0.0" + hast-util-whitespace "^3.0.0" + mdast-util-mdx-expression "^2.0.0" + mdast-util-mdx-jsx "^3.0.0" + mdast-util-mdxjs-esm "^2.0.0" + property-information "^7.0.0" + space-separated-tokens "^2.0.0" + style-to-js "^1.0.0" + unist-util-position "^5.0.0" + vfile-message "^4.0.0" + hast-util-to-parse5@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/hast-util-to-parse5/-/hast-util-to-parse5-6.0.0.tgz#1ec44650b631d72952066cea9b1445df699f8479" @@ -6180,6 +6293,13 @@ hast-util-whitespace@^1.0.0: resolved "https://registry.yarnpkg.com/hast-util-whitespace/-/hast-util-whitespace-1.0.4.tgz#e4fe77c4a9ae1cb2e6c25e02df0043d0164f6e41" integrity sha512-I5GTdSfhYfAPNztx2xJRQpG8cuDSNt599/7YUn7Gx/WxNMsG+a835k97TDkFgk123cwjfwINaZknkKkphx/f2A== +hast-util-whitespace@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz#7778ed9d3c92dd9e8c5c8f648a49c21fc51cb621" + integrity sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw== + dependencies: + "@types/hast" "^3.0.0" + hastscript@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/hastscript/-/hastscript-6.0.0.tgz#e8768d7eac56c3fdeac8a92830d58e811e5bf640" @@ -6263,6 +6383,11 @@ html-minifier-terser@^6.0.2: relateurl "^0.2.7" terser "^5.10.0" +html-url-attributes@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/html-url-attributes/-/html-url-attributes-3.0.1.tgz#83b052cd5e437071b756cd74ae70f708870c2d87" + integrity sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ== + html-void-elements@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483" @@ -6457,6 +6582,11 @@ inline-style-parser@0.1.1: resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.1.1.tgz#ec8a3b429274e9c0a1f1c4ffa9453a7fef72cea1" integrity sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q== +inline-style-parser@0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.2.4.tgz#f4af5fe72e612839fcd453d989a586566d695f22" + integrity sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q== + inter-ui@^3.19.3: version "3.19.3" resolved "https://registry.yarnpkg.com/inter-ui/-/inter-ui-3.19.3.tgz#cf4b4b6d30de8d5463e2462588654b325206488c" @@ -6486,6 +6616,11 @@ is-alphabetical@^1.0.0: resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.4.tgz#9e7d6b94916be22153745d184c298cbf986a686d" integrity sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg== +is-alphabetical@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-2.0.1.tgz#01072053ea7c1036df3c7d19a6daaec7f19e789b" + integrity sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ== + is-alphanumerical@^1.0.0: version "1.0.4" resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz#7eb9a2431f855f6b1ef1a78e326df515696c4dbf" @@ -6494,6 +6629,14 @@ is-alphanumerical@^1.0.0: is-alphabetical "^1.0.0" is-decimal "^1.0.0" +is-alphanumerical@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz#7c03fbe96e3e931113e57f964b0a368cc2dfd875" + integrity sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw== + dependencies: + is-alphabetical "^2.0.0" + is-decimal "^2.0.0" + is-array-buffer@^3.0.4, is-array-buffer@^3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.5.tgz#65742e1e687bd2cc666253068fd8707fe4d44280" @@ -6587,6 +6730,11 @@ is-decimal@^1.0.0: resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.4.tgz#65a3a5958a1c5b63a706e1b333d7cd9f630d3fa5" integrity sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw== +is-decimal@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-2.0.1.tgz#9469d2dc190d0214fd87d78b78caecc0cc14eef7" + integrity sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A== + is-docker@^2.0.0, is-docker@^2.1.1: version "2.2.1" resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" @@ -6636,6 +6784,11 @@ is-hexadecimal@^1.0.0: resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz#cc35c97588da4bd49a8eedd6bc4082d44dcb23a7" integrity sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw== +is-hexadecimal@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz#86b5bf668fca307498d319dfc03289d781a90027" + integrity sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg== + is-map@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.3.tgz#ede96b7fe1e270b3c4465e3a465658764926d62e" @@ -6684,6 +6837,11 @@ is-plain-obj@^3.0.0: resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-3.0.0.tgz#af6f2ea14ac5a646183a5bbdb5baabbc156ad9d7" integrity sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA== +is-plain-obj@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-4.1.0.tgz#d65025edec3657ce032fd7db63c97883eaed71f0" + integrity sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg== + is-plain-object@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" @@ -7721,6 +7879,11 @@ long@^5.0.0, long@^5.2.3: resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== +longest-streak@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-3.1.0.tgz#62fa67cd958742a1574af9f39866364102d90cd4" + integrity sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g== + loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -7830,6 +7993,74 @@ mdast-util-definitions@^4.0.0: dependencies: unist-util-visit "^2.0.0" +mdast-util-from-markdown@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.2.tgz#4850390ca7cf17413a9b9a0fbefcd1bc0eb4160a" + integrity sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA== + dependencies: + "@types/mdast" "^4.0.0" + "@types/unist" "^3.0.0" + decode-named-character-reference "^1.0.0" + devlop "^1.0.0" + mdast-util-to-string "^4.0.0" + micromark "^4.0.0" + micromark-util-decode-numeric-character-reference "^2.0.0" + micromark-util-decode-string "^2.0.0" + micromark-util-normalize-identifier "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + unist-util-stringify-position "^4.0.0" + +mdast-util-mdx-expression@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz#43f0abac9adc756e2086f63822a38c8d3c3a5096" + integrity sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ== + dependencies: + "@types/estree-jsx" "^1.0.0" + "@types/hast" "^3.0.0" + "@types/mdast" "^4.0.0" + devlop "^1.0.0" + mdast-util-from-markdown "^2.0.0" + mdast-util-to-markdown "^2.0.0" + +mdast-util-mdx-jsx@^3.0.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz#fd04c67a2a7499efb905a8a5c578dddc9fdada0d" + integrity sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q== + dependencies: + "@types/estree-jsx" "^1.0.0" + "@types/hast" "^3.0.0" + "@types/mdast" "^4.0.0" + "@types/unist" "^3.0.0" + ccount "^2.0.0" + devlop "^1.1.0" + mdast-util-from-markdown "^2.0.0" + mdast-util-to-markdown "^2.0.0" + parse-entities "^4.0.0" + stringify-entities "^4.0.0" + unist-util-stringify-position "^4.0.0" + vfile-message "^4.0.0" + +mdast-util-mdxjs-esm@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz#019cfbe757ad62dd557db35a695e7314bcc9fa97" + integrity sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg== + dependencies: + "@types/estree-jsx" "^1.0.0" + "@types/hast" "^3.0.0" + "@types/mdast" "^4.0.0" + devlop "^1.0.0" + mdast-util-from-markdown "^2.0.0" + mdast-util-to-markdown "^2.0.0" + +mdast-util-phrasing@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz#7cc0a8dec30eaf04b7b1a9661a92adb3382aa6e3" + integrity sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w== + dependencies: + "@types/mdast" "^4.0.0" + unist-util-is "^6.0.0" + mdast-util-to-hast@^10.2.0: version "10.2.0" resolved "https://registry.yarnpkg.com/mdast-util-to-hast/-/mdast-util-to-hast-10.2.0.tgz#61875526a017d8857b71abc9333942700b2d3604" @@ -7844,6 +8075,43 @@ mdast-util-to-hast@^10.2.0: unist-util-position "^3.0.0" unist-util-visit "^2.0.0" +mdast-util-to-hast@^13.0.0: + version "13.2.0" + resolved "https://registry.yarnpkg.com/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz#5ca58e5b921cc0a3ded1bc02eed79a4fe4fe41f4" + integrity sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA== + dependencies: + "@types/hast" "^3.0.0" + "@types/mdast" "^4.0.0" + "@ungap/structured-clone" "^1.0.0" + devlop "^1.0.0" + micromark-util-sanitize-uri "^2.0.0" + trim-lines "^3.0.0" + unist-util-position "^5.0.0" + unist-util-visit "^5.0.0" + vfile "^6.0.0" + +mdast-util-to-markdown@^2.0.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz#f910ffe60897f04bb4b7e7ee434486f76288361b" + integrity sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA== + dependencies: + "@types/mdast" "^4.0.0" + "@types/unist" "^3.0.0" + longest-streak "^3.0.0" + mdast-util-phrasing "^4.0.0" + mdast-util-to-string "^4.0.0" + micromark-util-classify-character "^2.0.0" + micromark-util-decode-string "^2.0.0" + unist-util-visit "^5.0.0" + zwitch "^2.0.0" + +mdast-util-to-string@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz#7a5121475556a04e7eddeb67b264aae79d312814" + integrity sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg== + dependencies: + "@types/mdast" "^4.0.0" + mdn-data@2.0.14: version "2.0.14" resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" @@ -7911,6 +8179,200 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== +micromark-core-commonmark@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz#c691630e485021a68cf28dbc2b2ca27ebf678cd4" + integrity sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg== + dependencies: + decode-named-character-reference "^1.0.0" + devlop "^1.0.0" + micromark-factory-destination "^2.0.0" + micromark-factory-label "^2.0.0" + micromark-factory-space "^2.0.0" + micromark-factory-title "^2.0.0" + micromark-factory-whitespace "^2.0.0" + micromark-util-character "^2.0.0" + micromark-util-chunked "^2.0.0" + micromark-util-classify-character "^2.0.0" + micromark-util-html-tag-name "^2.0.0" + micromark-util-normalize-identifier "^2.0.0" + micromark-util-resolve-all "^2.0.0" + micromark-util-subtokenize "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-factory-destination@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz#8fef8e0f7081f0474fbdd92deb50c990a0264639" + integrity sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA== + dependencies: + micromark-util-character "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-factory-label@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz#5267efa97f1e5254efc7f20b459a38cb21058ba1" + integrity sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg== + dependencies: + devlop "^1.0.0" + micromark-util-character "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-factory-space@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz#36d0212e962b2b3121f8525fc7a3c7c029f334fc" + integrity sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg== + dependencies: + micromark-util-character "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-factory-title@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz#237e4aa5d58a95863f01032d9ee9b090f1de6e94" + integrity sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw== + dependencies: + micromark-factory-space "^2.0.0" + micromark-util-character "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-factory-whitespace@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz#06b26b2983c4d27bfcc657b33e25134d4868b0b1" + integrity sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ== + dependencies: + micromark-factory-space "^2.0.0" + micromark-util-character "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-util-character@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/micromark-util-character/-/micromark-util-character-2.1.1.tgz#2f987831a40d4c510ac261e89852c4e9703ccda6" + integrity sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q== + dependencies: + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-util-chunked@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz#47fbcd93471a3fccab86cff03847fc3552db1051" + integrity sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA== + dependencies: + micromark-util-symbol "^2.0.0" + +micromark-util-classify-character@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz#d399faf9c45ca14c8b4be98b1ea481bced87b629" + integrity sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q== + dependencies: + micromark-util-character "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-util-combine-extensions@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz#2a0f490ab08bff5cc2fd5eec6dd0ca04f89b30a9" + integrity sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg== + dependencies: + micromark-util-chunked "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-util-decode-numeric-character-reference@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz#fcf15b660979388e6f118cdb6bf7d79d73d26fe5" + integrity sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw== + dependencies: + micromark-util-symbol "^2.0.0" + +micromark-util-decode-string@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz#6cb99582e5d271e84efca8e61a807994d7161eb2" + integrity sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ== + dependencies: + decode-named-character-reference "^1.0.0" + micromark-util-character "^2.0.0" + micromark-util-decode-numeric-character-reference "^2.0.0" + micromark-util-symbol "^2.0.0" + +micromark-util-encode@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz#0d51d1c095551cfaac368326963cf55f15f540b8" + integrity sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw== + +micromark-util-html-tag-name@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz#e40403096481986b41c106627f98f72d4d10b825" + integrity sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA== + +micromark-util-normalize-identifier@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz#c30d77b2e832acf6526f8bf1aa47bc9c9438c16d" + integrity sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q== + dependencies: + micromark-util-symbol "^2.0.0" + +micromark-util-resolve-all@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz#e1a2d62cdd237230a2ae11839027b19381e31e8b" + integrity sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg== + dependencies: + micromark-util-types "^2.0.0" + +micromark-util-sanitize-uri@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz#ab89789b818a58752b73d6b55238621b7faa8fd7" + integrity sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ== + dependencies: + micromark-util-character "^2.0.0" + micromark-util-encode "^2.0.0" + micromark-util-symbol "^2.0.0" + +micromark-util-subtokenize@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz#d8ade5ba0f3197a1cf6a2999fbbfe6357a1a19ee" + integrity sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA== + dependencies: + devlop "^1.0.0" + micromark-util-chunked "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + +micromark-util-symbol@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz#e5da494e8eb2b071a0d08fb34f6cefec6c0a19b8" + integrity sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q== + +micromark-util-types@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/micromark-util-types/-/micromark-util-types-2.0.2.tgz#f00225f5f5a0ebc3254f96c36b6605c4b393908e" + integrity sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA== + +micromark@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/micromark/-/micromark-4.0.2.tgz#91395a3e1884a198e62116e33c9c568e39936fdb" + integrity sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA== + dependencies: + "@types/debug" "^4.0.0" + debug "^4.0.0" + decode-named-character-reference "^1.0.0" + devlop "^1.0.0" + micromark-core-commonmark "^2.0.0" + micromark-factory-space "^2.0.0" + micromark-util-character "^2.0.0" + micromark-util-chunked "^2.0.0" + micromark-util-combine-extensions "^2.0.0" + micromark-util-decode-numeric-character-reference "^2.0.0" + micromark-util-encode "^2.0.0" + micromark-util-normalize-identifier "^2.0.0" + micromark-util-resolve-all "^2.0.0" + micromark-util-sanitize-uri "^2.0.0" + micromark-util-subtokenize "^2.0.0" + micromark-util-symbol "^2.0.0" + micromark-util-types "^2.0.0" + micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5, micromatch@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" @@ -8375,6 +8837,19 @@ parse-entities@^2.0.0: is-decimal "^1.0.0" is-hexadecimal "^1.0.0" +parse-entities@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-4.0.2.tgz#61d46f5ed28e4ee62e9ddc43d6b010188443f159" + integrity sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw== + dependencies: + "@types/unist" "^2.0.0" + character-entities-legacy "^3.0.0" + character-reference-invalid "^2.0.0" + decode-named-character-reference "^1.0.0" + is-alphanumerical "^2.0.0" + is-decimal "^2.0.0" + is-hexadecimal "^2.0.0" + parse-json@^5.0.0, parse-json@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" @@ -9116,6 +9591,11 @@ property-information@^5.0.0, property-information@^5.3.0: dependencies: xtend "^4.0.0" +property-information@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/property-information/-/property-information-7.0.0.tgz#3508a6d6b0b8eb3ca6eb2c6623b164d2ed2ab112" + integrity sha512-7D/qOz/+Y4X/rzSB6jKxKUsQnphO046ei8qxG59mtM3RG3DHgTK81HrxrmoDVINJb8NKT5ZsRbwHvQ6B68Iyhg== + protobufjs-cli@^1.1.3: version "1.2.0" resolved "https://registry.yarnpkg.com/protobufjs-cli/-/protobufjs-cli-1.2.0.tgz#653f53ad1866e81d16b9e3adf564bf59985af1bf" @@ -9375,6 +9855,23 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e" integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg== +react-markdown@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-10.1.0.tgz#e22bc20faddbc07605c15284255653c0f3bad5ca" + integrity sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ== + dependencies: + "@types/hast" "^3.0.0" + "@types/mdast" "^4.0.0" + devlop "^1.0.0" + hast-util-to-jsx-runtime "^2.0.0" + html-url-attributes "^3.0.0" + mdast-util-to-hast "^13.0.0" + remark-parse "^11.0.0" + remark-rehype "^11.0.0" + unified "^11.0.0" + unist-util-visit "^5.0.0" + vfile "^6.0.0" + react-query@^3.39.3: version "3.39.3" resolved "https://registry.yarnpkg.com/react-query/-/react-query-3.39.3.tgz#4cea7127c6c26bdea2de5fb63e51044330b03f35" @@ -9695,6 +10192,27 @@ remark-parse-no-trim@^8.0.4: vfile-location "^3.0.0" xtend "^4.0.1" +remark-parse@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-11.0.0.tgz#aa60743fcb37ebf6b069204eb4da304e40db45a1" + integrity sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA== + dependencies: + "@types/mdast" "^4.0.0" + mdast-util-from-markdown "^2.0.0" + micromark-util-types "^2.0.0" + unified "^11.0.0" + +remark-rehype@^11.0.0: + version "11.1.2" + resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-11.1.2.tgz#2addaadda80ca9bd9aa0da763e74d16327683b37" + integrity sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw== + dependencies: + "@types/hast" "^3.0.0" + "@types/mdast" "^4.0.0" + mdast-util-to-hast "^13.0.0" + unified "^11.0.0" + vfile "^6.0.0" + remark-rehype@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-8.1.0.tgz#610509a043484c1e697437fa5eb3fd992617c945" @@ -10280,6 +10798,11 @@ space-separated-tokens@^1.0.0: resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz#85f32c3d10d9682007e917414ddc5c26d1aa6899" integrity sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA== +space-separated-tokens@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz#1ecd9d2350a3844572c3f4a312bceb018348859f" + integrity sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q== + spdy-transport@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/spdy-transport/-/spdy-transport-3.0.0.tgz#00d4863a6400ad75df93361a1608605e5dcdcf31" @@ -10483,6 +11006,14 @@ stringify-entities@^3.0.1: character-entities-legacy "^1.0.0" xtend "^4.0.0" +stringify-entities@^4.0.0: + version "4.0.4" + resolved "https://registry.yarnpkg.com/stringify-entities/-/stringify-entities-4.0.4.tgz#b3b79ef5f277cc4ac73caeb0236c5ba939b3a4f3" + integrity sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg== + dependencies: + character-entities-html4 "^2.0.0" + character-entities-legacy "^3.0.0" + stringify-object@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/stringify-object/-/stringify-object-3.3.0.tgz#703065aefca19300d3ce88af4f5b3956d7556629" @@ -10543,6 +11074,20 @@ style-loader@^3.3.1: resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-3.3.4.tgz#f30f786c36db03a45cbd55b6a70d930c479090e7" integrity sha512-0WqXzrsMTyb8yjZJHDqwmnwRJvhALK9LfRtRc6B4UTWe8AijYLZYZ9thuJTZc2VfQWINADW/j+LiJnfy2RoC1w== +style-to-js@^1.0.0: + version "1.1.16" + resolved "https://registry.yarnpkg.com/style-to-js/-/style-to-js-1.1.16.tgz#e6bd6cd29e250bcf8fa5e6591d07ced7575dbe7a" + integrity sha512-/Q6ld50hKYPH3d/r6nr117TZkHR0w0kGGIVfpG9N6D8NymRPM9RqCUv4pRpJ62E5DqOYx2AFpbZMyCPnjQCnOw== + dependencies: + style-to-object "1.0.8" + +style-to-object@1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/style-to-object/-/style-to-object-1.0.8.tgz#67a29bca47eaa587db18118d68f9d95955e81292" + integrity sha512-xT47I/Eo0rwJmaXC4oilDGDWLohVhR6o/xAQcPQN8q6QBuZVL8qMYL85kLmST5cPjAorwvqIA4qXTRQoYHaL6g== + dependencies: + inline-style-parser "0.2.4" + style-to-object@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/style-to-object/-/style-to-object-0.3.0.tgz#b1b790d205991cc783801967214979ee19a76e46" @@ -10764,6 +11309,11 @@ tr46@^3.0.0: dependencies: punycode "^2.1.1" +trim-lines@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/trim-lines/-/trim-lines-3.0.1.tgz#d802e332a07df861c48802c04321017b1bd87338" + integrity sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg== + trim-trailing-lines@^1.0.0: version "1.1.4" resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.4.tgz#bd4abbec7cc880462f10b2c8b5ce1d8d1ec7c2c0" @@ -10774,6 +11324,11 @@ trough@^1.0.0: resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" integrity sha512-rvuRbTarPXmMb79SmzEp8aqXNKcK+y0XaB298IXueQ8I2PsrATcPBCSPyK/dDNa2iWOhKlfNnOjdAOTBU/nkFA== +trough@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/trough/-/trough-2.2.0.tgz#94a60bd6bd375c152c1df911a4b11d5b0256f50f" + integrity sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw== + tryer@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8" @@ -10981,6 +11536,19 @@ unicode-property-aliases-ecmascript@^2.0.0: resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz#43d41e3be698bd493ef911077c9b131f827e8ccd" integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w== +unified@^11.0.0: + version "11.0.5" + resolved "https://registry.yarnpkg.com/unified/-/unified-11.0.5.tgz#f66677610a5c0a9ee90cab2b8d4d66037026d9e1" + integrity sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA== + dependencies: + "@types/unist" "^3.0.0" + bail "^2.0.0" + devlop "^1.0.0" + extend "^3.0.0" + is-plain-obj "^4.0.0" + trough "^2.0.0" + vfile "^6.0.0" + unified@^9.2.2: version "9.2.2" resolved "https://registry.yarnpkg.com/unified/-/unified-9.2.2.tgz#67649a1abfc3ab85d2969502902775eb03146975" @@ -11020,11 +11588,25 @@ unist-util-is@^4.0.0: resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-4.1.0.tgz#976e5f462a7a5de73d94b706bac1b90671b57797" integrity sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg== +unist-util-is@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-6.0.0.tgz#b775956486aff107a9ded971d996c173374be424" + integrity sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw== + dependencies: + "@types/unist" "^3.0.0" + unist-util-position@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/unist-util-position/-/unist-util-position-3.1.0.tgz#1c42ee6301f8d52f47d14f62bbdb796571fa2d47" integrity sha512-w+PkwCbYSFw8vpgWD0v7zRCl1FpY3fjDSQ3/N/wNd9Ffa4gPi8+4keqt99N3XW6F99t/mUzp2xAhNmfKWp95QA== +unist-util-position@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/unist-util-position/-/unist-util-position-5.0.0.tgz#678f20ab5ca1207a97d7ea8a388373c9cf896be4" + integrity sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA== + dependencies: + "@types/unist" "^3.0.0" + unist-util-remove-position@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/unist-util-remove-position/-/unist-util-remove-position-2.0.1.tgz#5d19ca79fdba712301999b2b73553ca8f3b352cc" @@ -11039,6 +11621,13 @@ unist-util-stringify-position@^2.0.0: dependencies: "@types/unist" "^2.0.2" +unist-util-stringify-position@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz#449c6e21a880e0855bf5aabadeb3a740314abac2" + integrity sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ== + dependencies: + "@types/unist" "^3.0.0" + unist-util-visit-parents@^2.0.0: version "2.1.2" resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz#25e43e55312166f3348cae6743588781d112c1e9" @@ -11054,6 +11643,14 @@ unist-util-visit-parents@^3.0.0: "@types/unist" "^2.0.0" unist-util-is "^4.0.0" +unist-util-visit-parents@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz#4d5f85755c3b8f0dc69e21eca5d6d82d22162815" + integrity sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw== + dependencies: + "@types/unist" "^3.0.0" + unist-util-is "^6.0.0" + unist-util-visit@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-1.4.1.tgz#4724aaa8486e6ee6e26d7ff3c8685960d560b1e3" @@ -11070,6 +11667,15 @@ unist-util-visit@^2.0.0, unist-util-visit@^2.0.3: unist-util-is "^4.0.0" unist-util-visit-parents "^3.0.0" +unist-util-visit@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-5.0.0.tgz#a7de1f31f72ffd3519ea71814cccf5fd6a9217d6" + integrity sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg== + dependencies: + "@types/unist" "^3.0.0" + unist-util-is "^6.0.0" + unist-util-visit-parents "^6.0.0" + universalify@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" @@ -11205,6 +11811,14 @@ vfile-message@^2.0.0: "@types/unist" "^2.0.0" unist-util-stringify-position "^2.0.0" +vfile-message@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/vfile-message/-/vfile-message-4.0.2.tgz#c883c9f677c72c166362fd635f21fc165a7d1181" + integrity sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw== + dependencies: + "@types/unist" "^3.0.0" + unist-util-stringify-position "^4.0.0" + vfile@^4.0.0, vfile@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/vfile/-/vfile-4.2.1.tgz#03f1dce28fc625c625bc6514350fbdb00fa9e624" @@ -11215,6 +11829,14 @@ vfile@^4.0.0, vfile@^4.2.1: unist-util-stringify-position "^2.0.0" vfile-message "^2.0.0" +vfile@^6.0.0: + version "6.0.3" + resolved "https://registry.yarnpkg.com/vfile/-/vfile-6.0.3.tgz#3652ab1c496531852bf55a6bac57af981ebc38ab" + integrity sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q== + dependencies: + "@types/unist" "^3.0.0" + vfile-message "^4.0.0" + w3c-xmlserializer@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz#aebdc84920d806222936e3cdce408e32488a3073" @@ -11779,3 +12401,8 @@ zwitch@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920" integrity sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw== + +zwitch@^2.0.0: + version "2.0.4" + resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-2.0.4.tgz#c827d4b0acb76fc3e685a4c6ec2902d51070e9d7" + integrity sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==