From 43377875f815745127b50fd7e9a55296f6abf9c4 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 11 Dec 2018 14:53:28 +0800 Subject: [PATCH 1/6] Add eq functions for checking diff between specs --- .../java/feast/core/model/EntityInfo.java | 12 ++++++++ .../feast/core/model/FeatureGroupInfo.java | 19 ++++++++++++ .../java/feast/core/model/FeatureInfo.java | 30 +++++++++++++++++++ .../java/feast/core/model/StorageInfo.java | 16 ++++++++-- .../java/feast/core/model/EntityInfoTest.java | 10 +++++++ .../core/model/FeatureGroupInfoTest.java | 10 +++++++ .../feast/core/model/FeatureInfoTest.java | 10 +++++++ .../feast/core/model/StorageInfoTest.java | 10 +++++++ 8 files changed, 115 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/feast/core/model/EntityInfo.java b/core/src/main/java/feast/core/model/EntityInfo.java index 5df91a5b657..9cf0ad343bc 100644 --- a/core/src/main/java/feast/core/model/EntityInfo.java +++ b/core/src/main/java/feast/core/model/EntityInfo.java @@ -86,4 +86,16 @@ public EntityDetail getEntityDetail() { .setLastUpdated(convertTimestamp(this.getLastUpdated())) .build(); } + + /** + * Checks if this is eq to the other given entity + * + * @param otherEntity + * @return boolean + */ + public boolean eq(EntityInfo otherEntity) { + return otherEntity.getName().equals(this.getName()) && + otherEntity.getDescription().equals(this.getDescription()) && + otherEntity.getTags().equals(this.getTags()); + } } diff --git a/core/src/main/java/feast/core/model/FeatureGroupInfo.java b/core/src/main/java/feast/core/model/FeatureGroupInfo.java index 809dfe38f55..d97ea70ecae 100644 --- a/core/src/main/java/feast/core/model/FeatureGroupInfo.java +++ b/core/src/main/java/feast/core/model/FeatureGroupInfo.java @@ -111,4 +111,23 @@ public FeatureGroupDetail getFeatureGroupDetail() { .setLastUpdated(TypeConversion.convertTimestamp(this.getLastUpdated())) .build(); } + + /** + * Checks if this is eq to the other given feature group + * + * @param otherFeatureGroup + * @return boolean + */ + public boolean eq(FeatureGroupInfo otherFeatureGroup) { + return otherFeatureGroup.getId() == this.id && + otherFeatureGroup.getTags() == this.getTags() && + otherFeatureGroup.getServingStoreOpts() == this.servingStoreOpts && + getStorageId(otherFeatureGroup.getServingStore()) == getStorageId(this.getServingStore()) && + otherFeatureGroup.getWarehouseStoreOpts() == this.warehouseStoreOpts && + getStorageId(otherFeatureGroup.getWarehouseStore()) == getStorageId(this.getWarehouseStore()); + } + + private String getStorageId(StorageInfo storage) { + return storage == null ? "" : storage.getId(); + } } diff --git a/core/src/main/java/feast/core/model/FeatureInfo.java b/core/src/main/java/feast/core/model/FeatureInfo.java index 6d044102d15..b219565d9e2 100644 --- a/core/src/main/java/feast/core/model/FeatureInfo.java +++ b/core/src/main/java/feast/core/model/FeatureInfo.java @@ -256,4 +256,34 @@ private String createBigqueryViewLink(StorageInfo warehouseStore) { "https://bigquery.cloud.google.com/table/%s:%s.%s_%s_view", projectId, dataset, entity.getName(), granularity.toString().toLowerCase()); } + + /** + * Checks if this is eq to the other given feature + * + * @param otherFeature + * @return boolean + */ + public boolean eq(FeatureInfo otherFeature) { + return otherFeature.getId() == this.id && + otherFeature.getEntity().getName() == this.entity.getName() && + otherFeature.getOwner() == this.owner && + otherFeature.getUri() == this.uri && + otherFeature.getDescription() == this.description && + otherFeature.getGranularity() == this.granularity && + otherFeature.getValueType() == this.valueType && + otherFeature.getTags() == this.tags && + otherFeature.getOptions() == this.options && + getFeatureGroupId(otherFeature.getFeatureGroup()) == getFeatureGroupId(this.getFeatureGroup()) && + otherFeature.getServingStoreOpts() == this.servingStoreOpts && + getStorageId(otherFeature.getServingStore()) == getStorageId(this.getServingStore()) && + otherFeature.getWarehouseStoreOpts() == this.warehouseStoreOpts && + getStorageId(otherFeature.getWarehouseStore()) == getStorageId(this.getWarehouseStore()); + } + + private String getFeatureGroupId(FeatureGroupInfo featureGroupInfo) { + return featureGroupInfo == null ? "" : featureGroupInfo.getId(); + } + private String getStorageId(StorageInfo storage) { + return storage == null ? "" : storage.getId(); + } } diff --git a/core/src/main/java/feast/core/model/StorageInfo.java b/core/src/main/java/feast/core/model/StorageInfo.java index afc1a901cae..0de83fb61ca 100644 --- a/core/src/main/java/feast/core/model/StorageInfo.java +++ b/core/src/main/java/feast/core/model/StorageInfo.java @@ -17,11 +17,11 @@ package feast.core.model; +import feast.core.UIServiceProto.UIServiceTypes.StorageDetail; +import feast.specs.StorageSpecProto.StorageSpec; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; -import feast.core.UIServiceProto.UIServiceTypes.StorageDetail; -import feast.specs.StorageSpecProto.StorageSpec; import javax.persistence.Column; import javax.persistence.Entity; @@ -80,4 +80,16 @@ public StorageDetail getStorageDetail() { .setLastUpdated(convertTimestamp(this.getLastUpdated())) .build(); } + + /** + * Checks if this is eq to the other given storage + * + * @param otherStorage + * @return boolean + */ + public boolean eq(StorageInfo otherStorage) { + return otherStorage.getId().equals(this.id) && + otherStorage.getType().equals(this.type) && + otherStorage.getOptions().equals(this.options); + } } diff --git a/core/src/test/java/feast/core/model/EntityInfoTest.java b/core/src/test/java/feast/core/model/EntityInfoTest.java index d468822c909..eff317795af 100644 --- a/core/src/test/java/feast/core/model/EntityInfoTest.java +++ b/core/src/test/java/feast/core/model/EntityInfoTest.java @@ -23,6 +23,7 @@ import feast.core.UIServiceProto.UIServiceTypes.EntityDetail; import feast.specs.EntitySpecProto.EntitySpec; +import java.time.Instant; import java.util.Date; import static org.hamcrest.Matchers.equalTo; @@ -66,4 +67,13 @@ public void shouldBuildAndReturnCorrespondingDetail() { EntityDetail.newBuilder().setSpec(entitySpec).setLastUpdated(ts).build(); assertThat(entityInfo.getEntityDetail(), equalTo(expected)); } + + @Test + public void shouldBeEqualToEntityFromSameSpecs() { + EntityInfo entity1 = new EntityInfo(entitySpec); + entity1.setCreated(Date.from(Instant.ofEpochSecond(1))); + EntityInfo entity2 = new EntityInfo(entitySpec); + entity1.setCreated(Date.from(Instant.ofEpochSecond(2))); + assertThat(entity1.eq(entity2), equalTo(true)); + } } diff --git a/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java b/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java index 89c388fb3e3..ebfd7c3a350 100644 --- a/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java +++ b/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java @@ -25,6 +25,7 @@ import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; +import java.time.Instant; import java.util.Date; import static org.hamcrest.Matchers.equalTo; @@ -84,4 +85,13 @@ public void shouldBuildAndReturnCorrespondingDetail() { FeatureGroupDetail.newBuilder().setSpec(featureGroupSpec).setLastUpdated(ts).build(); assertThat(featureGroupInfo.getFeatureGroupDetail(), equalTo(expected)); } + + @Test + public void shouldBeEqualToFeatureGroupFromSameSpecs() { + FeatureGroupInfo featureGroup1 = new FeatureGroupInfo(featureGroupSpec, servingStorage, warehouseStorage); + featureGroup1.setCreated(Date.from(Instant.ofEpochSecond(1))); + FeatureGroupInfo featureGroup2 = new FeatureGroupInfo(featureGroupSpec, servingStorage, warehouseStorage); + featureGroup2.setCreated(Date.from(Instant.ofEpochSecond(2))); + assertThat(featureGroup1.eq(featureGroup2), equalTo(true)); + } } diff --git a/core/src/test/java/feast/core/model/FeatureInfoTest.java b/core/src/test/java/feast/core/model/FeatureInfoTest.java index 23358f9d210..85a8d5f13e3 100644 --- a/core/src/test/java/feast/core/model/FeatureInfoTest.java +++ b/core/src/test/java/feast/core/model/FeatureInfoTest.java @@ -27,6 +27,7 @@ import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.ValueProto.ValueType; +import java.time.Instant; import java.util.Date; import static org.hamcrest.Matchers.equalTo; @@ -168,4 +169,13 @@ public void shouldBuildCorrespondingResolvedSpec() { FeatureInfo resolved = featureInfo.resolve(); assertThat(resolved.getFeatureSpec(), equalTo(expected)); } + + @Test + public void shouldBeEqualToFeatureFromSameSpecs() { + FeatureInfo feature1 = new FeatureInfo(featureSpec, entityInfo, servingStorage, warehouseStorage, null); + feature1.setCreated(Date.from(Instant.ofEpochSecond(1))); + FeatureInfo feature2 = new FeatureInfo(featureSpec, entityInfo, servingStorage, warehouseStorage, null); + feature2.setCreated(Date.from(Instant.ofEpochSecond(2))); + assertThat(feature1.eq(feature2), equalTo(true)); + } } diff --git a/core/src/test/java/feast/core/model/StorageInfoTest.java b/core/src/test/java/feast/core/model/StorageInfoTest.java index d98ea0eea38..80cfa440de4 100644 --- a/core/src/test/java/feast/core/model/StorageInfoTest.java +++ b/core/src/test/java/feast/core/model/StorageInfoTest.java @@ -23,6 +23,7 @@ import org.junit.Before; import org.junit.Test; +import java.time.Instant; import java.util.Date; import static org.hamcrest.Matchers.equalTo; @@ -65,4 +66,13 @@ public void shouldBuildAndReturnCorrespondingDetail() { StorageDetail.newBuilder().setSpec(storageSpec).setLastUpdated(ts).build(); assertThat(storageInfo.getStorageDetail(), equalTo(expected)); } + + @Test + public void shouldBeEqualToStorageFromSameSpecs() { + StorageInfo storage1 = new StorageInfo(storageSpec); + storage1.setCreated(Date.from(Instant.ofEpochSecond(1))); + StorageInfo storage2 = new StorageInfo(storageSpec); + storage2.setCreated(Date.from(Instant.ofEpochSecond(2))); + assertThat(storage1.eq(storage2), equalTo(true)); + } } From 3fb8581b1459998415674074d3cc84b296c83610 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 12 Dec 2018 14:38:55 +0800 Subject: [PATCH 2/6] Add apply functions to core service --- .../java/feast/core/grpc/CoreServiceImpl.java | 18 +- .../java/feast/core/model/EntityInfo.java | 13 +- .../feast/core/model/FeatureGroupInfo.java | 96 +++++----- .../java/feast/core/model/FeatureInfo.java | 64 ++++--- .../java/feast/core/model/StorageInfo.java | 12 -- .../java/feast/core/service/SpecService.java | 171 +++++++++++------- .../java/feast/core/model/EntityInfoTest.java | 13 +- .../core/model/FeatureGroupInfoTest.java | 70 +++++-- .../feast/core/model/FeatureInfoTest.java | 150 ++++++++++----- .../feast/core/model/StorageInfoTest.java | 9 - .../feast/core/service/SpecServiceTest.java | 9 +- 11 files changed, 373 insertions(+), 252 deletions(-) diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 4dc3b5237af..a87397b0239 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -237,16 +237,16 @@ public void registerFeature( FeatureSpec request, StreamObserver responseObserver) { try { validator.validateFeatureSpec(request); - FeatureInfo feature = specService.registerFeature(request); + FeatureInfo feature = specService.applyFeature(request); RegisterFeatureResponse response = RegisterFeatureResponse.newBuilder().setFeatureId(feature.getId()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { - log.error("Error in registerFeature: {}", e); + log.error("Error in applyFeature: {}", e); responseObserver.onError(getRuntimeException(e)); } catch (IllegalArgumentException e) { - log.error("Error in registerFeature: {}", e); + log.error("Error in applyFeature: {}", e); responseObserver.onError(getBadRequestException(e)); } } @@ -262,16 +262,16 @@ public void registerFeatureGroup( StreamObserver responseObserver) { try { validator.validateFeatureGroupSpec(request); - FeatureGroupInfo featureGroup = specService.registerFeatureGroup(request); + FeatureGroupInfo featureGroup = specService.applyFeatureGroup(request); RegisterFeatureGroupResponse response = RegisterFeatureGroupResponse.newBuilder().setFeatureGroupId(featureGroup.getId()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { - log.error("Error in registerFeatureGroup: {}", e); + log.error("Error in applyFeatureGroup: {}", e); responseObserver.onError(getRuntimeException(e)); } catch (IllegalArgumentException e) { - log.error("Error in registerFeatureGroup: {}", e); + log.error("Error in applyFeatureGroup: {}", e); responseObserver.onError(getBadRequestException(e)); } } @@ -286,16 +286,16 @@ public void registerEntity( EntitySpec request, StreamObserver responseObserver) { try { validator.validateEntitySpec(request); - EntityInfo entity = specService.registerEntity(request); + EntityInfo entity = specService.applyEntity(request); RegisterEntityResponse response = RegisterEntityResponse.newBuilder().setEntityName(entity.getName()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { - log.error("Error in registerEntity: {}", e); + log.error("Error in applyEntity: {}", e); responseObserver.onError(getRuntimeException(e)); } catch (IllegalArgumentException e) { - log.error("Error in registerEntity: {}", e); + log.error("Error in applyEntity: {}", e); responseObserver.onError(getBadRequestException(e)); } } diff --git a/core/src/main/java/feast/core/model/EntityInfo.java b/core/src/main/java/feast/core/model/EntityInfo.java index 9cf0ad343bc..612c61ae359 100644 --- a/core/src/main/java/feast/core/model/EntityInfo.java +++ b/core/src/main/java/feast/core/model/EntityInfo.java @@ -88,14 +88,13 @@ public EntityDetail getEntityDetail() { } /** - * Checks if this is eq to the other given entity + * Updates the entity info with specifications from the incoming entity spec. * - * @param otherEntity - * @return boolean + * @param update new entity spec */ - public boolean eq(EntityInfo otherEntity) { - return otherEntity.getName().equals(this.getName()) && - otherEntity.getDescription().equals(this.getDescription()) && - otherEntity.getTags().equals(this.getTags()); + public void update(EntitySpec update) { + this.description = update.getDescription(); + this.tags = String.join(",", update.getTagsList()); } + } diff --git a/core/src/main/java/feast/core/model/FeatureGroupInfo.java b/core/src/main/java/feast/core/model/FeatureGroupInfo.java index d97ea70ecae..f302f540e5a 100644 --- a/core/src/main/java/feast/core/model/FeatureGroupInfo.java +++ b/core/src/main/java/feast/core/model/FeatureGroupInfo.java @@ -17,14 +17,14 @@ package feast.core.model; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; import feast.core.UIServiceProto.UIServiceTypes.FeatureGroupDetail; import feast.core.util.TypeConversion; import feast.specs.FeatureGroupSpecProto.FeatureGroupSpec; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; import javax.persistence.*; @@ -39,8 +39,7 @@ @Table(name = "feature_groups") public class FeatureGroupInfo extends AbstractTimestampEntity { - @Id - private String id; + @Id private String id; @Column(name = "tags") private String tags; @@ -63,43 +62,40 @@ public FeatureGroupInfo() { super(); } - public FeatureGroupInfo(FeatureGroupSpec spec, - StorageInfo servingStore, - StorageInfo warehouseStore) { + public FeatureGroupInfo( + FeatureGroupSpec spec, StorageInfo servingStore, StorageInfo warehouseStore) { this.id = spec.getId(); this.tags = String.join(",", spec.getTagsList()); this.servingStore = servingStore; this.warehouseStore = warehouseStore; this.servingStoreOpts = - TypeConversion.convertMapToJsonString(spec.getDataStores().getServing().getOptionsMap()); + TypeConversion.convertMapToJsonString(spec.getDataStores().getServing().getOptionsMap()); this.warehouseStoreOpts = - TypeConversion.convertMapToJsonString(spec.getDataStores().getWarehouse().getOptionsMap()); + TypeConversion.convertMapToJsonString(spec.getDataStores().getWarehouse().getOptionsMap()); } - /** - * Get the feature group spec associated with this record. - */ + /** Get the feature group spec associated with this record. */ public FeatureGroupSpec getFeatureGroupSpec() { DataStore servingDataStore = - DataStore.newBuilder() - .setId(servingStore.getId()) - .putAllOptions(TypeConversion.convertJsonStringToMap(servingStoreOpts)) - .build(); + DataStore.newBuilder() + .setId(servingStore.getId()) + .putAllOptions(TypeConversion.convertJsonStringToMap(servingStoreOpts)) + .build(); DataStore warehouseDataStore = - DataStore.newBuilder() - .setId(warehouseStore.getId()) - .putAllOptions(TypeConversion.convertJsonStringToMap(warehouseStoreOpts)) - .build(); + DataStore.newBuilder() + .setId(warehouseStore.getId()) + .putAllOptions(TypeConversion.convertJsonStringToMap(warehouseStoreOpts)) + .build(); DataStores dataStores = - DataStores.newBuilder() - .setWarehouse(warehouseDataStore) - .setServing(servingDataStore) - .build(); - return FeatureGroupSpec.newBuilder() - .setId(id) - .addAllTags(TypeConversion.convertTagStringToList(tags)) - .setDataStores(dataStores) + DataStores.newBuilder() + .setWarehouse(warehouseDataStore) + .setServing(servingDataStore) .build(); + return FeatureGroupSpec.newBuilder() + .setId(id) + .addAllTags(TypeConversion.convertTagStringToList(tags)) + .setDataStores(dataStores) + .build(); } /** @@ -107,24 +103,34 @@ public FeatureGroupSpec getFeatureGroupSpec() { */ public FeatureGroupDetail getFeatureGroupDetail() { return FeatureGroupDetail.newBuilder() - .setSpec(this.getFeatureGroupSpec()) - .setLastUpdated(TypeConversion.convertTimestamp(this.getLastUpdated())) - .build(); + .setSpec(this.getFeatureGroupSpec()) + .setLastUpdated(TypeConversion.convertTimestamp(this.getLastUpdated())) + .build(); } - /** - * Checks if this is eq to the other given feature group - * - * @param otherFeatureGroup - * @return boolean - */ - public boolean eq(FeatureGroupInfo otherFeatureGroup) { - return otherFeatureGroup.getId() == this.id && - otherFeatureGroup.getTags() == this.getTags() && - otherFeatureGroup.getServingStoreOpts() == this.servingStoreOpts && - getStorageId(otherFeatureGroup.getServingStore()) == getStorageId(this.getServingStore()) && - otherFeatureGroup.getWarehouseStoreOpts() == this.warehouseStoreOpts && - getStorageId(otherFeatureGroup.getWarehouseStore()) == getStorageId(this.getWarehouseStore()); + public void update(FeatureGroupSpec update) throws IllegalArgumentException { + if (!isLegalUpdate(update)) { + throw new IllegalArgumentException( + "Feature group already exists. Update only allowed for fields: [tags]"); + } + this.tags = String.join(",", update.getTagsList()); + } + + private boolean isLegalUpdate(FeatureGroupSpec update) { + DataStore updatedWarehouseStore = + update.getDataStores().hasWarehouse() ? update.getDataStores().getWarehouse() : null; + DataStore updatedServingStore = + update.getDataStores().hasServing() ? update.getDataStores().getServing() : null; + return isStoreEqual(this.warehouseStore, this.warehouseStoreOpts, updatedWarehouseStore) + && isStoreEqual(this.servingStore, this.servingStoreOpts, updatedServingStore); + } + + private boolean isStoreEqual(StorageInfo oldStore, String oldStoreOpts, DataStore newStore) { + return getStorageId(oldStore).equals(newStore == null ? "" : newStore.getId()) + && oldStoreOpts.equals( + newStore == null + ? "" + : TypeConversion.convertMapToJsonString(newStore.getOptionsMap())); } private String getStorageId(StorageInfo storage) { diff --git a/core/src/main/java/feast/core/model/FeatureInfo.java b/core/src/main/java/feast/core/model/FeatureInfo.java index b219565d9e2..567a4bec67b 100644 --- a/core/src/main/java/feast/core/model/FeatureInfo.java +++ b/core/src/main/java/feast/core/model/FeatureInfo.java @@ -17,8 +17,6 @@ package feast.core.model; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import feast.core.UIServiceProto.UIServiceTypes.FeatureDetail; import feast.core.storage.BigQueryStorageManager; import feast.core.util.TypeConversion; @@ -30,11 +28,9 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; -import org.apache.logging.log4j.util.Strings; import javax.persistence.*; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -133,10 +129,11 @@ public FeatureInfo( if (spec.getDataStores() != null) { this.servingStore = servingStore; this.servingStoreOpts = - TypeConversion.convertMapToJsonString(spec.getDataStores().getServing().getOptionsMap()); + TypeConversion.convertMapToJsonString(spec.getDataStores().getServing().getOptionsMap()); this.warehouseStore = warehouseStore; this.warehouseStoreOpts = - TypeConversion.convertMapToJsonString(spec.getDataStores().getWarehouse().getOptionsMap()); + TypeConversion.convertMapToJsonString( + spec.getDataStores().getWarehouse().getOptionsMap()); } this.bigQueryView = createBigqueryViewLink(warehouseStore); } @@ -258,31 +255,50 @@ private String createBigqueryViewLink(StorageInfo warehouseStore) { } /** - * Checks if this is eq to the other given feature + * Updates the feature info with specifications from the incoming feature spec. * - * @param otherFeature - * @return boolean + *

TODO: maybe allow changes to id, store etc if no jobs are feeding into this feature + * + * @param update new feature spec */ - public boolean eq(FeatureInfo otherFeature) { - return otherFeature.getId() == this.id && - otherFeature.getEntity().getName() == this.entity.getName() && - otherFeature.getOwner() == this.owner && - otherFeature.getUri() == this.uri && - otherFeature.getDescription() == this.description && - otherFeature.getGranularity() == this.granularity && - otherFeature.getValueType() == this.valueType && - otherFeature.getTags() == this.tags && - otherFeature.getOptions() == this.options && - getFeatureGroupId(otherFeature.getFeatureGroup()) == getFeatureGroupId(this.getFeatureGroup()) && - otherFeature.getServingStoreOpts() == this.servingStoreOpts && - getStorageId(otherFeature.getServingStore()) == getStorageId(this.getServingStore()) && - otherFeature.getWarehouseStoreOpts() == this.warehouseStoreOpts && - getStorageId(otherFeature.getWarehouseStore()) == getStorageId(this.getWarehouseStore()); + public void update(FeatureSpec update) throws IllegalArgumentException { + if (!isLegalUpdate(update)) { + throw new IllegalArgumentException( + "Feature already exists. Update only allowed for fields: [owner, description, uri, tags]"); + } + this.owner = update.getOwner(); + this.description = update.getDescription(); + this.uri = update.getUri(); + this.tags = String.join(",", update.getTagsList()); + } + + private boolean isLegalUpdate(FeatureSpec update) { + DataStore updatedWarehouseStore = + update.getDataStores().hasWarehouse() ? update.getDataStores().getWarehouse() : null; + DataStore updatedServingStore = + update.getDataStores().hasServing() ? update.getDataStores().getServing() : null; + return update.getName().equals(this.name) + && update.getEntity().equals(this.entity.getName()) + && update.getGranularityValue() == this.granularity.getNumber() + && update.getValueTypeValue() == this.valueType.getNumber() + && update.getGroup().equals(getFeatureGroupId(this.featureGroup)) + && TypeConversion.convertMapToJsonString(update.getOptionsMap()).equals(this.options) + && isStoreEqual(this.warehouseStore, this.warehouseStoreOpts, updatedWarehouseStore) + && isStoreEqual(this.servingStore, this.servingStoreOpts, updatedServingStore); + } + + private boolean isStoreEqual(StorageInfo oldStore, String oldStoreOpts, DataStore newStore) { + return getStorageId(oldStore).equals(newStore == null ? "" : newStore.getId()) + && oldStoreOpts.equals( + newStore == null + ? "" + : TypeConversion.convertMapToJsonString(newStore.getOptionsMap())); } private String getFeatureGroupId(FeatureGroupInfo featureGroupInfo) { return featureGroupInfo == null ? "" : featureGroupInfo.getId(); } + private String getStorageId(StorageInfo storage) { return storage == null ? "" : storage.getId(); } diff --git a/core/src/main/java/feast/core/model/StorageInfo.java b/core/src/main/java/feast/core/model/StorageInfo.java index 0de83fb61ca..f38463c9d22 100644 --- a/core/src/main/java/feast/core/model/StorageInfo.java +++ b/core/src/main/java/feast/core/model/StorageInfo.java @@ -80,16 +80,4 @@ public StorageDetail getStorageDetail() { .setLastUpdated(convertTimestamp(this.getLastUpdated())) .build(); } - - /** - * Checks if this is eq to the other given storage - * - * @param otherStorage - * @return boolean - */ - public boolean eq(StorageInfo otherStorage) { - return otherStorage.getId().equals(this.id) && - otherStorage.getType().equals(this.type) && - otherStorage.getOptions().equals(this.options); - } } diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 1a5f141a418..69f283fe932 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -31,13 +31,12 @@ import feast.core.model.FeatureInfo; import feast.core.model.StorageInfo; import feast.core.storage.SchemaManager; +import feast.core.util.TypeConversion; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureGroupSpecProto.FeatureGroupSpec; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -196,79 +195,98 @@ public List listStorage() throws RetrievalException { } /** - * Registers given feature spec to the registry + * Applies the given feature spec to the registry. If the feature does not yet exist, it will be + * registered to the system. If it does, the existing feature will be updated with the new + * information. + * + *

Note that specifications that will affect downstream resources (e.g. id, storage location) + * cannot be changed. * * @param spec FeatureSpec * @return registered FeatureInfo * @throws RegistrationException if registration fails */ - public FeatureInfo registerFeature(FeatureSpec spec) throws RegistrationException { + public FeatureInfo applyFeature(FeatureSpec spec) throws RegistrationException { try { - EntityInfo entity = entityInfoRepository.findById(spec.getEntity()).orElse(null); - FeatureGroupInfo featureGroupInfo = - featureGroupInfoRepository.findById(spec.getGroup()).orElse(null); - StorageInfo servingStore = - storageInfoRepository.findById(spec.getDataStores().getServing().getId()).orElse(null); - StorageInfo warehouseStore = - storageInfoRepository.findById(spec.getDataStores().getWarehouse().getId()).orElse(null); - FeatureInfo featureInfo = - new FeatureInfo(spec, entity, servingStore, warehouseStore, featureGroupInfo); - - FeatureInfo resolvedFeatureInfo = featureInfo.resolve(); - FeatureSpec resolvedFeatureSpec = resolvedFeatureInfo.getFeatureSpec(); - schemaManager.registerFeature(resolvedFeatureSpec); - + FeatureInfo featureInfo = featureInfoRepository.findById(spec.getId()).orElse(null); + String action; + if (featureInfo != null) { + featureInfo.update(spec); + action = "Updated"; + } else { + EntityInfo entity = entityInfoRepository.findById(spec.getEntity()).orElse(null); + FeatureGroupInfo featureGroupInfo = + featureGroupInfoRepository.findById(spec.getGroup()).orElse(null); + StorageInfo servingStore = + storageInfoRepository.findById(spec.getDataStores().getServing().getId()).orElse(null); + StorageInfo warehouseStore = + storageInfoRepository + .findById(spec.getDataStores().getWarehouse().getId()) + .orElse(null); + featureInfo = new FeatureInfo(spec, entity, servingStore, warehouseStore, featureGroupInfo); + FeatureInfo resolvedFeatureInfo = featureInfo.resolve(); + FeatureSpec resolvedFeatureSpec = resolvedFeatureInfo.getFeatureSpec(); + schemaManager.registerFeature(resolvedFeatureSpec); + action = "Registered"; + } FeatureInfo out = featureInfoRepository.saveAndFlush(featureInfo); if (!out.getId().equals(spec.getId())) { - throw new RegistrationException("failed to register new feature"); + throw new RegistrationException("failed to register or update feature"); } AuditLogger.log( - "Feature", - spec.getId(), - "Registered", - "New feature registered: %s", - JsonFormat.printer().print(spec)); + "Feature", spec.getId(), action, "Feature applied: %s", JsonFormat.printer().print(spec)); return out; + } catch (Exception e) { throw new RegistrationException( - Strings.lenientFormat("Failed to register new feature %s: %s", spec, e.getMessage()), e); + Strings.lenientFormat("Failed to apply feature %s: %s", spec, e.getMessage()), e); } } /** - * Registers given feature group spec to the registry + * Applies the given feature group spec to the registry. If the entity does not yet exist, it will be + * registered to the system. Otherwise, the fields will be updated as per the given feature group spec. * * @param spec FeatureGroupSpec * @return registered FeatureGroupInfo * @throws RegistrationException if registration fails */ - public FeatureGroupInfo registerFeatureGroup(FeatureGroupSpec spec) throws RegistrationException { + public FeatureGroupInfo applyFeatureGroup(FeatureGroupSpec spec) throws RegistrationException { try { - StorageInfo servingStore = - storageInfoRepository - .findById( - spec.getDataStores().hasServing() - ? spec.getDataStores().getServing().getId() - : "") - .orElse(null); - StorageInfo warehouseStore = - storageInfoRepository - .findById( - spec.getDataStores().hasServing() - ? spec.getDataStores().getWarehouse().getId() - : "") - .orElse(null); - FeatureGroupInfo featureGroupInfo = new FeatureGroupInfo(spec, servingStore, warehouseStore); + FeatureGroupInfo featureGroupInfo = featureGroupInfoRepository.findById(spec.getId()).orElse(null); + String action; + if (featureGroupInfo != null) { + featureGroupInfo.update(spec); + action = "Updated"; + } else { + StorageInfo servingStore = + storageInfoRepository + .findById( + spec.getDataStores().hasServing() + ? spec.getDataStores().getServing().getId() + : "") + .orElse(null); + StorageInfo warehouseStore = + storageInfoRepository + .findById( + spec.getDataStores().hasServing() + ? spec.getDataStores().getWarehouse().getId() + : "") + .orElse(null); + featureGroupInfo = + new FeatureGroupInfo(spec, servingStore, warehouseStore); + action = "Registered"; + } FeatureGroupInfo out = featureGroupInfoRepository.saveAndFlush(featureGroupInfo); if (!out.getId().equals(spec.getId())) { - throw new RegistrationException("failed to register new feature group"); + throw new RegistrationException("failed to register or update feature group"); } AuditLogger.log( - "FeatureGroup", - spec.getId(), - "Registered", - "New feature group registered: %s", - JsonFormat.printer().print(spec)); + "FeatureGroup", + spec.getId(), + action, + "Feature group applied: %s", + JsonFormat.printer().print(spec)); return out; } catch (Exception e) { throw new RegistrationException( @@ -279,29 +297,34 @@ public FeatureGroupInfo registerFeatureGroup(FeatureGroupSpec spec) throws Regis } /** - * Registers given entity spec to the registry + * Applies the given entity spec to the registry. If the entity does not yet exist, it will be + * registered to the system. Otherwise, the fields will be updated as per the given entity spec. * * @param spec EntitySpec * @return registered EntityInfo * @throws RegistrationException if registration fails */ - public EntityInfo registerEntity(EntitySpec spec) throws RegistrationException { + public EntityInfo applyEntity(EntitySpec spec) throws RegistrationException { try { - EntityInfo entityInfo = new EntityInfo(spec); + EntityInfo entityInfo = entityInfoRepository.findById(spec.getName()).orElse(null); + String action; + if (entityInfo != null) { + entityInfo.update(spec); + action = "Updated"; + } else { + entityInfo = new EntityInfo(spec); + action = "Registered"; + } EntityInfo out = entityInfoRepository.saveAndFlush(entityInfo); if (!out.getName().equals(spec.getName())) { - throw new RegistrationException("failed to register new entity"); + throw new RegistrationException("failed to register or update entity"); } AuditLogger.log( - "Entity", - spec.getName(), - "Registered", - "New entity registered: %s", - JsonFormat.printer().print(spec)); + "Entity", spec.getName(), action, "Entity: %s", JsonFormat.printer().print(spec)); return out; } catch (Exception e) { throw new RegistrationException( - Strings.lenientFormat("Failed to register new entity %s: %s", spec, e.getMessage()), e); + Strings.lenientFormat("Failed to apply entity %s: %s", spec, e.getMessage()), e); } } @@ -314,19 +337,27 @@ public EntityInfo registerEntity(EntitySpec spec) throws RegistrationException { */ public StorageInfo registerStorage(StorageSpec spec) throws RegistrationException { try { - StorageInfo storageInfo = new StorageInfo(spec); - StorageInfo out = storageInfoRepository.saveAndFlush(storageInfo); - if (!out.getId().equals(spec.getId())) { - throw new RegistrationException("failed to register new storage"); + StorageInfo storageInfo = storageInfoRepository.findById(spec.getId()).orElse(null); + if (storageInfo != null) { + if (!storageInfo.getType().equals(spec.getType()) && !storageInfo.getOptions().equals(TypeConversion.convertMapToJsonString(spec.getOptionsMap()))) { + throw new IllegalArgumentException("updating storage specs is not allowed"); + } + return storageInfo; + } else { + storageInfo = new StorageInfo(spec); + StorageInfo out = storageInfoRepository.saveAndFlush(storageInfo); + if (!out.getId().equals(spec.getId())) { + throw new RegistrationException("failed to register or update storage"); + } + schemaManager.registerStorage(spec); + AuditLogger.log( + "Storage", + spec.getId(), + "Registered", + "New storage registered: %s", + JsonFormat.printer().print(spec)); + return out; } - schemaManager.registerStorage(spec); - AuditLogger.log( - "Storage", - spec.getId(), - "Registered", - "New storage registered: %s", - JsonFormat.printer().print(spec)); - return out; } catch (Exception e) { throw new RegistrationException( Strings.lenientFormat("Failed to register new storage %s: %s", spec, e.getMessage()), e); diff --git a/core/src/test/java/feast/core/model/EntityInfoTest.java b/core/src/test/java/feast/core/model/EntityInfoTest.java index eff317795af..9554c577016 100644 --- a/core/src/test/java/feast/core/model/EntityInfoTest.java +++ b/core/src/test/java/feast/core/model/EntityInfoTest.java @@ -17,6 +17,7 @@ package feast.core.model; +import com.google.api.client.util.Lists; import com.google.protobuf.Timestamp; import org.junit.Before; import org.junit.Test; @@ -69,11 +70,11 @@ public void shouldBuildAndReturnCorrespondingDetail() { } @Test - public void shouldBeEqualToEntityFromSameSpecs() { - EntityInfo entity1 = new EntityInfo(entitySpec); - entity1.setCreated(Date.from(Instant.ofEpochSecond(1))); - EntityInfo entity2 = new EntityInfo(entitySpec); - entity1.setCreated(Date.from(Instant.ofEpochSecond(2))); - assertThat(entity1.eq(entity2), equalTo(true)); + public void shouldUpdateTagAndDescription() { + EntityInfo entityInfo = new EntityInfo("entity", "test entity", "tag1,tag2", Lists.newArrayList(), false); + EntitySpec update = EntitySpec.newBuilder().setName("entity").setDescription("overwrite").addTags("newtag").build(); + EntityInfo expected = new EntityInfo("entity", "overwrite", "newtag", Lists.newArrayList(), false); + entityInfo.update(update); + assertThat(entityInfo, equalTo(expected)); } } diff --git a/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java b/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java index ebfd7c3a350..6943fbf76cf 100644 --- a/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java +++ b/core/src/test/java/feast/core/model/FeatureGroupInfoTest.java @@ -18,14 +18,16 @@ package feast.core.model; import com.google.protobuf.Timestamp; -import org.junit.Before; -import org.junit.Test; import feast.core.UIServiceProto.UIServiceTypes.FeatureGroupDetail; +import feast.core.exception.RetrievalException; import feast.specs.FeatureGroupSpecProto.FeatureGroupSpec; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; -import java.time.Instant; import java.util.Date; import static org.hamcrest.Matchers.equalTo; @@ -37,6 +39,8 @@ public class FeatureGroupInfoTest { private StorageInfo servingStorage; private StorageInfo warehouseStorage; + @Rule public final ExpectedException exception = ExpectedException.none(); + @Before public void setUp() { servingStorage = new StorageInfo(); @@ -56,15 +60,15 @@ public void setUp() { DataStore servingStore = DataStore.newBuilder().setId("REDIS1").build(); DataStore warehouseStore = DataStore.newBuilder().setId("REDIS2").build(); DataStores dataStores = - DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); + DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); featureGroupSpec = - FeatureGroupSpec.newBuilder() - .setId("test") - .addTags("tag1") - .addTags("tag2") - .setDataStores(dataStores) - .build(); + FeatureGroupSpec.newBuilder() + .setId("test") + .addTags("tag1") + .addTags("tag2") + .setDataStores(dataStores) + .build(); } @Test @@ -74,7 +78,9 @@ public void shouldBuildAndReturnCorrespondingSpec() { @Test public void shouldCorrectlyInitialiseFromGivenSpec() { - assertThat(new FeatureGroupInfo(featureGroupSpec, servingStorage, warehouseStorage), equalTo(featureGroupInfo)); + assertThat( + new FeatureGroupInfo(featureGroupSpec, servingStorage, warehouseStorage), + equalTo(featureGroupInfo)); } @Test @@ -82,16 +88,44 @@ public void shouldBuildAndReturnCorrespondingDetail() { featureGroupInfo.setLastUpdated(new Date(1000)); Timestamp ts = Timestamp.newBuilder().setSeconds(1).build(); FeatureGroupDetail expected = - FeatureGroupDetail.newBuilder().setSpec(featureGroupSpec).setLastUpdated(ts).build(); + FeatureGroupDetail.newBuilder().setSpec(featureGroupSpec).setLastUpdated(ts).build(); assertThat(featureGroupInfo.getFeatureGroupDetail(), equalTo(expected)); } @Test - public void shouldBeEqualToFeatureGroupFromSameSpecs() { - FeatureGroupInfo featureGroup1 = new FeatureGroupInfo(featureGroupSpec, servingStorage, warehouseStorage); - featureGroup1.setCreated(Date.from(Instant.ofEpochSecond(1))); - FeatureGroupInfo featureGroup2 = new FeatureGroupInfo(featureGroupSpec, servingStorage, warehouseStorage); - featureGroup2.setCreated(Date.from(Instant.ofEpochSecond(2))); - assertThat(featureGroup1.eq(featureGroup2), equalTo(true)); + public void shouldUpdateTags() { + DataStore servingStore = DataStore.newBuilder().setId("REDIS1").build(); + DataStore warehouseStore = DataStore.newBuilder().setId("REDIS2").build(); + DataStores dataStores = + DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); + + FeatureGroupSpec update = + FeatureGroupSpec.newBuilder() + .setId("test") + .addTags("newtag") + .setDataStores(dataStores) + .build(); + featureGroupInfo.update(update); + + FeatureGroupInfo expected = new FeatureGroupInfo(update, servingStorage, warehouseStorage); + assertThat(featureGroupInfo, equalTo(expected)); + } + + @Test + public void shouldThrowErrorIfDatastoresChanged() { + DataStore servingStore = DataStore.newBuilder().setId("REDIS3").build(); + DataStore warehouseStore = DataStore.newBuilder().setId("REDIS2").build(); + DataStores dataStores = + DataStores.newBuilder().setServing(servingStore).setWarehouse(warehouseStore).build(); + + FeatureGroupSpec update = + FeatureGroupSpec.newBuilder() + .setId("test") + .addTags("newtag") + .setDataStores(dataStores) + .build(); + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Feature group already exists. Update only allowed for fields: [tags]"); + featureGroupInfo.update(update); } } diff --git a/core/src/test/java/feast/core/model/FeatureInfoTest.java b/core/src/test/java/feast/core/model/FeatureInfoTest.java index 85a8d5f13e3..c3083fbce18 100644 --- a/core/src/test/java/feast/core/model/FeatureInfoTest.java +++ b/core/src/test/java/feast/core/model/FeatureInfoTest.java @@ -18,16 +18,17 @@ package feast.core.model; import com.google.protobuf.Timestamp; -import org.junit.Before; -import org.junit.Test; import feast.core.UIServiceProto.UIServiceTypes.FeatureDetail; -import feast.types.GranularityProto.Granularity; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.types.GranularityProto.Granularity; import feast.types.ValueProto.ValueType; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; -import java.time.Instant; import java.util.Date; import static org.hamcrest.Matchers.equalTo; @@ -40,6 +41,8 @@ public class FeatureInfoTest { private StorageInfo servingStorage; private StorageInfo warehouseStorage; + @Rule public final ExpectedException exception = ExpectedException.none(); + @Before public void setUp() { entityInfo = new EntityInfo(); @@ -72,25 +75,25 @@ public void setUp() { DataStore servingDataStore = DataStore.newBuilder().setId("REDIS1").build(); DataStore warehouseDataStore = DataStore.newBuilder().setId("BIGQUERY").build(); DataStores dataStores = - DataStores.newBuilder() - .setServing(servingDataStore) - .setWarehouse(warehouseDataStore) - .build(); + DataStores.newBuilder() + .setServing(servingDataStore) + .setWarehouse(warehouseDataStore) + .build(); featureSpec = - FeatureSpec.newBuilder() - .setId("entity.NONE.name") - .setName("name") - .setOwner("owner") - .setDescription("desc") - .setEntity("entity") - .setUri("uri") - .setGranularity(Granularity.Enum.NONE) - .setValueType(ValueType.Enum.BYTES) - .addTags("tag1") - .addTags("tag2") - .setDataStores(dataStores) - .build(); + FeatureSpec.newBuilder() + .setId("entity.NONE.name") + .setName("name") + .setOwner("owner") + .setDescription("desc") + .setEntity("entity") + .setUri("uri") + .setGranularity(Granularity.Enum.NONE) + .setValueType(ValueType.Enum.BYTES) + .addTags("tag1") + .addTags("tag2") + .setDataStores(dataStores) + .build(); } @Test @@ -100,7 +103,9 @@ public void shouldBuildAndReturnCorrespondingSpec() { @Test public void shouldCorrectlyInitialiseFromGivenSpec() { - assertThat(new FeatureInfo(featureSpec, entityInfo, servingStorage, warehouseStorage, null), equalTo(featureInfo)); + assertThat( + new FeatureInfo(featureSpec, entityInfo, servingStorage, warehouseStorage, null), + equalTo(featureInfo)); } @Test @@ -110,13 +115,13 @@ public void shouldBuildAndReturnCorrespondingDetail() { featureInfo.setBigQueryView("bqviewurl"); Timestamp ts = Timestamp.newBuilder().setSeconds(1).build(); FeatureDetail expected = - FeatureDetail.newBuilder() - .setSpec(featureSpec) - .setBigqueryView("bqviewurl") - .setEnabled(true) - .setLastUpdated(ts) - .setCreated(ts) - .build(); + FeatureDetail.newBuilder() + .setSpec(featureSpec) + .setBigqueryView("bqviewurl") + .setEnabled(true) + .setLastUpdated(ts) + .setCreated(ts) + .build(); assertThat(featureInfo.getFeatureDetail(), equalTo(expected)); } @@ -144,38 +149,87 @@ public void shouldBuildCorrespondingResolvedSpec() { DataStore servingDataStore = DataStore.newBuilder().setId("REDIS1").build(); DataStore warehouseDataStore = DataStore.newBuilder().setId("BIGQUERY").build(); + DataStores dataStores = + DataStores.newBuilder() + .setServing(servingDataStore) + .setWarehouse(warehouseDataStore) + .build(); + + FeatureSpec expected = + FeatureSpec.newBuilder() + .setId("entity.NONE.name") + .setName("name") + .setOwner("owner") + .setDescription("desc") + .setEntity("entity") + .setUri("uri") + .setGroup("testGroup") + .setGranularity(Granularity.Enum.NONE) + .setValueType(ValueType.Enum.BYTES) + .addTags("tag1") + .addTags("tag2") + .addTags("inherited") + .setDataStores(dataStores) + .build(); + FeatureInfo resolved = featureInfo.resolve(); + assertThat(resolved.getFeatureSpec(), equalTo(expected)); + } + + @Test + public void shouldUpdateMutableFields() { + DataStore servingDataStore = DataStore.newBuilder().setId("REDIS1").build(); + DataStore warehouseDataStore = DataStore.newBuilder().setId("BIGQUERY").build(); + DataStores dataStores = + DataStores.newBuilder() + .setServing(servingDataStore) + .setWarehouse(warehouseDataStore) + .build(); + + FeatureSpec update = + FeatureSpec.newBuilder() + .setId("entity.NONE.name") + .setName("name") + .setOwner("owner2") + .setDescription("overwrite") + .setEntity("entity") + .setUri("new_uri") + .setGranularity(Granularity.Enum.NONE) + .setValueType(ValueType.Enum.BYTES) + .addTags("new_tag") + .setDataStores(dataStores) + .build(); + featureInfo.update(featureSpec); + FeatureInfo expected = + new FeatureInfo(update, entityInfo, servingStorage, warehouseStorage, null); + assertThat(featureInfo, equalTo(expected)); + } + + @Test + public void shouldThrowExceptionIfImmutableFieldsChanged() { + DataStore servingDataStore = DataStore.newBuilder().setId("REDIS2").build(); + DataStore warehouseDataStore = DataStore.newBuilder().setId("BIGQUERY").build(); DataStores dataStores = DataStores.newBuilder() .setServing(servingDataStore) .setWarehouse(warehouseDataStore) .build(); - FeatureSpec expected = + FeatureSpec update = FeatureSpec.newBuilder() .setId("entity.NONE.name") .setName("name") - .setOwner("owner") - .setDescription("desc") + .setOwner("owner2") + .setDescription("overwrite") .setEntity("entity") - .setUri("uri") - .setGroup("testGroup") + .setUri("new_uri") .setGranularity(Granularity.Enum.NONE) - .setValueType(ValueType.Enum.BYTES) - .addTags("tag1") - .addTags("tag2") - .addTags("inherited") + .setValueType(ValueType.Enum.INT32) + .addTags("new_tag") .setDataStores(dataStores) .build(); - FeatureInfo resolved = featureInfo.resolve(); - assertThat(resolved.getFeatureSpec(), equalTo(expected)); - } - @Test - public void shouldBeEqualToFeatureFromSameSpecs() { - FeatureInfo feature1 = new FeatureInfo(featureSpec, entityInfo, servingStorage, warehouseStorage, null); - feature1.setCreated(Date.from(Instant.ofEpochSecond(1))); - FeatureInfo feature2 = new FeatureInfo(featureSpec, entityInfo, servingStorage, warehouseStorage, null); - feature2.setCreated(Date.from(Instant.ofEpochSecond(2))); - assertThat(feature1.eq(feature2), equalTo(true)); + exception.expect(IllegalArgumentException.class); + exception.expectMessage( "Feature already exists. Update only allowed for fields: [owner, description, uri, tags]"); + featureInfo.update(update); } } diff --git a/core/src/test/java/feast/core/model/StorageInfoTest.java b/core/src/test/java/feast/core/model/StorageInfoTest.java index 80cfa440de4..7c4e7134411 100644 --- a/core/src/test/java/feast/core/model/StorageInfoTest.java +++ b/core/src/test/java/feast/core/model/StorageInfoTest.java @@ -66,13 +66,4 @@ public void shouldBuildAndReturnCorrespondingDetail() { StorageDetail.newBuilder().setSpec(storageSpec).setLastUpdated(ts).build(); assertThat(storageInfo.getStorageDetail(), equalTo(expected)); } - - @Test - public void shouldBeEqualToStorageFromSameSpecs() { - StorageInfo storage1 = new StorageInfo(storageSpec); - storage1.setCreated(Date.from(Instant.ofEpochSecond(1))); - StorageInfo storage2 = new StorageInfo(storageSpec); - storage2.setCreated(Date.from(Instant.ofEpochSecond(2))); - assertThat(storage1.eq(storage2), equalTo(true)); - } } diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 674f87840e4..531adf6b615 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -326,7 +326,7 @@ public void shouldRegisterFeatureWithGroupInheritance() { storageInfoRepository, featureGroupInfoRepository, schemaManager); - FeatureInfo actual = specService.registerFeature(spec); + FeatureInfo actual = specService.applyFeature(spec); verify(schemaManager).registerFeature(resolvedSpecCaptor.capture()); assertThat(resolvedSpecCaptor.getValue(), equalTo(resolvedSpec)); @@ -363,7 +363,7 @@ public void shouldRegisterFeatureGroupIfStoresArePresent() { storageInfoRepository, featureGroupInfoRepository, schemaManager); - FeatureGroupInfo actual = specService.registerFeatureGroup(spec); + FeatureGroupInfo actual = specService.applyFeatureGroup(spec); assertThat(actual, equalTo(expectedFeatureGroupInfo)); } @@ -395,7 +395,7 @@ public void shouldThrowRegistrationExceptionWhenRegisteringFeatureGroupIfStoresM schemaManager); exception.expect(RegistrationException.class); - specService.registerFeatureGroup(spec); + specService.applyFeatureGroup(spec); } @Test @@ -415,7 +415,7 @@ public void shouldRegisterEntity() { storageInfoRepository, featureGroupInfoRepository, schemaManager); - EntityInfo actual = specService.registerEntity(spec); + EntityInfo actual = specService.applyEntity(spec); assertThat(actual, equalTo(entityInfo)); } @@ -434,4 +434,5 @@ public void shouldRegisterStorage() { StorageInfo actual = specService.registerStorage(spec); assertThat(actual, equalTo(storageInfo)); } + } From 5eb5c1d104b8c3000b8f5c7fb07d65c58883c95d Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 12 Dec 2018 15:48:16 +0800 Subject: [PATCH 3/6] Update go package to reflect new repository --- cli/feast/cmd/jobs.go | 5 ++--- cli/feast/cmd/list.go | 2 +- cli/feast/cmd/root.go | 2 -- cli/feast/main.go | 2 +- cli/feast/pkg/printer/job.go | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cli/feast/cmd/jobs.go b/cli/feast/cmd/jobs.go index 3f151a803b1..16ec75ae32c 100644 --- a/cli/feast/cmd/jobs.go +++ b/cli/feast/cmd/jobs.go @@ -20,9 +20,8 @@ import ( "fmt" "io/ioutil" - "feast/cli/feast/pkg/parse" - "feast/cli/feast/pkg/printer" - + "github.com/gojektech/feast/cli/feast/pkg/parse" + "github.com/gojektech/feast/cli/feast/pkg/printer" "github.com/gojektech/feast/go-feast-proto/feast/core" "github.com/spf13/cobra" diff --git a/cli/feast/cmd/list.go b/cli/feast/cmd/list.go index 8cd52f20b5d..9918e36c772 100644 --- a/cli/feast/cmd/list.go +++ b/cli/feast/cmd/list.go @@ -17,12 +17,12 @@ package cmd import ( "context" "errors" - "feast/cli/feast/pkg/util" "fmt" "os" "strings" "text/tabwriter" + "github.com/gojektech/feast/cli/feast/pkg/util" "github.com/gojektech/feast/go-feast-proto/feast/core" "github.com/golang/protobuf/ptypes/empty" diff --git a/cli/feast/cmd/root.go b/cli/feast/cmd/root.go index 10af51f075a..18ab4d6a5d4 100644 --- a/cli/feast/cmd/root.go +++ b/cli/feast/cmd/root.go @@ -15,7 +15,6 @@ package cmd import ( - "fmt" "os" "github.com/spf13/cobra" @@ -42,7 +41,6 @@ func init() { func handleErr(err error) { if err != nil { - fmt.Println(err) os.Exit(1) } } diff --git a/cli/feast/main.go b/cli/feast/main.go index e07feefd224..ae3f14a8edd 100644 --- a/cli/feast/main.go +++ b/cli/feast/main.go @@ -14,7 +14,7 @@ package main -import "feast/cli/feast/cmd" +import "github.com/gojektech/feast/cli/feast/cmd" func main() { cmd.Execute() diff --git a/cli/feast/pkg/printer/job.go b/cli/feast/pkg/printer/job.go index c18ededc4d0..c128f30f899 100644 --- a/cli/feast/pkg/printer/job.go +++ b/cli/feast/pkg/printer/job.go @@ -15,10 +15,10 @@ package printer import ( - "feast/cli/feast/pkg/util" "fmt" "strings" + "github.com/gojektech/feast/cli/feast/pkg/util" "github.com/gojektech/feast/go-feast-proto/feast/core" ) From 3443bae6d360915bbe97012ac7db266c25f2743c Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 12 Dec 2018 15:54:30 +0800 Subject: [PATCH 4/6] Change register command to apply --- cli/feast/cmd/{register.go => apply.go} | 57 ++- cli/feast/cmd/version.go | 2 +- .../java/feast/core/grpc/CoreServiceImpl.java | 32 +- go-feast-proto/feast/core/CoreService.pb.go | 370 +++++++++--------- protos/feast/core/CoreService.proto | 24 +- 5 files changed, 239 insertions(+), 246 deletions(-) rename cli/feast/cmd/{register.go => apply.go} (56%) diff --git a/cli/feast/cmd/register.go b/cli/feast/cmd/apply.go similarity index 56% rename from cli/feast/cmd/register.go rename to cli/feast/cmd/apply.go index 6f4b72aae6b..652e047d7db 100644 --- a/cli/feast/cmd/register.go +++ b/cli/feast/cmd/apply.go @@ -21,18 +21,17 @@ import ( "io/ioutil" "path/filepath" + "github.com/gojektech/feast/cli/feast/pkg/parse" "github.com/gojektech/feast/go-feast-proto/feast/core" - "feast/cli/feast/pkg/parse" - "github.com/spf13/cobra" ) -// registerCmd represents the register command -var registerCmd = &cobra.Command{ - Use: "register [resource] [filepaths...]", - Short: "Register a resource given one or many yaml files.", - Long: `Register a resource from one or multiple yamls. +// applyCmd represents the apply command +var applyCmd = &cobra.Command{ + Use: "apply [resource] [filepaths...]", + Short: "Apply a resource given one or many yaml files.", + Long: `Apply a resource from one or multiple yamls. Valid resources include: - entity @@ -41,9 +40,9 @@ Valid resources include: - storage Examples: -- feast register entity entity.yml -- feast register storage storage1.yml storage2.yml -- feast register feature *-feature.yml`, +- feast apply entity entity.yml +- feast apply storage storage1.yml storage2.yml +- feast apply feature *-feature.yml`, RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return cmd.Help() @@ -51,7 +50,7 @@ Examples: if len(args) < 2 { fmt.Println(args) - return errors.New("invalid number of arguments for register command") + return errors.New("invalid number of arguments for apply command") } initConn() @@ -62,12 +61,12 @@ Examples: for _, fp := range paths { if isYaml(fp) { - fmt.Printf("Registering %s at %s\n", resource, fp) - regID, err := register(ctx, coreCli, resource, fp) + fmt.Printf("Applying %s at %s\n", resource, fp) + regID, err := apply(ctx, coreCli, resource, fp) if err != nil { - return fmt.Errorf("failed to register %s at path %s: %v", resource, fp, err) + return fmt.Errorf("failed to apply %s at path %s: %v", resource, fp, err) } - fmt.Printf("Successfully registered %s %s\n", resource, regID) + fmt.Printf("Successfully applied %s %s\n", resource, regID) } } return nil @@ -75,10 +74,10 @@ Examples: } func init() { - rootCmd.AddCommand(registerCmd) + rootCmd.AddCommand(applyCmd) } -func register(ctx context.Context, coreCli core.CoreServiceClient, resource string, fileLocation string) (string, error) { +func apply(ctx context.Context, coreCli core.CoreServiceClient, resource string, fileLocation string) (string, error) { yml, err := ioutil.ReadFile(fileLocation) if err != nil { return "", fmt.Errorf("error reading file at %s: %v", fileLocation, err) @@ -86,51 +85,51 @@ func register(ctx context.Context, coreCli core.CoreServiceClient, resource stri switch resource { case "feature": - return registerFeature(ctx, coreCli, yml) + return applyFeature(ctx, coreCli, yml) case "featureGroup": - return registerFeatureGroup(ctx, coreCli, yml) + return applyFeatureGroup(ctx, coreCli, yml) case "entity": - return registerEntity(ctx, coreCli, yml) + return applyEntity(ctx, coreCli, yml) case "storage": - return registerStorage(ctx, coreCli, yml) + return applyStorage(ctx, coreCli, yml) default: return "", fmt.Errorf("invalid resource %s: please choose one of [feature, featureGroup, entity, storage]", resource) } } -func registerFeature(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { +func applyFeature(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { fs, err := parse.YamlToFeatureSpec(yml) if err != nil { return "", err } - _, err = coreCli.RegisterFeature(ctx, fs) + _, err = coreCli.ApplyFeature(ctx, fs) return fs.GetId(), err } -func registerFeatureGroup(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { +func applyFeatureGroup(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { fgs, err := parse.YamlToFeatureGroupSpec(yml) if err != nil { return "", err } - _, err = coreCli.RegisterFeatureGroup(ctx, fgs) + _, err = coreCli.ApplyFeatureGroup(ctx, fgs) return fgs.GetId(), err } -func registerEntity(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { +func applyEntity(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { es, err := parse.YamlToEntitySpec(yml) if err != nil { return "", err } - _, err = coreCli.RegisterEntity(ctx, es) + _, err = coreCli.ApplyEntity(ctx, es) return es.GetName(), err } -func registerStorage(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { +func applyStorage(ctx context.Context, coreCli core.CoreServiceClient, yml []byte) (string, error) { ss, err := parse.YamlToStorageSpec(yml) if err != nil { return "", err } - _, err = coreCli.RegisterStorage(ctx, ss) + _, err = coreCli.ApplyStorage(ctx, ss) return ss.GetId(), err } diff --git a/cli/feast/cmd/version.go b/cli/feast/cmd/version.go index 1349ad84238..ca01d1dc59e 100644 --- a/cli/feast/cmd/version.go +++ b/cli/feast/cmd/version.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/cobra" ) -var version = "0.2.0" +var version = "0.3.0" var versionCmd = &cobra.Command{ Use: "version", diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index a87397b0239..a48222a536a 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -233,13 +233,13 @@ public void listStorage(Empty request, StreamObserver respo * error will be returned. */ @Override - public void registerFeature( - FeatureSpec request, StreamObserver responseObserver) { + public void applyFeature( + FeatureSpec request, StreamObserver responseObserver) { try { validator.validateFeatureSpec(request); FeatureInfo feature = specService.applyFeature(request); - RegisterFeatureResponse response = - RegisterFeatureResponse.newBuilder().setFeatureId(feature.getId()).build(); + ApplyFeatureResponse response = + ApplyFeatureResponse.newBuilder().setFeatureId(feature.getId()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { @@ -257,14 +257,14 @@ public void registerFeature( * error will be returned. */ @Override - public void registerFeatureGroup( + public void applyFeatureGroup( FeatureGroupSpecProto.FeatureGroupSpec request, - StreamObserver responseObserver) { + StreamObserver responseObserver) { try { validator.validateFeatureGroupSpec(request); FeatureGroupInfo featureGroup = specService.applyFeatureGroup(request); - RegisterFeatureGroupResponse response = - RegisterFeatureGroupResponse.newBuilder().setFeatureGroupId(featureGroup.getId()).build(); + ApplyFeatureGroupResponse response = + ApplyFeatureGroupResponse.newBuilder().setFeatureGroupId(featureGroup.getId()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { @@ -282,13 +282,13 @@ public void registerFeatureGroup( * be returned. */ @Override - public void registerEntity( - EntitySpec request, StreamObserver responseObserver) { + public void applyEntity( + EntitySpec request, StreamObserver responseObserver) { try { validator.validateEntitySpec(request); EntityInfo entity = specService.applyEntity(request); - RegisterEntityResponse response = - RegisterEntityResponse.newBuilder().setEntityName(entity.getName()).build(); + ApplyEntityResponse response = + ApplyEntityResponse.newBuilder().setEntityName(entity.getName()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { @@ -306,13 +306,13 @@ public void registerEntity( * be returned. */ @Override - public void registerStorage( - StorageSpec request, StreamObserver responseObserver) { + public void applyStorage( + StorageSpec request, StreamObserver responseObserver) { try { validator.validateStorageSpec(request); StorageInfo storage = specService.registerStorage(request); - RegisterStorageResponse response = - RegisterStorageResponse.newBuilder().setStorageId(storage.getId()).build(); + ApplyStorageResponse response = + ApplyStorageResponse.newBuilder().setStorageId(storage.getId()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); } catch (RegistrationException e) { diff --git a/go-feast-proto/feast/core/CoreService.pb.go b/go-feast-proto/feast/core/CoreService.pb.go index afc42bafd53..f5c35a8c1bb 100644 --- a/go-feast-proto/feast/core/CoreService.pb.go +++ b/go-feast-proto/feast/core/CoreService.pb.go @@ -35,7 +35,7 @@ func (m *CoreServiceTypes) Reset() { *m = CoreServiceTypes{} } func (m *CoreServiceTypes) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes) ProtoMessage() {} func (*CoreServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0} } func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes.Unmarshal(m, b) @@ -66,7 +66,7 @@ func (m *CoreServiceTypes_GetEntitiesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetEntitiesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 0} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 0} } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Unmarshal(m, b) @@ -104,7 +104,7 @@ func (m *CoreServiceTypes_GetEntitiesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 1} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 1} } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Unmarshal(m, b) @@ -142,7 +142,7 @@ func (m *CoreServiceTypes_ListEntitiesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 2} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 2} } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -181,7 +181,7 @@ func (m *CoreServiceTypes_GetFeaturesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 3} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 3} } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Unmarshal(m, b) @@ -219,7 +219,7 @@ func (m *CoreServiceTypes_GetFeaturesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 4} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 4} } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Unmarshal(m, b) @@ -257,7 +257,7 @@ func (m *CoreServiceTypes_ListFeaturesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 5} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 5} } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -296,7 +296,7 @@ func (m *CoreServiceTypes_GetStorageRequest) Reset() { *m = CoreServiceT func (m *CoreServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageRequest) ProtoMessage() {} func (*CoreServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 6} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 6} } func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -334,7 +334,7 @@ func (m *CoreServiceTypes_GetStorageResponse) Reset() { *m = CoreService func (m *CoreServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 7} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 7} } func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -372,7 +372,7 @@ func (m *CoreServiceTypes_ListStorageResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 8} + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 8} } func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -400,40 +400,38 @@ func (m *CoreServiceTypes_ListStorageResponse) GetStorageSpecs() []*specs.Storag } // Entity registration response -type CoreServiceTypes_RegisterEntityResponse struct { +type CoreServiceTypes_ApplyEntityResponse struct { EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *CoreServiceTypes_RegisterEntityResponse) Reset() { - *m = CoreServiceTypes_RegisterEntityResponse{} +func (m *CoreServiceTypes_ApplyEntityResponse) Reset() { *m = CoreServiceTypes_ApplyEntityResponse{} } +func (m *CoreServiceTypes_ApplyEntityResponse) String() string { return proto.CompactTextString(m) } +func (*CoreServiceTypes_ApplyEntityResponse) ProtoMessage() {} +func (*CoreServiceTypes_ApplyEntityResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 9} } -func (m *CoreServiceTypes_RegisterEntityResponse) String() string { return proto.CompactTextString(m) } -func (*CoreServiceTypes_RegisterEntityResponse) ProtoMessage() {} -func (*CoreServiceTypes_RegisterEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 9} +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Unmarshal(m, b) } -func (m *CoreServiceTypes_RegisterEntityResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CoreServiceTypes_RegisterEntityResponse.Unmarshal(m, b) +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Marshal(b, m, deterministic) } -func (m *CoreServiceTypes_RegisterEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CoreServiceTypes_RegisterEntityResponse.Marshal(b, m, deterministic) +func (dst *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(dst, src) } -func (dst *CoreServiceTypes_RegisterEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_RegisterEntityResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Size() int { + return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Size(m) } -func (m *CoreServiceTypes_RegisterEntityResponse) XXX_Size() int { - return xxx_messageInfo_CoreServiceTypes_RegisterEntityResponse.Size(m) -} -func (m *CoreServiceTypes_RegisterEntityResponse) XXX_DiscardUnknown() { - xxx_messageInfo_CoreServiceTypes_RegisterEntityResponse.DiscardUnknown(m) +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.DiscardUnknown(m) } -var xxx_messageInfo_CoreServiceTypes_RegisterEntityResponse proto.InternalMessageInfo +var xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse proto.InternalMessageInfo -func (m *CoreServiceTypes_RegisterEntityResponse) GetEntityName() string { +func (m *CoreServiceTypes_ApplyEntityResponse) GetEntityName() string { if m != nil { return m.EntityName } @@ -441,40 +439,38 @@ func (m *CoreServiceTypes_RegisterEntityResponse) GetEntityName() string { } // Feature registration response -type CoreServiceTypes_RegisterFeatureResponse struct { +type CoreServiceTypes_ApplyFeatureResponse struct { FeatureId string `protobuf:"bytes,1,opt,name=featureId,proto3" json:"featureId,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *CoreServiceTypes_RegisterFeatureResponse) Reset() { - *m = CoreServiceTypes_RegisterFeatureResponse{} -} -func (m *CoreServiceTypes_RegisterFeatureResponse) String() string { return proto.CompactTextString(m) } -func (*CoreServiceTypes_RegisterFeatureResponse) ProtoMessage() {} -func (*CoreServiceTypes_RegisterFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 10} +func (m *CoreServiceTypes_ApplyFeatureResponse) Reset() { *m = CoreServiceTypes_ApplyFeatureResponse{} } +func (m *CoreServiceTypes_ApplyFeatureResponse) String() string { return proto.CompactTextString(m) } +func (*CoreServiceTypes_ApplyFeatureResponse) ProtoMessage() {} +func (*CoreServiceTypes_ApplyFeatureResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 10} } -func (m *CoreServiceTypes_RegisterFeatureResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CoreServiceTypes_RegisterFeatureResponse.Unmarshal(m, b) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Unmarshal(m, b) } -func (m *CoreServiceTypes_RegisterFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CoreServiceTypes_RegisterFeatureResponse.Marshal(b, m, deterministic) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_RegisterFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_RegisterFeatureResponse.Merge(dst, src) +func (dst *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(dst, src) } -func (m *CoreServiceTypes_RegisterFeatureResponse) XXX_Size() int { - return xxx_messageInfo_CoreServiceTypes_RegisterFeatureResponse.Size(m) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Size() int { + return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Size(m) } -func (m *CoreServiceTypes_RegisterFeatureResponse) XXX_DiscardUnknown() { - xxx_messageInfo_CoreServiceTypes_RegisterFeatureResponse.DiscardUnknown(m) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.DiscardUnknown(m) } -var xxx_messageInfo_CoreServiceTypes_RegisterFeatureResponse proto.InternalMessageInfo +var xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse proto.InternalMessageInfo -func (m *CoreServiceTypes_RegisterFeatureResponse) GetFeatureId() string { +func (m *CoreServiceTypes_ApplyFeatureResponse) GetFeatureId() string { if m != nil { return m.FeatureId } @@ -482,42 +478,42 @@ func (m *CoreServiceTypes_RegisterFeatureResponse) GetFeatureId() string { } // Feature group registration response -type CoreServiceTypes_RegisterFeatureGroupResponse struct { +type CoreServiceTypes_ApplyFeatureGroupResponse struct { FeatureGroupId string `protobuf:"bytes,1,opt,name=featureGroupId,proto3" json:"featureGroupId,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) Reset() { - *m = CoreServiceTypes_RegisterFeatureGroupResponse{} +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) Reset() { + *m = CoreServiceTypes_ApplyFeatureGroupResponse{} } -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) String() string { +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) String() string { return proto.CompactTextString(m) } -func (*CoreServiceTypes_RegisterFeatureGroupResponse) ProtoMessage() {} -func (*CoreServiceTypes_RegisterFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 11} +func (*CoreServiceTypes_ApplyFeatureGroupResponse) ProtoMessage() {} +func (*CoreServiceTypes_ApplyFeatureGroupResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 11} } -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CoreServiceTypes_RegisterFeatureGroupResponse.Unmarshal(m, b) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Unmarshal(m, b) } -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CoreServiceTypes_RegisterFeatureGroupResponse.Marshal(b, m, deterministic) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_RegisterFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_RegisterFeatureGroupResponse.Merge(dst, src) +func (dst *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(dst, src) } -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) XXX_Size() int { - return xxx_messageInfo_CoreServiceTypes_RegisterFeatureGroupResponse.Size(m) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Size() int { + return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Size(m) } -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) XXX_DiscardUnknown() { - xxx_messageInfo_CoreServiceTypes_RegisterFeatureGroupResponse.DiscardUnknown(m) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.DiscardUnknown(m) } -var xxx_messageInfo_CoreServiceTypes_RegisterFeatureGroupResponse proto.InternalMessageInfo +var xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse proto.InternalMessageInfo -func (m *CoreServiceTypes_RegisterFeatureGroupResponse) GetFeatureGroupId() string { +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) GetFeatureGroupId() string { if m != nil { return m.FeatureGroupId } @@ -525,40 +521,38 @@ func (m *CoreServiceTypes_RegisterFeatureGroupResponse) GetFeatureGroupId() stri } // Storage registration response -type CoreServiceTypes_RegisterStorageResponse struct { +type CoreServiceTypes_ApplyStorageResponse struct { StorageId string `protobuf:"bytes,1,opt,name=storageId,proto3" json:"storageId,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *CoreServiceTypes_RegisterStorageResponse) Reset() { - *m = CoreServiceTypes_RegisterStorageResponse{} -} -func (m *CoreServiceTypes_RegisterStorageResponse) String() string { return proto.CompactTextString(m) } -func (*CoreServiceTypes_RegisterStorageResponse) ProtoMessage() {} -func (*CoreServiceTypes_RegisterStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_aeabad36f2a7449f, []int{0, 12} +func (m *CoreServiceTypes_ApplyStorageResponse) Reset() { *m = CoreServiceTypes_ApplyStorageResponse{} } +func (m *CoreServiceTypes_ApplyStorageResponse) String() string { return proto.CompactTextString(m) } +func (*CoreServiceTypes_ApplyStorageResponse) ProtoMessage() {} +func (*CoreServiceTypes_ApplyStorageResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_CoreService_ab3e56c9944743b5, []int{0, 12} } -func (m *CoreServiceTypes_RegisterStorageResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CoreServiceTypes_RegisterStorageResponse.Unmarshal(m, b) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Unmarshal(m, b) } -func (m *CoreServiceTypes_RegisterStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CoreServiceTypes_RegisterStorageResponse.Marshal(b, m, deterministic) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_RegisterStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_RegisterStorageResponse.Merge(dst, src) +func (dst *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(dst, src) } -func (m *CoreServiceTypes_RegisterStorageResponse) XXX_Size() int { - return xxx_messageInfo_CoreServiceTypes_RegisterStorageResponse.Size(m) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Size() int { + return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Size(m) } -func (m *CoreServiceTypes_RegisterStorageResponse) XXX_DiscardUnknown() { - xxx_messageInfo_CoreServiceTypes_RegisterStorageResponse.DiscardUnknown(m) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.DiscardUnknown(m) } -var xxx_messageInfo_CoreServiceTypes_RegisterStorageResponse proto.InternalMessageInfo +var xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse proto.InternalMessageInfo -func (m *CoreServiceTypes_RegisterStorageResponse) GetStorageId() string { +func (m *CoreServiceTypes_ApplyStorageResponse) GetStorageId() string { if m != nil { return m.StorageId } @@ -576,10 +570,10 @@ func init() { proto.RegisterType((*CoreServiceTypes_GetStorageRequest)(nil), "feast.core.CoreServiceTypes.GetStorageRequest") proto.RegisterType((*CoreServiceTypes_GetStorageResponse)(nil), "feast.core.CoreServiceTypes.GetStorageResponse") proto.RegisterType((*CoreServiceTypes_ListStorageResponse)(nil), "feast.core.CoreServiceTypes.ListStorageResponse") - proto.RegisterType((*CoreServiceTypes_RegisterEntityResponse)(nil), "feast.core.CoreServiceTypes.RegisterEntityResponse") - proto.RegisterType((*CoreServiceTypes_RegisterFeatureResponse)(nil), "feast.core.CoreServiceTypes.RegisterFeatureResponse") - proto.RegisterType((*CoreServiceTypes_RegisterFeatureGroupResponse)(nil), "feast.core.CoreServiceTypes.RegisterFeatureGroupResponse") - proto.RegisterType((*CoreServiceTypes_RegisterStorageResponse)(nil), "feast.core.CoreServiceTypes.RegisterStorageResponse") + proto.RegisterType((*CoreServiceTypes_ApplyEntityResponse)(nil), "feast.core.CoreServiceTypes.ApplyEntityResponse") + proto.RegisterType((*CoreServiceTypes_ApplyFeatureResponse)(nil), "feast.core.CoreServiceTypes.ApplyFeatureResponse") + proto.RegisterType((*CoreServiceTypes_ApplyFeatureGroupResponse)(nil), "feast.core.CoreServiceTypes.ApplyFeatureGroupResponse") + proto.RegisterType((*CoreServiceTypes_ApplyStorageResponse)(nil), "feast.core.CoreServiceTypes.ApplyStorageResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -619,21 +613,21 @@ type CoreServiceClient interface { // This process returns a list of storage specs. ListStorage(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*CoreServiceTypes_ListStorageResponse, error) // - // Register a new feature to the metadata store. + // Register a new feature to the metadata store, or update an existing feature. // If any validation errors occur, only the first encountered error will be returned. - RegisterFeature(ctx context.Context, in *specs.FeatureSpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterFeatureResponse, error) + ApplyFeature(ctx context.Context, in *specs.FeatureSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyFeatureResponse, error) // - // Register a new feature group to the metadata store. + // Register a new feature group to the metadata store, or update an existing feature group. // If any validation errors occur, only the first encountered error will be returned. - RegisterFeatureGroup(ctx context.Context, in *specs.FeatureGroupSpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterFeatureGroupResponse, error) + ApplyFeatureGroup(ctx context.Context, in *specs.FeatureGroupSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyFeatureGroupResponse, error) // - // Register a new entity to the metadata store. + // Register a new entity to the metadata store, or update an existing entity. // If any validation errors occur, only the first encountered error will be returned. - RegisterEntity(ctx context.Context, in *specs.EntitySpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterEntityResponse, error) + ApplyEntity(ctx context.Context, in *specs.EntitySpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyEntityResponse, error) // - // Register a new storage spec to the metadata store. + // Register a new storage spec to the metadata store, or update an existing storage. // If any validation errors occur, only the first encountered error will be returned. - RegisterStorage(ctx context.Context, in *specs.StorageSpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterStorageResponse, error) + ApplyStorage(ctx context.Context, in *specs.StorageSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyStorageResponse, error) } type coreServiceClient struct { @@ -698,36 +692,36 @@ func (c *coreServiceClient) ListStorage(ctx context.Context, in *empty.Empty, op return out, nil } -func (c *coreServiceClient) RegisterFeature(ctx context.Context, in *specs.FeatureSpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterFeatureResponse, error) { - out := new(CoreServiceTypes_RegisterFeatureResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/RegisterFeature", in, out, opts...) +func (c *coreServiceClient) ApplyFeature(ctx context.Context, in *specs.FeatureSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyFeatureResponse, error) { + out := new(CoreServiceTypes_ApplyFeatureResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyFeature", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *coreServiceClient) RegisterFeatureGroup(ctx context.Context, in *specs.FeatureGroupSpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterFeatureGroupResponse, error) { - out := new(CoreServiceTypes_RegisterFeatureGroupResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/RegisterFeatureGroup", in, out, opts...) +func (c *coreServiceClient) ApplyFeatureGroup(ctx context.Context, in *specs.FeatureGroupSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyFeatureGroupResponse, error) { + out := new(CoreServiceTypes_ApplyFeatureGroupResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyFeatureGroup", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *coreServiceClient) RegisterEntity(ctx context.Context, in *specs.EntitySpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterEntityResponse, error) { - out := new(CoreServiceTypes_RegisterEntityResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/RegisterEntity", in, out, opts...) +func (c *coreServiceClient) ApplyEntity(ctx context.Context, in *specs.EntitySpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyEntityResponse, error) { + out := new(CoreServiceTypes_ApplyEntityResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyEntity", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *coreServiceClient) RegisterStorage(ctx context.Context, in *specs.StorageSpec, opts ...grpc.CallOption) (*CoreServiceTypes_RegisterStorageResponse, error) { - out := new(CoreServiceTypes_RegisterStorageResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/RegisterStorage", in, out, opts...) +func (c *coreServiceClient) ApplyStorage(ctx context.Context, in *specs.StorageSpec, opts ...grpc.CallOption) (*CoreServiceTypes_ApplyStorageResponse, error) { + out := new(CoreServiceTypes_ApplyStorageResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyStorage", in, out, opts...) if err != nil { return nil, err } @@ -761,21 +755,21 @@ type CoreServiceServer interface { // This process returns a list of storage specs. ListStorage(context.Context, *empty.Empty) (*CoreServiceTypes_ListStorageResponse, error) // - // Register a new feature to the metadata store. + // Register a new feature to the metadata store, or update an existing feature. // If any validation errors occur, only the first encountered error will be returned. - RegisterFeature(context.Context, *specs.FeatureSpec) (*CoreServiceTypes_RegisterFeatureResponse, error) + ApplyFeature(context.Context, *specs.FeatureSpec) (*CoreServiceTypes_ApplyFeatureResponse, error) // - // Register a new feature group to the metadata store. + // Register a new feature group to the metadata store, or update an existing feature group. // If any validation errors occur, only the first encountered error will be returned. - RegisterFeatureGroup(context.Context, *specs.FeatureGroupSpec) (*CoreServiceTypes_RegisterFeatureGroupResponse, error) + ApplyFeatureGroup(context.Context, *specs.FeatureGroupSpec) (*CoreServiceTypes_ApplyFeatureGroupResponse, error) // - // Register a new entity to the metadata store. + // Register a new entity to the metadata store, or update an existing entity. // If any validation errors occur, only the first encountered error will be returned. - RegisterEntity(context.Context, *specs.EntitySpec) (*CoreServiceTypes_RegisterEntityResponse, error) + ApplyEntity(context.Context, *specs.EntitySpec) (*CoreServiceTypes_ApplyEntityResponse, error) // - // Register a new storage spec to the metadata store. + // Register a new storage spec to the metadata store, or update an existing storage. // If any validation errors occur, only the first encountered error will be returned. - RegisterStorage(context.Context, *specs.StorageSpec) (*CoreServiceTypes_RegisterStorageResponse, error) + ApplyStorage(context.Context, *specs.StorageSpec) (*CoreServiceTypes_ApplyStorageResponse, error) } func RegisterCoreServiceServer(s *grpc.Server, srv CoreServiceServer) { @@ -890,74 +884,74 @@ func _CoreService_ListStorage_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _CoreService_RegisterFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _CoreService_ApplyFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(specs.FeatureSpec) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(CoreServiceServer).RegisterFeature(ctx, in) + return srv.(CoreServiceServer).ApplyFeature(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/feast.core.CoreService/RegisterFeature", + FullMethod: "/feast.core.CoreService/ApplyFeature", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).RegisterFeature(ctx, req.(*specs.FeatureSpec)) + return srv.(CoreServiceServer).ApplyFeature(ctx, req.(*specs.FeatureSpec)) } return interceptor(ctx, in, info, handler) } -func _CoreService_RegisterFeatureGroup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _CoreService_ApplyFeatureGroup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(specs.FeatureGroupSpec) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(CoreServiceServer).RegisterFeatureGroup(ctx, in) + return srv.(CoreServiceServer).ApplyFeatureGroup(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/feast.core.CoreService/RegisterFeatureGroup", + FullMethod: "/feast.core.CoreService/ApplyFeatureGroup", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).RegisterFeatureGroup(ctx, req.(*specs.FeatureGroupSpec)) + return srv.(CoreServiceServer).ApplyFeatureGroup(ctx, req.(*specs.FeatureGroupSpec)) } return interceptor(ctx, in, info, handler) } -func _CoreService_RegisterEntity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _CoreService_ApplyEntity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(specs.EntitySpec) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(CoreServiceServer).RegisterEntity(ctx, in) + return srv.(CoreServiceServer).ApplyEntity(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/feast.core.CoreService/RegisterEntity", + FullMethod: "/feast.core.CoreService/ApplyEntity", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).RegisterEntity(ctx, req.(*specs.EntitySpec)) + return srv.(CoreServiceServer).ApplyEntity(ctx, req.(*specs.EntitySpec)) } return interceptor(ctx, in, info, handler) } -func _CoreService_RegisterStorage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _CoreService_ApplyStorage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(specs.StorageSpec) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(CoreServiceServer).RegisterStorage(ctx, in) + return srv.(CoreServiceServer).ApplyStorage(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/feast.core.CoreService/RegisterStorage", + FullMethod: "/feast.core.CoreService/ApplyStorage", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).RegisterStorage(ctx, req.(*specs.StorageSpec)) + return srv.(CoreServiceServer).ApplyStorage(ctx, req.(*specs.StorageSpec)) } return interceptor(ctx, in, info, handler) } @@ -991,20 +985,20 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ Handler: _CoreService_ListStorage_Handler, }, { - MethodName: "RegisterFeature", - Handler: _CoreService_RegisterFeature_Handler, + MethodName: "ApplyFeature", + Handler: _CoreService_ApplyFeature_Handler, }, { - MethodName: "RegisterFeatureGroup", - Handler: _CoreService_RegisterFeatureGroup_Handler, + MethodName: "ApplyFeatureGroup", + Handler: _CoreService_ApplyFeatureGroup_Handler, }, { - MethodName: "RegisterEntity", - Handler: _CoreService_RegisterEntity_Handler, + MethodName: "ApplyEntity", + Handler: _CoreService_ApplyEntity_Handler, }, { - MethodName: "RegisterStorage", - Handler: _CoreService_RegisterStorage_Handler, + MethodName: "ApplyStorage", + Handler: _CoreService_ApplyStorage_Handler, }, }, Streams: []grpc.StreamDesc{}, @@ -1012,47 +1006,47 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_aeabad36f2a7449f) -} - -var fileDescriptor_CoreService_aeabad36f2a7449f = []byte{ - // 605 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5f, 0x4f, 0x13, 0x41, - 0x10, 0x6f, 0x43, 0x42, 0xda, 0x29, 0x41, 0x5c, 0x08, 0x34, 0x6b, 0x31, 0xa4, 0x89, 0x84, 0x17, - 0xf6, 0x90, 0x36, 0x51, 0x13, 0x9f, 0x30, 0xd0, 0x28, 0xc4, 0x98, 0x2b, 0x2f, 0xea, 0x8b, 0xed, - 0x31, 0x3d, 0x4e, 0x2d, 0x7b, 0xde, 0x6e, 0x4d, 0xfa, 0x59, 0xfc, 0x66, 0x7e, 0x1a, 0x73, 0x7b, - 0xdb, 0xbd, 0xbd, 0xeb, 0x5d, 0x5b, 0x49, 0xdf, 0xda, 0xf9, 0xff, 0x9b, 0x99, 0xdf, 0xec, 0x41, - 0x6b, 0x84, 0x03, 0x21, 0x1d, 0x8f, 0x47, 0xe8, 0xbc, 0xe3, 0x11, 0xf6, 0x31, 0xfa, 0x1d, 0x78, - 0xc8, 0xc2, 0x88, 0x4b, 0x4e, 0x40, 0x69, 0x59, 0xac, 0xa5, 0xda, 0x52, 0x84, 0xe8, 0x09, 0xe7, - 0xf2, 0x41, 0x06, 0x72, 0xda, 0x0f, 0xd1, 0x4b, 0x2c, 0xe9, 0xa1, 0xad, 0xbd, 0xc2, 0x81, 0x9c, - 0x44, 0x68, 0xa9, 0xdb, 0x05, 0xea, 0x5e, 0xc4, 0x27, 0x61, 0x59, 0x88, 0xbe, 0xe4, 0xd1, 0xc0, - 0xb7, 0x43, 0x3c, 0xf3, 0x39, 0xf7, 0x7f, 0xa2, 0xa3, 0xfe, 0x0d, 0x27, 0x23, 0x07, 0xc7, 0xa1, - 0x9c, 0x26, 0xca, 0xf6, 0xdf, 0x4d, 0xd8, 0xb1, 0xca, 0xbf, 0x9d, 0x86, 0x28, 0xe8, 0x31, 0x90, - 0x1e, 0x4a, 0x55, 0x6a, 0x80, 0xc2, 0xc5, 0x5f, 0x13, 0x14, 0x92, 0xec, 0xc0, 0x46, 0x70, 0x27, - 0x9a, 0xd5, 0xa3, 0x8d, 0x93, 0xba, 0x1b, 0xff, 0xa4, 0x1f, 0x60, 0x37, 0x63, 0x27, 0x42, 0xfe, - 0x20, 0x90, 0x74, 0xa0, 0x86, 0x5a, 0xa6, 0xac, 0x1b, 0xe7, 0x07, 0x2c, 0xe9, 0x87, 0x2a, 0x91, - 0xa5, 0x3d, 0x70, 0x8d, 0x21, 0xbd, 0x86, 0xbd, 0x9b, 0x40, 0xac, 0x29, 0x58, 0x02, 0x40, 0xb7, - 0x6b, 0x01, 0x80, 0x6b, 0x05, 0x20, 0xb5, 0xd3, 0x39, 0xbb, 0x50, 0x1b, 0x69, 0x99, 0xce, 0xd9, - 0xcc, 0xe4, 0xb4, 0xc6, 0xe4, 0x1a, 0x4b, 0x7a, 0x93, 0x20, 0x58, 0x53, 0xb4, 0x17, 0xf0, 0xb4, - 0x87, 0x52, 0x4f, 0xb3, 0x1c, 0x81, 0xab, 0x90, 0x1a, 0x33, 0x9d, 0xf2, 0x2d, 0x6c, 0x89, 0x74, - 0x0f, 0x8a, 0xd3, 0x5a, 0x8b, 0xe2, 0x66, 0xac, 0x69, 0x1f, 0x76, 0x63, 0x20, 0xeb, 0x0d, 0xfa, - 0x1a, 0xf6, 0x5d, 0xf4, 0x03, 0x21, 0x31, 0x4a, 0x46, 0x66, 0xe2, 0x3e, 0x07, 0x50, 0x83, 0x9b, - 0x7e, 0x1c, 0x8c, 0xb1, 0x59, 0x3d, 0xaa, 0x9e, 0xd4, 0x5d, 0x4b, 0x42, 0x5f, 0xc1, 0xc1, 0xcc, - 0x53, 0xb7, 0xca, 0xb8, 0xb6, 0xa0, 0xae, 0x1b, 0xf6, 0xfe, 0x4e, 0x7b, 0xa6, 0x02, 0x7a, 0x05, - 0xad, 0x9c, 0xa3, 0x62, 0x8e, 0xf1, 0x3e, 0x86, 0xed, 0x91, 0x25, 0x37, 0x21, 0x72, 0x52, 0xbb, - 0x80, 0x7c, 0x4f, 0x5a, 0x50, 0xd7, 0x28, 0xd3, 0x02, 0x8c, 0xe0, 0xfc, 0x4f, 0x0d, 0x1a, 0x16, - 0xb9, 0x48, 0x04, 0x0d, 0x8b, 0x2f, 0xc4, 0x61, 0xe9, 0x95, 0x60, 0x79, 0x12, 0xb2, 0x79, 0x06, - 0xd2, 0xb3, 0xd5, 0x1d, 0x92, 0xfa, 0xda, 0x15, 0xf2, 0x15, 0xb6, 0x6c, 0x5e, 0x91, 0x7d, 0x96, - 0x9c, 0x03, 0x36, 0x3b, 0x07, 0xec, 0x32, 0x3e, 0x07, 0xf4, 0xe5, 0xc2, 0xd8, 0x45, 0xd4, 0x6c, - 0x57, 0x34, 0xa0, 0xd9, 0xc6, 0x2f, 0x07, 0x94, 0x63, 0xe4, 0x72, 0x40, 0x79, 0x32, 0xa5, 0x80, - 0x4c, 0xd2, 0xc7, 0x03, 0x2a, 0x08, 0xce, 0x01, 0x52, 0x3a, 0x11, 0xb6, 0xac, 0xbc, 0x2c, 0x3d, - 0xa9, 0xb3, 0xb2, 0xbd, 0x49, 0xf8, 0x19, 0x1a, 0x16, 0xd7, 0x4a, 0xc1, 0x9c, 0x2d, 0x05, 0x33, - 0x1f, 0xda, 0x83, 0x27, 0xb9, 0xf5, 0x27, 0xa5, 0x87, 0x87, 0x76, 0x17, 0x26, 0x28, 0xe1, 0x9f, - 0xda, 0x80, 0xbd, 0x22, 0x8e, 0x91, 0xc3, 0xa2, 0x4c, 0xe6, 0xe1, 0xa2, 0x6f, 0xfe, 0x27, 0x5d, - 0x86, 0xb5, 0xed, 0x0a, 0xf9, 0x06, 0xdb, 0xd9, 0x53, 0x42, 0xca, 0x9e, 0x04, 0xda, 0x59, 0x29, - 0x4f, 0xf6, 0x20, 0x65, 0x5b, 0x37, 0x9b, 0x4c, 0xe9, 0x9d, 0x5b, 0xb1, 0x75, 0x73, 0xf3, 0xb9, - 0xb8, 0x05, 0xeb, 0x2b, 0xe1, 0xc2, 0x7e, 0x85, 0x3f, 0xc5, 0xc3, 0xff, 0xd2, 0xf5, 0x03, 0x79, - 0x3f, 0x19, 0x32, 0x8f, 0x8f, 0x1d, 0x9f, 0x7f, 0xc7, 0x1f, 0x12, 0xbd, 0x7b, 0x27, 0x79, 0xeb, - 0x7d, 0x7e, 0xaa, 0x7e, 0x9c, 0xaa, 0x3d, 0x71, 0xd2, 0x6f, 0x91, 0xe1, 0xa6, 0x92, 0x74, 0xfe, - 0x05, 0x00, 0x00, 0xff, 0xff, 0x78, 0x92, 0x55, 0xe6, 0xa0, 0x08, 0x00, 0x00, + proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_ab3e56c9944743b5) +} + +var fileDescriptor_CoreService_ab3e56c9944743b5 = []byte{ + // 601 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcf, 0x6f, 0xd3, 0x30, + 0x14, 0x6e, 0x35, 0x69, 0x6a, 0x5f, 0x2b, 0xb4, 0xb9, 0x13, 0x14, 0xd3, 0xa1, 0x29, 0x12, 0xd3, + 0x2e, 0x73, 0xc6, 0x56, 0x38, 0x71, 0x61, 0xd3, 0xa8, 0x60, 0x13, 0x42, 0xe9, 0x2e, 0x0c, 0x71, + 0x68, 0xb3, 0xd7, 0x2c, 0xd0, 0xce, 0x21, 0x76, 0x91, 0x7a, 0xe6, 0xff, 0xe3, 0x6f, 0x42, 0x75, + 0x5c, 0xc7, 0x4d, 0x93, 0xa5, 0x42, 0xbd, 0xb5, 0xef, 0xd7, 0xe7, 0xef, 0x3d, 0x7f, 0xcf, 0x81, + 0xce, 0x08, 0x07, 0x42, 0xba, 0x3e, 0x8f, 0xd1, 0xbd, 0xe0, 0x31, 0xf6, 0x31, 0xfe, 0x1d, 0xfa, + 0xc8, 0xa2, 0x98, 0x4b, 0x4e, 0x40, 0x79, 0xd9, 0xdc, 0x4b, 0x75, 0xa4, 0x88, 0xd0, 0x17, 0xee, + 0xe5, 0x83, 0x0c, 0xe5, 0xac, 0x1f, 0xa1, 0x9f, 0x44, 0xd2, 0x7d, 0xdb, 0xfb, 0x01, 0x07, 0x72, + 0x1a, 0xa3, 0xe5, 0x76, 0x72, 0xdc, 0xbd, 0x98, 0x4f, 0xa3, 0xa2, 0x12, 0x7d, 0xc9, 0xe3, 0x41, + 0x60, 0x97, 0x78, 0x11, 0x70, 0x1e, 0x8c, 0xd1, 0x55, 0xff, 0x86, 0xd3, 0x91, 0x8b, 0x93, 0x48, + 0xce, 0x12, 0xa7, 0xf3, 0x77, 0x1b, 0x76, 0xac, 0xe3, 0xdf, 0xcc, 0x22, 0x14, 0xf4, 0x10, 0x48, + 0x0f, 0xa5, 0x3a, 0x6a, 0x88, 0xc2, 0xc3, 0x5f, 0x53, 0x14, 0x92, 0xec, 0xc0, 0x56, 0x78, 0x27, + 0xda, 0xd5, 0x83, 0xad, 0xa3, 0xba, 0x37, 0xff, 0x49, 0x3f, 0x41, 0x6b, 0x29, 0x4e, 0x44, 0xfc, + 0x41, 0x20, 0x39, 0x83, 0x1a, 0x6a, 0x9b, 0x8a, 0x6e, 0x9c, 0x3e, 0x63, 0x49, 0x3f, 0xd4, 0x11, + 0x59, 0xda, 0x03, 0xcf, 0x04, 0xd2, 0x2b, 0xd8, 0xbb, 0x0e, 0xc5, 0x86, 0x8a, 0x25, 0x04, 0x74, + 0xbb, 0x1e, 0x21, 0x70, 0xa5, 0x08, 0xa4, 0x71, 0x1a, 0xb3, 0x0b, 0xb5, 0x91, 0xb6, 0x69, 0xcc, + 0xf6, 0x12, 0xa6, 0x35, 0x26, 0xcf, 0x44, 0xd2, 0xeb, 0x84, 0xc1, 0x86, 0xaa, 0xbd, 0x82, 0xdd, + 0x1e, 0x4a, 0x3d, 0xcd, 0x62, 0x06, 0x9e, 0x62, 0x6a, 0xc2, 0x34, 0xe4, 0x3b, 0x68, 0x8a, 0xf4, + 0x1e, 0xe4, 0xc3, 0x5a, 0x17, 0xc5, 0x5b, 0x8a, 0xa6, 0x7d, 0x68, 0xcd, 0x89, 0x6c, 0xb6, 0xe8, + 0x1b, 0x68, 0xbd, 0x8f, 0xa2, 0xf1, 0x2c, 0x99, 0x97, 0x29, 0xfa, 0x12, 0x40, 0x4d, 0x6d, 0xf6, + 0x79, 0x30, 0xc1, 0x76, 0xf5, 0xa0, 0x7a, 0x54, 0xf7, 0x2c, 0x0b, 0xed, 0xc2, 0x9e, 0x4a, 0xd3, + 0x4d, 0x32, 0x79, 0x1d, 0xa8, 0xeb, 0x56, 0x7d, 0xbc, 0xd3, 0x69, 0xa9, 0x81, 0x5e, 0xc0, 0x73, + 0x3b, 0x4b, 0x09, 0xc6, 0xa4, 0x1e, 0xc2, 0x93, 0x91, 0x65, 0x37, 0xf9, 0x19, 0xab, 0x81, 0xce, + 0xf6, 0xa1, 0x03, 0x75, 0xcd, 0x2c, 0x85, 0x36, 0x86, 0xd3, 0x3f, 0x35, 0x68, 0x58, 0x82, 0x22, + 0x31, 0x34, 0x2c, 0x8d, 0x10, 0x97, 0xa5, 0x9b, 0x81, 0x65, 0x85, 0xc7, 0x56, 0x55, 0x47, 0x4f, + 0xd6, 0x4f, 0x48, 0xce, 0xe7, 0x54, 0xc8, 0x37, 0x68, 0xda, 0x5a, 0x22, 0x4f, 0x59, 0xb2, 0x02, + 0xd8, 0x62, 0x05, 0xb0, 0xcb, 0xf9, 0x0a, 0xa0, 0xaf, 0x1f, 0xad, 0x9d, 0x27, 0x47, 0xa7, 0xa2, + 0x09, 0x2d, 0x6e, 0x79, 0x39, 0xa1, 0x8c, 0x0a, 0xcb, 0x09, 0x65, 0x05, 0x94, 0x12, 0x32, 0xa0, + 0xff, 0x4f, 0x28, 0xa7, 0x38, 0x07, 0x48, 0x25, 0x44, 0x58, 0xd9, 0xf1, 0x96, 0x25, 0x49, 0xdd, + 0xb5, 0xe3, 0x0d, 0xe0, 0x57, 0x68, 0x58, 0xfa, 0x2a, 0x24, 0x73, 0x52, 0x4a, 0x66, 0xb5, 0xf4, + 0x77, 0x68, 0xda, 0x17, 0x9f, 0x14, 0x6e, 0x9a, 0x92, 0x56, 0xe5, 0x69, 0xce, 0xa9, 0x90, 0x31, + 0xec, 0xae, 0xe8, 0x8a, 0xec, 0xe7, 0x61, 0x98, 0x37, 0x8a, 0xbe, 0x5d, 0x1b, 0x68, 0x49, 0xa6, + 0x4e, 0x85, 0xdc, 0x42, 0xc3, 0x5a, 0x19, 0xa4, 0x68, 0xef, 0x97, 0x34, 0x2a, 0x67, 0xeb, 0x58, + 0x8d, 0x5a, 0x0c, 0xa1, 0x70, 0x8d, 0xad, 0xd3, 0xa8, 0x95, 0x39, 0x9c, 0xdf, 0x80, 0xf5, 0x05, + 0x70, 0x6e, 0xbf, 0xb0, 0x5f, 0xe6, 0x43, 0xbe, 0xed, 0x06, 0xa1, 0xbc, 0x9f, 0x0e, 0x99, 0xcf, + 0x27, 0x6e, 0xc0, 0x7f, 0xe0, 0x4f, 0x89, 0xfe, 0xbd, 0x9b, 0xbc, 0xe3, 0x01, 0x3f, 0x56, 0x3f, + 0x8e, 0xd5, 0x7d, 0x70, 0xd3, 0xef, 0x8c, 0xe1, 0xb6, 0xb2, 0x9c, 0xfd, 0x0b, 0x00, 0x00, 0xff, + 0xff, 0x7f, 0xa2, 0x2d, 0x17, 0x7c, 0x08, 0x00, 0x00, } diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index 446b2fd1bc7..7ea8f07ffc9 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -66,28 +66,28 @@ service CoreService { rpc ListStorage(google.protobuf.Empty) returns (CoreServiceTypes.ListStorageResponse) {}; /* - Register a new feature to the metadata store. + Register a new feature to the metadata store, or update an existing feature. If any validation errors occur, only the first encountered error will be returned. */ - rpc RegisterFeature(feast.specs.FeatureSpec) returns (CoreServiceTypes.RegisterFeatureResponse) {}; + rpc ApplyFeature(feast.specs.FeatureSpec) returns (CoreServiceTypes.ApplyFeatureResponse) {}; /* - Register a new feature group to the metadata store. + Register a new feature group to the metadata store, or update an existing feature group. If any validation errors occur, only the first encountered error will be returned. */ - rpc RegisterFeatureGroup(feast.specs.FeatureGroupSpec) returns (CoreServiceTypes.RegisterFeatureGroupResponse) {}; + rpc ApplyFeatureGroup(feast.specs.FeatureGroupSpec) returns (CoreServiceTypes.ApplyFeatureGroupResponse) {}; /* - Register a new entity to the metadata store. + Register a new entity to the metadata store, or update an existing entity. If any validation errors occur, only the first encountered error will be returned. */ - rpc RegisterEntity(feast.specs.EntitySpec) returns (CoreServiceTypes.RegisterEntityResponse) {}; + rpc ApplyEntity(feast.specs.EntitySpec) returns (CoreServiceTypes.ApplyEntityResponse) {}; /* - Register a new storage spec to the metadata store. + Register a new storage spec to the metadata store, or update an existing storage. If any validation errors occur, only the first encountered error will be returned. */ - rpc RegisterStorage(feast.specs.StorageSpec) returns (CoreServiceTypes.RegisterStorageResponse) {}; + rpc ApplyStorage(feast.specs.StorageSpec) returns (CoreServiceTypes.ApplyStorageResponse) {}; } message CoreServiceTypes { @@ -130,22 +130,22 @@ message CoreServiceTypes { } // Entity registration response - message RegisterEntityResponse { + message ApplyEntityResponse { string entityName = 1; } // Feature registration response - message RegisterFeatureResponse { + message ApplyFeatureResponse { string featureId = 1; } // Feature group registration response - message RegisterFeatureGroupResponse { + message ApplyFeatureGroupResponse { string featureGroupId = 1; } // Storage registration response - message RegisterStorageResponse { + message ApplyStorageResponse { string storageId = 1; } } From 0c66305e4b46678618aa07c11d3f3f5b7e45a540 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 12 Dec 2018 18:24:06 +0800 Subject: [PATCH 5/6] Create audit log objects --- .../feast/core/job/ScheduledJobMonitor.java | 6 ++- core/src/main/java/feast/core/log/Action.java | 19 +++++++ .../main/java/feast/core/log/AuditLogger.java | 6 +-- .../main/java/feast/core/log/Resource.java | 12 +++++ .../core/service/JobExecutionService.java | 28 +++++++---- .../core/service/JobManagementService.java | 12 +++-- .../java/feast/core/service/SpecService.java | 50 +++++++++++-------- .../core/storage/BigQueryStorageManager.java | 12 +++-- .../core/storage/BigTableStorageManager.java | 12 +++-- .../core/storage/PostgresStorageManager.java | 12 +++-- 10 files changed, 116 insertions(+), 53 deletions(-) create mode 100644 core/src/main/java/feast/core/log/Action.java create mode 100644 core/src/main/java/feast/core/log/Resource.java diff --git a/core/src/main/java/feast/core/job/ScheduledJobMonitor.java b/core/src/main/java/feast/core/job/ScheduledJobMonitor.java index 83641cedfe8..f8b5af16d0d 100644 --- a/core/src/main/java/feast/core/job/ScheduledJobMonitor.java +++ b/core/src/main/java/feast/core/job/ScheduledJobMonitor.java @@ -19,7 +19,9 @@ import com.google.common.base.Strings; import feast.core.dao.JobInfoRepository; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import feast.core.model.JobInfo; import feast.core.model.JobStatus; import feast.core.model.Metrics; @@ -74,9 +76,9 @@ public void pollStatusAndMetrics() { JobStatus jobStatus = jobMonitor.getJobStatus(jobId); if (job.getStatus() != jobStatus) { AuditLogger.log( - "Jobs", + Resource.JOB, jobId, - "Status Update", + Action.STATUS_CHANGE, "Job status updated from %s to %s", job.getStatus(), jobStatus); diff --git a/core/src/main/java/feast/core/log/Action.java b/core/src/main/java/feast/core/log/Action.java new file mode 100644 index 00000000000..c020434ee66 --- /dev/null +++ b/core/src/main/java/feast/core/log/Action.java @@ -0,0 +1,19 @@ +package feast.core.log; + +/** + * Actions taken for audit logging purposes + */ +public enum Action { + // Job-related actions + SUBMIT, + STATUS_CHANGE, + ABORT, + + // Spec-related + UPDATE, + REGISTER, + + // Storage-related + ADD, + SCHEMA_UPDATE, +} diff --git a/core/src/main/java/feast/core/log/AuditLogger.java b/core/src/main/java/feast/core/log/AuditLogger.java index 17b1b499aa5..bd667206601 100644 --- a/core/src/main/java/feast/core/log/AuditLogger.java +++ b/core/src/main/java/feast/core/log/AuditLogger.java @@ -39,12 +39,12 @@ public class AuditLogger { * @param detail additional detail. Supports string formatting. * @param args arguments to the detail string */ - public static void log(String resource, String id, String action, String detail, Object... args) { + public static void log(Resource resource, String id, Action action, String detail, Object... args) { Map map = new TreeMap<>(); map.put("timestamp", new Date().toString()); - map.put("resource", resource); + map.put("resource", resource.toString()); map.put("id", id); - map.put("action", action); + map.put("action", action.toString()); map.put("detail", Strings.lenientFormat(detail, args)); ObjectMessage msg = new ObjectMessage(map); diff --git a/core/src/main/java/feast/core/log/Resource.java b/core/src/main/java/feast/core/log/Resource.java new file mode 100644 index 00000000000..d57af53bbbb --- /dev/null +++ b/core/src/main/java/feast/core/log/Resource.java @@ -0,0 +1,12 @@ +package feast.core.log; + +/** + * Resources interacted with, for audit logging purposes + */ +public enum Resource { + FEATURE, + FEATURE_GROUP, + ENTITY, + STORAGE, + JOB +} diff --git a/core/src/main/java/feast/core/service/JobExecutionService.java b/core/src/main/java/feast/core/service/JobExecutionService.java index 337eaad40fa..b5259eff1ba 100644 --- a/core/src/main/java/feast/core/service/JobExecutionService.java +++ b/core/src/main/java/feast/core/service/JobExecutionService.java @@ -21,7 +21,9 @@ import feast.core.config.ImportJobDefaults; import feast.core.dao.JobInfoRepository; import feast.core.exception.JobExecutionException; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import feast.core.model.JobInfo; import feast.core.model.JobStatus; import feast.core.util.TypeConversion; @@ -54,8 +56,8 @@ public JobExecutionService(JobInfoRepository jobInfoRepository, ImportJobDefault } /** - * Submits a job defined by the importSpec to the runner and writes details about the - * job to the core database. + * Submits a job defined by the importSpec to the runner and writes details about the job to the + * core database. * * @param importSpec job import spec * @param jobPrefix prefix for job name @@ -67,7 +69,11 @@ public SubmitImportJobResponse submitJob(ImportSpec importSpec, String jobPrefix ProcessBuilder pb = getProcessBuilder(importSpec, jobId); log.info(String.format("Executing command: %s", String.join(" ", pb.command()))); AuditLogger.log( - "Jobs", jobId, "Submitting", "Building graph and submitting to %s", defaults.getRunner()); + Resource.JOB, + jobId, + Action.SUBMIT, + "Building graph and submitting to %s", + defaults.getRunner()); try { JobInfo jobInfo = new JobInfo(jobId, "", defaults.getRunner(), importSpec, JobStatus.PENDING); jobInfoRepository.saveAndFlush(jobInfo); @@ -79,20 +85,20 @@ public SubmitImportJobResponse submitJob(ImportSpec importSpec, String jobPrefix } updateJobExtId(jobId, jobExtId); AuditLogger.log( - "Jobs", + Resource.JOB, jobId, - "Submitted", - "Job submitted to runner %s with runner id %s", + Action.STATUS_CHANGE, + "Job submitted to runner %s with runner id %s.", defaults.getRunner(), jobExtId); return SubmitImportJobResponse.newBuilder().setJobId(jobId).build(); } catch (Exception e) { updateJobStatus(jobId, JobStatus.ERROR); AuditLogger.log( - "Jobs", + Resource.JOB, jobId, - "Submit Error", - "Job failed to be submitted to runner %s", + Action.STATUS_CHANGE, + "Job failed to be submitted to runner %s. Job status changed to ERROR.", defaults.getRunner()); throw new JobExecutionException(String.format("Error running ingestion job: %s", e), e); } @@ -100,6 +106,7 @@ public SubmitImportJobResponse submitJob(ImportSpec importSpec, String jobPrefix /** * Update a given job's status + * * @param jobId * @param status */ @@ -114,6 +121,7 @@ public void updateJobStatus(String jobId, JobStatus status) { /** * Update a given job's external id + * * @param jobId * @param jobExtId */ @@ -128,6 +136,7 @@ public void updateJobExtId(String jobId, String jobExtId) { /** * Builds the command to execute the ingestion job + * * @param importSpec * @param jobId * @return configured ProcessBuilder @@ -155,6 +164,7 @@ private String option(String key, String value) { /** * Run the given process and extract the job id from the output logs + * * @param p Process * @return job id */ diff --git a/core/src/main/java/feast/core/service/JobManagementService.java b/core/src/main/java/feast/core/service/JobManagementService.java index 79f3fc92423..01257e33a7b 100644 --- a/core/src/main/java/feast/core/service/JobManagementService.java +++ b/core/src/main/java/feast/core/service/JobManagementService.java @@ -23,7 +23,9 @@ import feast.core.dao.MetricsRepository; import feast.core.exception.RetrievalException; import feast.core.job.JobManager; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import feast.core.model.JobInfo; import feast.core.model.JobStatus; import feast.core.model.Metrics; @@ -54,6 +56,7 @@ public JobManagementService( /** * Lists all jobs registered to the db. + * * @return list of JobDetails */ @Transactional @@ -64,6 +67,7 @@ public List listJobs() { /** * Gets information regarding a single job. + * * @param id feast-internal job id * @return JobDetail for that job */ @@ -82,10 +86,10 @@ public JobDetail getJob(String id) { } /** - * Drain the given job. If this is successful, the job will start the draining process. - * When the draining process is complete, the job will be cleaned up and removed. + * Drain the given job. If this is successful, the job will start the draining process. When the + * draining process is complete, the job will be cleaned up and removed. * - * Batch jobs will be cancelled, as draining these jobs is not supported by beam. + *

Batch jobs will be cancelled, as draining these jobs is not supported by beam. * * @param id feast-internal id of a job */ @@ -101,7 +105,7 @@ public void abortJob(String id) { jobManager.abortJob(job.getExtId()); job.setStatus(JobStatus.ABORTING); - AuditLogger.log("Jobs", id, "Aborting", "Triggering draining of job"); + AuditLogger.log(Resource.JOB, id, Action.ABORT, "Triggering draining of job"); jobInfoRepository.saveAndFlush(job); } } diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 69f283fe932..f4ab5c6428f 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -25,7 +25,9 @@ import feast.core.dao.StorageInfoRepository; import feast.core.exception.RegistrationException; import feast.core.exception.RetrievalException; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import feast.core.model.EntityInfo; import feast.core.model.FeatureGroupInfo; import feast.core.model.FeatureInfo; @@ -209,10 +211,10 @@ public List listStorage() throws RetrievalException { public FeatureInfo applyFeature(FeatureSpec spec) throws RegistrationException { try { FeatureInfo featureInfo = featureInfoRepository.findById(spec.getId()).orElse(null); - String action; + Action action; if (featureInfo != null) { featureInfo.update(spec); - action = "Updated"; + action = Action.UPDATE; } else { EntityInfo entity = entityInfoRepository.findById(spec.getEntity()).orElse(null); FeatureGroupInfo featureGroupInfo = @@ -227,14 +229,18 @@ public FeatureInfo applyFeature(FeatureSpec spec) throws RegistrationException { FeatureInfo resolvedFeatureInfo = featureInfo.resolve(); FeatureSpec resolvedFeatureSpec = resolvedFeatureInfo.getFeatureSpec(); schemaManager.registerFeature(resolvedFeatureSpec); - action = "Registered"; + action = Action.REGISTER; } FeatureInfo out = featureInfoRepository.saveAndFlush(featureInfo); if (!out.getId().equals(spec.getId())) { throw new RegistrationException("failed to register or update feature"); } AuditLogger.log( - "Feature", spec.getId(), action, "Feature applied: %s", JsonFormat.printer().print(spec)); + Resource.FEATURE, + spec.getId(), + action, + "Feature applied: %s", + JsonFormat.printer().print(spec)); return out; } catch (Exception e) { @@ -244,8 +250,9 @@ public FeatureInfo applyFeature(FeatureSpec spec) throws RegistrationException { } /** - * Applies the given feature group spec to the registry. If the entity does not yet exist, it will be - * registered to the system. Otherwise, the fields will be updated as per the given feature group spec. + * Applies the given feature group spec to the registry. If the entity does not yet exist, it will + * be registered to the system. Otherwise, the fields will be updated as per the given feature + * group spec. * * @param spec FeatureGroupSpec * @return registered FeatureGroupInfo @@ -253,11 +260,12 @@ public FeatureInfo applyFeature(FeatureSpec spec) throws RegistrationException { */ public FeatureGroupInfo applyFeatureGroup(FeatureGroupSpec spec) throws RegistrationException { try { - FeatureGroupInfo featureGroupInfo = featureGroupInfoRepository.findById(spec.getId()).orElse(null); - String action; + FeatureGroupInfo featureGroupInfo = + featureGroupInfoRepository.findById(spec.getId()).orElse(null); + Action action; if (featureGroupInfo != null) { featureGroupInfo.update(spec); - action = "Updated"; + action = Action.UPDATE; } else { StorageInfo servingStore = storageInfoRepository @@ -273,16 +281,15 @@ public FeatureGroupInfo applyFeatureGroup(FeatureGroupSpec spec) throws Registra ? spec.getDataStores().getWarehouse().getId() : "") .orElse(null); - featureGroupInfo = - new FeatureGroupInfo(spec, servingStore, warehouseStore); - action = "Registered"; + featureGroupInfo = new FeatureGroupInfo(spec, servingStore, warehouseStore); + action = Action.REGISTER; } FeatureGroupInfo out = featureGroupInfoRepository.saveAndFlush(featureGroupInfo); if (!out.getId().equals(spec.getId())) { throw new RegistrationException("failed to register or update feature group"); } AuditLogger.log( - "FeatureGroup", + Resource.FEATURE_GROUP, spec.getId(), action, "Feature group applied: %s", @@ -307,20 +314,20 @@ public FeatureGroupInfo applyFeatureGroup(FeatureGroupSpec spec) throws Registra public EntityInfo applyEntity(EntitySpec spec) throws RegistrationException { try { EntityInfo entityInfo = entityInfoRepository.findById(spec.getName()).orElse(null); - String action; + Action action; if (entityInfo != null) { entityInfo.update(spec); - action = "Updated"; + action = Action.UPDATE; } else { entityInfo = new EntityInfo(spec); - action = "Registered"; + action = Action.REGISTER; } EntityInfo out = entityInfoRepository.saveAndFlush(entityInfo); if (!out.getName().equals(spec.getName())) { throw new RegistrationException("failed to register or update entity"); } AuditLogger.log( - "Entity", spec.getName(), action, "Entity: %s", JsonFormat.printer().print(spec)); + Resource.FEATURE_GROUP, spec.getName(), action, "Entity: %s", JsonFormat.printer().print(spec)); return out; } catch (Exception e) { throw new RegistrationException( @@ -339,7 +346,10 @@ public StorageInfo registerStorage(StorageSpec spec) throws RegistrationExceptio try { StorageInfo storageInfo = storageInfoRepository.findById(spec.getId()).orElse(null); if (storageInfo != null) { - if (!storageInfo.getType().equals(spec.getType()) && !storageInfo.getOptions().equals(TypeConversion.convertMapToJsonString(spec.getOptionsMap()))) { + if (!storageInfo.getType().equals(spec.getType()) + && !storageInfo + .getOptions() + .equals(TypeConversion.convertMapToJsonString(spec.getOptionsMap()))) { throw new IllegalArgumentException("updating storage specs is not allowed"); } return storageInfo; @@ -351,9 +361,9 @@ public StorageInfo registerStorage(StorageSpec spec) throws RegistrationExceptio } schemaManager.registerStorage(spec); AuditLogger.log( - "Storage", + Resource.STORAGE, spec.getId(), - "Registered", + Action.REGISTER, "New storage registered: %s", JsonFormat.printer().print(spec)); return out; diff --git a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java index 40e1088b02c..5ad463ec4f4 100644 --- a/core/src/main/java/feast/core/storage/BigQueryStorageManager.java +++ b/core/src/main/java/feast/core/storage/BigQueryStorageManager.java @@ -21,7 +21,9 @@ import com.google.cloud.bigquery.TimePartitioning.Type; import com.google.common.base.Strings; import com.google.protobuf.util.JsonFormat; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.ValueProto.ValueType; import feast.types.ValueProto.ValueType.Enum; @@ -138,11 +140,11 @@ public void registerNewFeature(FeatureSpec featureSpec) { && !f.equals(FIELD_EVENT_TIMESTAMP)) .collect(Collectors.toList())); AuditLogger.log( - "Storage", - this.id, - "Schema Updated", - "Bigquery schema updated for feature %s", - featureSpec.getId()); + Resource.STORAGE, + this.id, + Action.SCHEMA_UPDATE, + "Bigquery schema updated for feature %s", + featureSpec.getId()); } private void checkTableCreation(Table table, FeatureSpec featureSpec) { diff --git a/core/src/main/java/feast/core/storage/BigTableStorageManager.java b/core/src/main/java/feast/core/storage/BigTableStorageManager.java index a42a6d01c40..cfd0ddd7705 100644 --- a/core/src/main/java/feast/core/storage/BigTableStorageManager.java +++ b/core/src/main/java/feast/core/storage/BigTableStorageManager.java @@ -18,7 +18,9 @@ package feast.core.storage; import com.google.common.base.Strings; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import feast.specs.FeatureSpecProto.FeatureSpec; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hbase.TableName; @@ -81,11 +83,11 @@ public void registerNewFeature(FeatureSpec featureSpec) { log.info("Created new column family: {} for entity: {}", columnFamily, entityName); } AuditLogger.log( - "Storage", - this.id, - "Schema Updated", - "Bigtable schema updated for feature %s", - featureSpec.getId()); + Resource.STORAGE, + this.id, + Action.SCHEMA_UPDATE, + "Bigtable schema updated for feature %s", + featureSpec.getId()); } catch (IOException e) { log.error("Unable to create table in BigTable: {}", e); throw new StorageInitializationException("Unable to create table in BigTable", e); diff --git a/core/src/main/java/feast/core/storage/PostgresStorageManager.java b/core/src/main/java/feast/core/storage/PostgresStorageManager.java index 02143f97e98..9fef9f0d2cb 100644 --- a/core/src/main/java/feast/core/storage/PostgresStorageManager.java +++ b/core/src/main/java/feast/core/storage/PostgresStorageManager.java @@ -17,7 +17,9 @@ package feast.core.storage; +import feast.core.log.Action; import feast.core.log.AuditLogger; +import feast.core.log.Resource; import org.jdbi.v3.core.Jdbi; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.ValueProto.ValueType.Enum; @@ -76,11 +78,11 @@ public void registerNewFeature(FeatureSpec featureSpec) { return null; }); AuditLogger.log( - "Storage", - this.id, - "Schema Updated", - "Postgres schema updated for feature %s", - featureSpec.getId()); + Resource.STORAGE, + this.id, + Action.SCHEMA_UPDATE, + "Postgres schema updated for feature %s", + featureSpec.getId()); } private String createFieldType(FeatureSpec featureSpec) { From f878d3f03d4fda2d199e727c9fb2479bbd286e4e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 13 Dec 2018 11:03:07 +0800 Subject: [PATCH 6/6] Clean up update code --- .../java/feast/core/model/EntityInfo.java | 28 ++++++--------- .../feast/core/model/FeatureGroupInfo.java | 20 ++--------- .../java/feast/core/model/FeatureInfo.java | 36 +++++-------------- .../feast/core/model/FeatureInfoTest.java | 21 +++++++++++ 4 files changed, 42 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/feast/core/model/EntityInfo.java b/core/src/main/java/feast/core/model/EntityInfo.java index 612c61ae359..e990361e498 100644 --- a/core/src/main/java/feast/core/model/EntityInfo.java +++ b/core/src/main/java/feast/core/model/EntityInfo.java @@ -17,14 +17,13 @@ package feast.core.model; +import feast.core.UIServiceProto.UIServiceTypes.EntityDetail; +import feast.specs.EntitySpecProto.EntitySpec; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; -import feast.core.UIServiceProto.UIServiceTypes.EntityDetail; -import feast.specs.EntitySpecProto.EntitySpec; import javax.persistence.*; - import java.util.List; import static feast.core.util.TypeConversion.convertTagStringToList; @@ -66,25 +65,21 @@ public EntityInfo(EntitySpec spec) { this.tags = String.join(",", spec.getTagsList()); } - /** - * Get the entity spec associated with this record. - */ + /** Get the entity spec associated with this record. */ public EntitySpec getEntitySpec() { return EntitySpec.newBuilder() - .setName(name) - .setDescription(description) - .addAllTags(convertTagStringToList(tags)) - .build(); + .setName(name) + .setDescription(description) + .addAllTags(convertTagStringToList(tags)) + .build(); } - /** - * Get the entity detail containing both spec and metadata, associated with this record. - */ + /** Get the entity detail containing both spec and metadata, associated with this record. */ public EntityDetail getEntityDetail() { return EntityDetail.newBuilder() - .setSpec(this.getEntitySpec()) - .setLastUpdated(convertTimestamp(this.getLastUpdated())) - .build(); + .setSpec(this.getEntitySpec()) + .setLastUpdated(convertTimestamp(this.getLastUpdated())) + .build(); } /** @@ -96,5 +91,4 @@ public void update(EntitySpec update) { this.description = update.getDescription(); this.tags = String.join(",", update.getTagsList()); } - } diff --git a/core/src/main/java/feast/core/model/FeatureGroupInfo.java b/core/src/main/java/feast/core/model/FeatureGroupInfo.java index f302f540e5a..4fef15512df 100644 --- a/core/src/main/java/feast/core/model/FeatureGroupInfo.java +++ b/core/src/main/java/feast/core/model/FeatureGroupInfo.java @@ -117,23 +117,7 @@ public void update(FeatureGroupSpec update) throws IllegalArgumentException { } private boolean isLegalUpdate(FeatureGroupSpec update) { - DataStore updatedWarehouseStore = - update.getDataStores().hasWarehouse() ? update.getDataStores().getWarehouse() : null; - DataStore updatedServingStore = - update.getDataStores().hasServing() ? update.getDataStores().getServing() : null; - return isStoreEqual(this.warehouseStore, this.warehouseStoreOpts, updatedWarehouseStore) - && isStoreEqual(this.servingStore, this.servingStoreOpts, updatedServingStore); - } - - private boolean isStoreEqual(StorageInfo oldStore, String oldStoreOpts, DataStore newStore) { - return getStorageId(oldStore).equals(newStore == null ? "" : newStore.getId()) - && oldStoreOpts.equals( - newStore == null - ? "" - : TypeConversion.convertMapToJsonString(newStore.getOptionsMap())); - } - - private String getStorageId(StorageInfo storage) { - return storage == null ? "" : storage.getId(); + FeatureGroupSpec spec = this.getFeatureGroupSpec(); + return spec.getDataStores().equals(update.getDataStores()); } } diff --git a/core/src/main/java/feast/core/model/FeatureInfo.java b/core/src/main/java/feast/core/model/FeatureInfo.java index 567a4bec67b..d2ac1ed68d6 100644 --- a/core/src/main/java/feast/core/model/FeatureInfo.java +++ b/core/src/main/java/feast/core/model/FeatureInfo.java @@ -273,33 +273,13 @@ public void update(FeatureSpec update) throws IllegalArgumentException { } private boolean isLegalUpdate(FeatureSpec update) { - DataStore updatedWarehouseStore = - update.getDataStores().hasWarehouse() ? update.getDataStores().getWarehouse() : null; - DataStore updatedServingStore = - update.getDataStores().hasServing() ? update.getDataStores().getServing() : null; - return update.getName().equals(this.name) - && update.getEntity().equals(this.entity.getName()) - && update.getGranularityValue() == this.granularity.getNumber() - && update.getValueTypeValue() == this.valueType.getNumber() - && update.getGroup().equals(getFeatureGroupId(this.featureGroup)) - && TypeConversion.convertMapToJsonString(update.getOptionsMap()).equals(this.options) - && isStoreEqual(this.warehouseStore, this.warehouseStoreOpts, updatedWarehouseStore) - && isStoreEqual(this.servingStore, this.servingStoreOpts, updatedServingStore); - } - - private boolean isStoreEqual(StorageInfo oldStore, String oldStoreOpts, DataStore newStore) { - return getStorageId(oldStore).equals(newStore == null ? "" : newStore.getId()) - && oldStoreOpts.equals( - newStore == null - ? "" - : TypeConversion.convertMapToJsonString(newStore.getOptionsMap())); - } - - private String getFeatureGroupId(FeatureGroupInfo featureGroupInfo) { - return featureGroupInfo == null ? "" : featureGroupInfo.getId(); - } - - private String getStorageId(StorageInfo storage) { - return storage == null ? "" : storage.getId(); + FeatureSpec spec = this.getFeatureSpec(); + return spec.getName().equals(update.getName()) + && spec.getEntity().equals(update.getEntity()) + && spec.getGranularity().equals(update.getGranularity()) + && spec.getValueType().equals(update.getValueType()) + && spec.getGroup().equals(update.getGroup()) + && spec.getOptionsMap().equals(update.getOptionsMap()) + && spec.getDataStores().equals(update.getDataStores()); } } diff --git a/core/src/test/java/feast/core/model/FeatureInfoTest.java b/core/src/test/java/feast/core/model/FeatureInfoTest.java index c3083fbce18..a85b99711f3 100644 --- a/core/src/test/java/feast/core/model/FeatureInfoTest.java +++ b/core/src/test/java/feast/core/model/FeatureInfoTest.java @@ -232,4 +232,25 @@ public void shouldThrowExceptionIfImmutableFieldsChanged() { exception.expectMessage( "Feature already exists. Update only allowed for fields: [owner, description, uri, tags]"); featureInfo.update(update); } + + + @Test + public void shouldThrowExceptionIfImmutableFieldsChangedToNull() { + FeatureSpec update = + FeatureSpec.newBuilder() + .setId("entity.NONE.name") + .setName("name") + .setOwner("owner2") + .setDescription("overwrite") + .setEntity("entity") + .setUri("new_uri") + .setGranularity(Granularity.Enum.NONE) + .setValueType(ValueType.Enum.BYTES) + .addTags("new_tag") + .build(); + + exception.expect(IllegalArgumentException.class); + exception.expectMessage( "Feature already exists. Update only allowed for fields: [owner, description, uri, tags]"); + featureInfo.update(update); + } }