feat: add UpsertDatapoints and RemoveDatapoints rpcs to IndexService …

…in aiplatform v1beta1 index_service.proto

PiperOrigin-RevId: 469481692

google/cloud/aiplatform/v1beta1/dataset_service.proto

@@ -399,8 +399,7 @@ message ListSavedQueriesRequest {
 
 // Response message for [DatasetService.ListSavedQueries][google.cloud.aiplatform.v1beta1.DatasetService.ListSavedQueries].
 message ListSavedQueriesResponse {
-  // A list of SavedQueries that matches the specified filter in the
-  // request.
+  // A list of SavedQueries that match the specified filter in the request.
   repeated SavedQuery saved_queries = 1;
 
   // The standard List next-page token.

google/cloud/aiplatform/v1beta1/explanation.proto

@@ -183,8 +183,8 @@ message ExplanationSpec {
   // Required. Parameters that configure explaining of the Model's predictions.
   ExplanationParameters parameters = 1 [(google.api.field_behavior) = REQUIRED];
 
-  // Required. Metadata describing the Model's input and output for explanation.
-  ExplanationMetadata metadata = 2 [(google.api.field_behavior) = REQUIRED];
+  // Optional. Metadata describing the Model's input and output for explanation.
+  ExplanationMetadata metadata = 2 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Parameters to configure explaining for Model's predictions.
@@ -229,7 +229,7 @@ message ExplanationParameters {
   // explaining.
   //
   // If not populated, returns attributions for [top_k][google.cloud.aiplatform.v1beta1.ExplanationParameters.top_k] indices of outputs.
-  // If neither top_k nor output_indeices is populated, returns the argmax
+  // If neither top_k nor output_indices is populated, returns the argmax
   // index of the outputs.
   //
   // Only applicable to Models that predict multiple outputs (e,g, multi-class

google/cloud/aiplatform/v1beta1/index.proto

@@ -38,6 +38,21 @@ message Index {
     pattern: "projects/{project}/locations/{location}/indexes/{index}"
   };
 
+  // The update method of an Index.
+  enum IndexUpdateMethod {
+    // Should not be used.
+    INDEX_UPDATE_METHOD_UNSPECIFIED = 0;
+
+    // BatchUpdate: user can call UpdateIndex with files on Cloud Storage of
+    // datapoints to update.
+    BATCH_UPDATE = 1;
+
+    // StreamUpdate: user can call UpsertDatapoints/DeleteDatapoints to update
+    // the Index and the updates will be applied in corresponding
+    // DeployedIndexes in nearly real-time.
+    STREAM_UPDATE = 2;
+  }
+
   // Output only. The resource name of the Index.
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
@@ -93,4 +108,65 @@ message Index {
   // in the Index. Result of any successfully completed Operation on the Index
   // is reflected in it.
   google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Stats of the index resource.
+  IndexStats index_stats = 14 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Immutable. The update method to use with this Index. If not set, BATCH_UPDATE will be
+  // used by default.
+  IndexUpdateMethod index_update_method = 16 [(google.api.field_behavior) = IMMUTABLE];
+}
+
+// A datapoint of Index.
+message IndexDatapoint {
+  // Restriction of a datapoint which describe its attributes(tokens) from each
+  // of several attribute categories(namespaces).
+  message Restriction {
+    // The namespace of this restriction. eg: color.
+    string namespace = 1;
+
+    // The attributes to allow in this namespace. eg: 'red'
+    repeated string allow_list = 2;
+
+    // The attributes to deny in this namespace. eg: 'blue'
+    repeated string deny_list = 3;
+  }
+
+  // Crowding tag is a constraint on a neighbor list produced by nearest
+  // neighbor search requiring that no more than some value k' of the k
+  // neighbors returned have the same value of crowding_attribute.
+  message CrowdingTag {
+    // The attribute value used for crowding.  The maximum number of neighbors
+    // to return per crowding attribute value
+    // (per_crowding_attribute_num_neighbors) is configured per-query. This
+    // field is ignored if per_crowding_attribute_num_neighbors is larger than
+    // the total number of neighbors to return for a given query.
+    string crowding_attribute = 1;
+  }
+
+  // Required. Unique identifier of the datapoint.
+  string datapoint_id = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. Feature embedding vector. An array of numbers with the length of
+  // [NearestNeighborSearchConfig.dimensions].
+  repeated float feature_vector = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. List of Restrict of the datapoint, used to perform "restricted searches"
+  // where boolean rule are used to filter the subset of the database eligible
+  // for matching.
+  // See: https://cloud.google.com/vertex-ai/docs/matching-engine/filtering
+  repeated Restriction restricts = 4 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. CrowdingTag of the datapoint, the number of neighbors to return in each
+  // crowding can be configured during query.
+  CrowdingTag crowding_tag = 5 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Stats of the Index.
+message IndexStats {
+  // Output only. The number of vectors in the Index.
+  int64 vectors_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. The number of shards in the Index.
+  int32 shards_count = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
 }

google/cloud/aiplatform/v1beta1/index_service.proto

@@ -93,6 +93,22 @@ service IndexService {
       metadata_type: "DeleteOperationMetadata"
     };
   }
+
+  // Add/update Datapoints into an Index.
+  rpc UpsertDatapoints(UpsertDatapointsRequest) returns (UpsertDatapointsResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/{index=projects/*/locations/*/indexes/*}:upsertDatapoints"
+      body: "*"
+    };
+  }
+
+  // Remove Datapoints from an Index.
+  rpc RemoveDatapoints(RemoveDatapointsRequest) returns (RemoveDatapointsResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/{index=projects/*/locations/*/indexes/*}:removeDatapoints"
+      body: "*"
+    };
+  }
 }
 
 // Request message for [IndexService.CreateIndex][google.cloud.aiplatform.v1beta1.IndexService.CreateIndex].
@@ -201,6 +217,48 @@ message DeleteIndexRequest {
   ];
 }
 
+// Request message for [IndexService.UpsertDatapoints][google.cloud.aiplatform.v1beta1.IndexService.UpsertDatapoints]
+message UpsertDatapointsRequest {
+  // Required. The name of the Index resource to be updated.
+  // Format:
+  // `projects/{project}/locations/{location}/indexes/{index}`
+  string index = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Index"
+    }
+  ];
+
+  // A list of datapoints to be created/updated.
+  repeated IndexDatapoint datapoints = 2;
+}
+
+// Response message for [IndexService.UpsertDatapoints][google.cloud.aiplatform.v1beta1.IndexService.UpsertDatapoints]
+message UpsertDatapointsResponse {
+
+}
+
+// Request message for [IndexService.RemoveDatapoints][google.cloud.aiplatform.v1beta1.IndexService.RemoveDatapoints]
+message RemoveDatapointsRequest {
+  // Required. The name of the Index resource to be updated.
+  // Format:
+  // `projects/{project}/locations/{location}/indexes/{index}`
+  string index = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Index"
+    }
+  ];
+
+  // A list of datapoint ids to be deleted.
+  repeated string datapoint_ids = 2;
+}
+
+// Response message for [IndexService.RemoveDatapoints][google.cloud.aiplatform.v1beta1.IndexService.RemoveDatapoints]
+message RemoveDatapointsResponse {
+
+}
+
 // Runtime operation metadata with regard to Matching Engine Index.
 message NearestNeighborSearchOperationMetadata {
   message RecordError {

google/cloud/aiplatform/v1beta1/model_monitoring.proto

@@ -27,7 +27,7 @@ option java_package = "com.google.cloud.aiplatform.v1beta1";
 option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
 option ruby_package = "Google::Cloud::AIPlatform::V1beta1";
 
-// Next ID: 5
+// The model monitoring configuration used for Batch Prediction Job.
 message ModelMonitoringConfig {
   // Model monitoring objective config.
   repeated ModelMonitoringObjectiveConfig objective_configs = 3;
@@ -43,17 +43,24 @@ message ModelMonitoringConfig {
   // For models trained with Vertex AI, this field must be set as all the
   // fields in predict instance formatted as string.
   string analysis_instance_schema_uri = 4;
+
+  // A Google Cloud Storage location for batch prediction model monitoring to
+  // dump statistics and anomalies.
+  // If not provided, a folder will be created in customer project to hold
+  // statistics and anomalies.
+  GcsDestination stats_anomalies_base_directory = 5;
 }
 
-// Next ID: 8
+// The objective configuration for model monitoring, including the information
+// needed to detect anomalies for one particular model.
 message ModelMonitoringObjectiveConfig {
   // Training Dataset information.
   message TrainingDataset {
     oneof data_source {
       // The resource name of the Dataset used to train this Model.
       string dataset = 3 [(google.api.resource_reference) = {
-                            type: "aiplatform.googleapis.com/Dataset"
-                          }];
+        type: "aiplatform.googleapis.com/Dataset"
+      }];
 
       // The Google Cloud Storage uri of the unmanaged Dataset used to train
       // this Model.
@@ -128,8 +135,10 @@ message ModelMonitoringObjectiveConfig {
   // The config for integrating with Vertex Explainable AI. Only applicable if
   // the Model has explanation_spec populated.
   message ExplanationConfig {
-    // Output from [BatchPredictionJob][google.cloud.aiplatform.v1beta1.BatchPredictionJob] for Model Monitoring baseline dataset,
-    // which can be used to generate baseline attribution scores.
+    // Output from
+    // [BatchPredictionJob][google.cloud.aiplatform.v1beta1.BatchPredictionJob]
+    // for Model Monitoring baseline dataset, which can be used to generate
+    // baseline attribution scores.
     message ExplanationBaseline {
       // The storage format of the predictions generated BatchPrediction job.
       enum PredictionFormat {
@@ -171,7 +180,8 @@ message ModelMonitoringObjectiveConfig {
   TrainingDataset training_dataset = 1;
 
   // The config for skew between training data and prediction data.
-  TrainingPredictionSkewDetectionConfig training_prediction_skew_detection_config = 2;
+  TrainingPredictionSkewDetectionConfig
+      training_prediction_skew_detection_config = 2;
 
   // The config for drift of prediction data.
   PredictionDriftDetectionConfig prediction_drift_detection_config = 3;
@@ -180,7 +190,6 @@ message ModelMonitoringObjectiveConfig {
   ExplanationConfig explanation_config = 5;
 }
 
-// Next ID: 3
 message ModelMonitoringAlertConfig {
   // The config for email alert.
   message EmailAlertConfig {
@@ -202,7 +211,6 @@ message ModelMonitoringAlertConfig {
 }
 
 // The config for feature monitoring threshold.
-// Next ID: 3
 message ThresholdConfig {
   oneof threshold {
     // Specify a threshold value that can trigger the alert.
@@ -219,7 +227,6 @@ message ThresholdConfig {
 
 // Sampling Strategy for logging, can be for both training and prediction
 // dataset.
-// Next ID: 2
 message SamplingStrategy {
   // Requests are randomly selected.
   message RandomSampleConfig {

google/cloud/aiplatform/v1beta1/model_service.proto

@@ -465,7 +465,7 @@ message MergeVersionAliasesRequest {
 
   // Required. The set of version aliases to merge.
   // The alias should be at most 128 characters, and match
-  // `[a-z][a-z0-9-]{0,126}[a-z-0-9]`.
+  // `[a-z][a-zA-Z0-9-]{0,126}[a-z-0-9]`.
   // Add the `-` prefix to an alias means removing that alias from the version.
   // `-` is NOT counted in the 128 characters. Example: `-golden` means removing
   // the `golden` alias from the version.

google/cloud/aiplatform/v1beta1/saved_query.proto

@@ -59,6 +59,7 @@ message SavedQuery {
 
   // Required. Problem type of the SavedQuery.
   // Allowed values:
+  //
   // * IMAGE_CLASSIFICATION_SINGLE_LABEL
   // * IMAGE_CLASSIFICATION_MULTI_LABEL
   // * IMAGE_BOUNDING_POLY
@@ -74,7 +75,7 @@ message SavedQuery {
   // Output only. Number of AnnotationSpecs in the context of the SavedQuery.
   int32 annotation_spec_count = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
 
-  // Used to perform consistent read-modify-write updates. If not set, a blind
+  // Used to perform a consistent read-modify-write update. If not set, a blind
   // "overwrite" update happens.
   string etag = 8;
 

google/cloud/aiplatform/v1beta1/vizier_service.proto

@@ -285,7 +285,7 @@ message SuggestTrialsRequest {
     }
   ];
 
-  // Required. The number of suggestions requested.
+  // Required. The number of suggestions requested. It must be positive.
   int32 suggestion_count = 2 [(google.api.field_behavior) = REQUIRED];
 
   // Required. The identifier of the client that is requesting the suggestion.