Added search response processors.

Signed-off-by: dblock <dblock@amazon.com>

.cspell

@@ -91,7 +91,6 @@ mmapfs
 mmdb
 mokotoff
 Moneyball
-Moneyball
 msearch
 msmarco
 mtermvectors
@@ -138,6 +137,9 @@ Reindex
 relo
 reloadcerts
 remotestore
+rerank
+Rerank
+Reranker
 rethrottle
 Rethrottle
 rolesmapping

CHANGELOG.md

@@ -71,6 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added API spec for security plugin  ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271))
 - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
 - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504))
+- Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, and `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#440](https://github.com/opensearch-project/opensearch-api-specification/pull/440))
 
 ### Changed
 

spec/schemas/search_pipeline._common.yaml

@@ -23,7 +23,7 @@ components:
         response_processors:
           type: array
           items:
-            $ref: '#/components/schemas/RequestProcessor'
+            $ref: '#/components/schemas/ResponseProcessor'
         phase_results_processors:
           type: array
           items:
@@ -187,6 +187,219 @@ components:
           type: string
       required:
         - sample_factor
+    ResponseProcessor:
+      oneOf:
+        - type: object
+          title: personalize_search_ranking
+          properties:
+            personalize_search_ranking:
+              $ref: '#/components/schemas/PersonalizeSearchRankingResponseProcessor'
+          required:
+            - personalize_search_ranking
+        - type: object
+          title: retrieval_augmented_generation
+          properties:
+            retrieval_augmented_generation:
+              $ref: '#/components/schemas/RetrievalAugmentedGenerationResponseProcessor'
+          required:
+            - retrieval_augmented_generation
+        - type: object
+          title: rename_field
+          properties:
+            rename_field:
+              $ref: '#/components/schemas/RenameFieldResponseProcessor'
+          required:
+            - rename_field
+        - type: object
+          title: rerank
+          properties:
+            rerank:
+              $ref: '#/components/schemas/RerankResponseProcessor'
+          required:
+            - rerank
+        - type: object
+          title: collapse
+          properties:
+            collapse:
+              $ref: '#/components/schemas/CollapseResponseProcessor'
+          required:
+            - collapse
+        - type: object
+          title: truncate_hits
+          properties:
+            truncate_hits:
+              $ref: '#/components/schemas/TruncateHitsResponseProcessor'
+          required:
+            - truncate_hits
+        - type: object
+          title: sort
+          properties:
+            sort:
+              $ref: '#/components/schemas/SortResponseProcessor'
+          required:
+            - sort
+        - type: object
+          title: split
+          properties:
+            split:
+              $ref: '#/components/schemas/SplitResponseProcessor'
+          required:
+            - split
+    PersonalizeSearchRankingResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        campaign_arn:
+          type: string
+        recipe:
+          type: string
+        weight:
+          type: number
+          format: float
+        item_id_field:
+          type: string
+        iam_role_arn:
+          type: string
+      required:
+        - campaign_arn
+        - recipe
+        - weight
+    RetrievalAugmentedGenerationResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        model_id:
+          type: string
+        context_field_list:
+          type: array
+          items:
+            type: string
+        system_prompt:
+          type: string
+        user_instructions:
+          type: string
+      required:
+        - context_field_list
+        - model_id
+    RenameFieldResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        field:
+          type: string
+        target_field:
+          type: string
+      required:
+        - field
+        - target_field
+    RerankContext:
+      type: object
+      properties:
+        document_fields:
+          type: array
+          items:
+            type: string
+      required:
+        - document_fields
+    MLOpenSearchReranker:
+      type: object
+      properties:
+        model_id:
+          type: string
+      required:
+        - model_id
+    RerankResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        ml_opensearch:
+          $ref: '#/components/schemas/MLOpenSearchReranker'
+        context:
+          $ref: '#/components/schemas/RerankContext'
+    CollapseResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        field:
+          type: string
+        context_prefix:
+          type: string
+      required:
+        - field
+    TruncateHitsResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        target_size:
+          type: integer
+          format: int32
+        context_prefix:
+          type: string
+    SortResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        field:
+          type: string
+        order:
+          type: string
+        target_field:
+          type: string
+      required:
+        - field
+    SplitResponseProcessor:
+      type: object
+      properties:
+        tag:
+          type: string
+        description:
+          type: string
+        ignore_failure:
+          type: boolean
+        field:
+          type: string
+        separator:
+          type: string
+        preserve_trailing:
+          type: boolean
+        target_field:
+          type: string
+      required:
+        - field
+        - separator
     PhaseResultsProcessor:
       oneOf:
         - type: object

tests/default/_core/search/pipeline/rename_field.yaml

@@ -0,0 +1,60 @@
+$schema: ../../../../../json_schemas/test_story.schema.yaml
+
+description: |-
+  Test the creation of a search pipeline with a response processor.
+version: '>= 2.8'
+prologues:
+  - path: /movies/_doc/1
+    method: POST
+    parameters:
+      refresh: true
+    request:
+      payload:
+        name: "Drive"
+    status: [201]
+epilogues:
+  - path: /_search/pipeline/names_pipeline
+    method: DELETE
+    status: [200, 404]
+  - path: /movies
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Create search pipeline.
+    path: /_search/pipeline/{id}
+    method: PUT
+    parameters:
+      id: names_pipeline
+    request:
+      payload:
+        response_processors:
+          - rename_field:
+              field: name
+              target_field: title
+    response:
+      status: 200
+      payload:
+        acknowledged: true
+  - synopsis: Query created pipeline.
+    path: /_search/pipeline/{id}
+    method: GET
+    parameters:
+      id: names_pipeline
+    response:
+      status: 200
+  - synopsis: Search.
+    path: /{index}/_search
+    method: GET
+    parameters:
+      index: movies
+      search_pipeline: names_pipeline
+    response:
+      status: 200
+      payload:
+        hits:
+          total:
+            value: 1
+          hits:
+            - _index: movies
+              _source:
+                title: Drive

tests/default/_core/search/pipeline/sort.yaml

@@ -0,0 +1,68 @@
+$schema: ../../../../../json_schemas/test_story.schema.yaml
+
+description: |-
+  Test the creation of a search pipeline with a response processor.
+version: '>= 2.16'
+prologues:
+  - path: /movies/_doc/1
+    method: POST
+    parameters:
+      refresh: true
+    request:
+      payload:
+        names: ["Drive", "1984", "Moneyball"]
+    status: [201]
+epilogues:
+  - path: /_search/pipeline/names_pipeline
+    method: DELETE
+    status: [200, 404]
+  - path: /movies
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Create search pipeline.
+    path: /_search/pipeline/{id}
+    method: PUT
+    parameters:
+      id: names_pipeline
+    request:
+      payload:
+        response_processors:
+          - sort:
+              field: names
+              order: asc
+              target_field: sorted_names
+    response:
+      status: 200
+      payload:
+        acknowledged: true
+  - synopsis: Query created pipeline.
+    path: /_search/pipeline/{id}
+    method: GET
+    parameters:
+      id: names_pipeline
+    response:
+      status: 200
+  - synopsis: Search.
+    path: /{index}/_search
+    method: GET
+    parameters:
+      index: movies
+      search_pipeline: names_pipeline
+    response:
+      status: 200
+      payload:
+        hits:
+          total:
+            value: 1
+          hits:
+            - _index: movies
+              _source:
+                names:
+                  - Drive
+                  - '1984'
+                  - Moneyball
+                sorted_names:
+                  - '1984'
+                  - Drive
+                  - Moneyball