Skip to content

Commit

Permalink
Search pipelines added (opensearch-project#172)
Browse files Browse the repository at this point in the history
* Search pipelines added

Signed-off-by: Tokesh <tokesh789@gmail.com>

* search_pipeline folder added

Signed-off-by: Tokesh <tokesh789@gmail.com>

* small fix of operation group in create method

Signed-off-by: Tokesh <tokesh789@gmail.com>

* rename search pipeline folders

Signed-off-by: Tokesh <tokesh789@gmail.com>

* renaming search pipeline to resolve conflict with main branch

Signed-off-by: Tokesh <tokesh789@gmail.com>

* adding map for main structure, main search pipeline API spces, phase result processor

Signed-off-by: Tokesh <tokesh789@gmail.com>

* Update model/search_pipeline/search_pipeline.smithy

Co-authored-by: Thomas Farr <xtansia@xtansia.com>
Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com>

* spotless Apply

Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com>

---------

Signed-off-by: Tokesh <tokesh789@gmail.com>
Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com>
Co-authored-by: Thomas Farr <xtansia@xtansia.com>
  • Loading branch information
Tokesh and Xtansia authored Mar 24, 2024
1 parent a44e6d9 commit 4ef4b7d
Show file tree
Hide file tree
Showing 6 changed files with 317 additions and 0 deletions.
2 changes: 2 additions & 0 deletions model/opensearch.smithy
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ service OpenSearch {
CreateUser,
Create_Post,
Create_Put,
CreateSearchPipeline,
DanglingIndicesDeleteDanglingIndex,
DanglingIndicesImportDanglingIndex,
DanglingIndicesListDanglingIndices,
Expand Down Expand Up @@ -143,6 +144,7 @@ service OpenSearch {
GetRoles,
GetRoleMapping,
GetRoleMappings,
GetSearchPipeline,
GetTenant,
GetTenants,
GetUser,
Expand Down
23 changes: 23 additions & 0 deletions model/search_pipeline/create/operations.smithy
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// SPDX-License-Identifier: Apache-2.0
//
// The OpenSearch Contributors require contributions made to
// this file be licensed under the Apache-2.0 license or a
// compatible open source license.

$version: "2"
namespace OpenSearch

@externalDocumentation(
"API Reference": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/creating-search-pipeline/"
)

@xOperationGroup("search_pipeline.create")
@xVersionAdded("2.9")
@idempotent
@suppress(["HttpUriConflict"])
@http(method: "PUT", uri: "/_search/pipeline/{pipeline}")
@documentation("Creates or replaces the specified search pipeline.")
operation CreateSearchPipeline {
input: CreateSearchPipeline_Input,
output: CreateSearchPipeline_Output
}
24 changes: 24 additions & 0 deletions model/search_pipeline/create/structures.smithy
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: Apache-2.0
//
// The OpenSearch Contributors require contributions made to
// this file be licensed under the Apache-2.0 license or a
// compatible open source license.

$version: "2"
namespace OpenSearch


@input
structure CreateSearchPipeline_Input{
@required
@httpLabel
pipeline: String
@required
@httpPayload
content: SearchPipelineStructure
}

@output
structure CreateSearchPipeline_Output {
acknowledged: Boolean
}
34 changes: 34 additions & 0 deletions model/search_pipeline/get/operations.smithy
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// SPDX-License-Identifier: Apache-2.0
//
// The OpenSearch Contributors require contributions made to
// this file be licensed under the Apache-2.0 license or a
// compatible open source license.

$version: "2"
namespace OpenSearch

@externalDocumentation(
"API Reference": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/"
)

@xOperationGroup("search_pipeline.get")
@xVersionAdded("2.9")
@readonly
@suppress(["HttpUriConflict"])
@http(method: "GET", uri: "/_search/pipeline")
@documentation("Retrieves information about search pipelines.")
operation GetSearchPipelines {
input: GetSearchPipelines_Input,
output: GetSearchPipelines_Output
}

@xOperationGroup("search_pipeline.get")
@xVersionAdded("2.9")
@readonly
@suppress(["HttpUriConflict"])
@http(method: "GET", uri: "/_search/pipeline/{pipeline}")
@documentation("Retrieves information about a specified search pipeline.")
operation GetSearchPipeline {
input: GetSearchPipeline_Input,
output: GetSearchPipeline_Output
}
31 changes: 31 additions & 0 deletions model/search_pipeline/get/structures.smithy
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// SPDX-License-Identifier: Apache-2.0
//
// The OpenSearch Contributors require contributions made to
// this file be licensed under the Apache-2.0 license or a
// compatible open source license.

$version: "2"
namespace OpenSearch

@input
structure GetSearchPipelines_Input {
}

@output
structure GetSearchPipelines_Output {
@httpPayload
content: SearchPipelineMap
}

@input
structure GetSearchPipeline_Input {
@required
@httpLabel
pipeline: String,
}

@output
structure GetSearchPipeline_Output {
@httpPayload
content: SearchPipelineMap
}
203 changes: 203 additions & 0 deletions model/search_pipeline/search_pipeline.smithy
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
// SPDX-License-Identifier: Apache-2.0
//
// The OpenSearch Contributors require contributions made to
// this file be licensed under the Apache-2.0 license or a
// compatible open source license.

$version: "2"
namespace OpenSearch

map SearchPipelineMap {
key: String
value: SearchPipelineStructure
}

structure SearchPipelineStructure {
version: Integer
request_processors: RequestProcessorList
response_processors: ResponseProcessorList
phase_results_processors: PhaseResultsProcessorList
}

// === Common Properties ===

@mixin
structure ProcessorBase {
tag: String
description: String
}

@mixin
structure IgnoreableProcessorBase with [ProcessorBase] {
ignore_failure: Boolean
}

// === Request Processors ===

list RequestProcessorList {
member: RequestProcessor
}

union RequestProcessor {
filter_query: FilterQueryRequestProcessor
neural_query_enricher: NeuralQueryEnricherRequestProcessor
script: SearchScriptRequestProcessor
oversample: OversampleRequestProcessor
}

structure FilterQueryRequestProcessor with [IgnoreableProcessorBase] {
query: UserDefinedObjectStructure
}

structure NeuralQueryEnricherRequestProcessor with [ProcessorBase] {
default_model_id: String
neural_field_default_id: NeuralFieldMap
}

map NeuralFieldMap {
key: String
value: String
}

structure SearchScriptRequestProcessor with [IgnoreableProcessorBase] {
@required
source: String

lang: String
}

structure OversampleRequestProcessor with [IgnoreableProcessorBase] {
@required
sample_factor: Float

content_prefix: String
}

// === Response Processors ===

list ResponseProcessorList {
member: RequestProcessor
}

union ResponseProcessor {
personalize_search_ranking: PersonalizeSearchRankingResponseProcessor
retrieval_augmented_generation: RetrievalAugmentedGenerationResponseProcessor
rename_field: RenameFieldResponseProcessor
rerank: RerankResponseProcessor
collapse: CollapseResponseProcessor
truncate_hits: TruncateHitsResponseProcessor
}

structure PersonalizeSearchRankingResponseProcessor with [IgnoreableProcessorBase] {
@required
campaign_arn: String

@required
recipe: String

@required
weight: Float

item_id_field: String

iam_role_arn: String
}

structure RetrievalAugmentedGenerationResponseProcessor with [ProcessorBase] {
@required
model_id: String

@required
context_field_list: ContextFieldList

system_prompt: String

user_instructions: String
}

list ContextFieldList {
member: String
}

structure RenameFieldResponseProcessor with [IgnoreableProcessorBase] {
@required
field: String

@required
target_field: String
}

structure RerankResponseProcessor with [IgnoreableProcessorBase] {
ml_opensearch: MLOpenSearchReranker
context: RerankContext
}

structure MLOpenSearchReranker {
@required
model_id: String
}

structure RerankContext {
@required
document_fields: DocumentFieldList
}

list DocumentFieldList {
member: String
}

structure CollapseResponseProcessor with [IgnoreableProcessorBase] {
@required
field: String

context_prefix: String
}

structure TruncateHitsResponseProcessor with [IgnoreableProcessorBase] {
target_size: Integer
context_prefix: String
}

// === Phase Results Processors ===

list PhaseResultsProcessorList {
member: PhaseResultsProcessor
}

union PhaseResultsProcessor {
@jsonName("normalization-processor")
normalization_processor: NormalizationPhaseResultsProcessor
}

structure NormalizationPhaseResultsProcessor with [IgnoreableProcessorBase] {
normalization: ScoreNormalization
combination: ScoreCombination
}

structure ScoreNormalization {
technique: ScoreNormalizationTechnique
}

enum ScoreNormalizationTechnique {
MIN_MAX = "min_max"
L2 = "l2"
}

structure ScoreCombination {
technique: ScoreCombinationTechnique
parameters: ScoreWeights
}

enum ScoreCombinationTechnique {
ARITHMETIC_MEAN = "arithmetic_mean"
GEOMETRIC_MEAN = "geometric_mean"
HARMONIC_MEAN = "harmonic_mean"
}

structure ScoreCombinationParameters {
weights: ScoreWeights
}

list ScoreWeights {
member: Float
}

0 comments on commit 4ef4b7d

Please sign in to comment.