From 4ef4b7dfa6e3f9846b79b777c5346a770d624334 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Mon, 25 Mar 2024 02:41:08 +0500 Subject: [PATCH] Search pipelines added (#172) * Search pipelines added Signed-off-by: Tokesh * search_pipeline folder added Signed-off-by: Tokesh * small fix of operation group in create method Signed-off-by: Tokesh * rename search pipeline folders Signed-off-by: Tokesh * renaming search pipeline to resolve conflict with main branch Signed-off-by: Tokesh * adding map for main structure, main search pipeline API spces, phase result processor Signed-off-by: Tokesh * Update model/search_pipeline/search_pipeline.smithy Co-authored-by: Thomas Farr Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> * spotless Apply Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> --------- Signed-off-by: Tokesh Signed-off-by: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Co-authored-by: Thomas Farr --- model/opensearch.smithy | 2 + .../search_pipeline/create/operations.smithy | 23 ++ .../search_pipeline/create/structures.smithy | 24 +++ model/search_pipeline/get/operations.smithy | 34 +++ model/search_pipeline/get/structures.smithy | 31 +++ model/search_pipeline/search_pipeline.smithy | 203 ++++++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 model/search_pipeline/create/operations.smithy create mode 100644 model/search_pipeline/create/structures.smithy create mode 100644 model/search_pipeline/get/operations.smithy create mode 100644 model/search_pipeline/get/structures.smithy create mode 100644 model/search_pipeline/search_pipeline.smithy diff --git a/model/opensearch.smithy b/model/opensearch.smithy index 94b08cdb3..09126b23b 100644 --- a/model/opensearch.smithy +++ b/model/opensearch.smithy @@ -101,6 +101,7 @@ service OpenSearch { CreateUser, Create_Post, Create_Put, + CreateSearchPipeline, DanglingIndicesDeleteDanglingIndex, DanglingIndicesImportDanglingIndex, DanglingIndicesListDanglingIndices, @@ -143,6 +144,7 @@ service OpenSearch { GetRoles, GetRoleMapping, GetRoleMappings, + GetSearchPipeline, GetTenant, GetTenants, GetUser, diff --git a/model/search_pipeline/create/operations.smithy b/model/search_pipeline/create/operations.smithy new file mode 100644 index 000000000..13d7c5762 --- /dev/null +++ b/model/search_pipeline/create/operations.smithy @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@externalDocumentation( + "API Reference": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/creating-search-pipeline/" +) + +@xOperationGroup("search_pipeline.create") +@xVersionAdded("2.9") +@idempotent +@suppress(["HttpUriConflict"]) +@http(method: "PUT", uri: "/_search/pipeline/{pipeline}") +@documentation("Creates or replaces the specified search pipeline.") +operation CreateSearchPipeline { + input: CreateSearchPipeline_Input, + output: CreateSearchPipeline_Output +} diff --git a/model/search_pipeline/create/structures.smithy b/model/search_pipeline/create/structures.smithy new file mode 100644 index 000000000..69c89c77d --- /dev/null +++ b/model/search_pipeline/create/structures.smithy @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + + +@input +structure CreateSearchPipeline_Input{ + @required + @httpLabel + pipeline: String + @required + @httpPayload + content: SearchPipelineStructure +} + +@output +structure CreateSearchPipeline_Output { + acknowledged: Boolean +} diff --git a/model/search_pipeline/get/operations.smithy b/model/search_pipeline/get/operations.smithy new file mode 100644 index 000000000..5029c140a --- /dev/null +++ b/model/search_pipeline/get/operations.smithy @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@externalDocumentation( + "API Reference": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/" +) + +@xOperationGroup("search_pipeline.get") +@xVersionAdded("2.9") +@readonly +@suppress(["HttpUriConflict"]) +@http(method: "GET", uri: "/_search/pipeline") +@documentation("Retrieves information about search pipelines.") +operation GetSearchPipelines { + input: GetSearchPipelines_Input, + output: GetSearchPipelines_Output +} + +@xOperationGroup("search_pipeline.get") +@xVersionAdded("2.9") +@readonly +@suppress(["HttpUriConflict"]) +@http(method: "GET", uri: "/_search/pipeline/{pipeline}") +@documentation("Retrieves information about a specified search pipeline.") +operation GetSearchPipeline { + input: GetSearchPipeline_Input, + output: GetSearchPipeline_Output +} diff --git a/model/search_pipeline/get/structures.smithy b/model/search_pipeline/get/structures.smithy new file mode 100644 index 000000000..ab90164c7 --- /dev/null +++ b/model/search_pipeline/get/structures.smithy @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +@input +structure GetSearchPipelines_Input { +} + +@output +structure GetSearchPipelines_Output { + @httpPayload + content: SearchPipelineMap +} + +@input +structure GetSearchPipeline_Input { + @required + @httpLabel + pipeline: String, +} + +@output +structure GetSearchPipeline_Output { + @httpPayload + content: SearchPipelineMap +} diff --git a/model/search_pipeline/search_pipeline.smithy b/model/search_pipeline/search_pipeline.smithy new file mode 100644 index 000000000..7cfa4568e --- /dev/null +++ b/model/search_pipeline/search_pipeline.smithy @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. + +$version: "2" +namespace OpenSearch + +map SearchPipelineMap { + key: String + value: SearchPipelineStructure +} + +structure SearchPipelineStructure { + version: Integer + request_processors: RequestProcessorList + response_processors: ResponseProcessorList + phase_results_processors: PhaseResultsProcessorList +} + +// === Common Properties === + +@mixin +structure ProcessorBase { + tag: String + description: String +} + +@mixin +structure IgnoreableProcessorBase with [ProcessorBase] { + ignore_failure: Boolean +} + +// === Request Processors === + +list RequestProcessorList { + member: RequestProcessor +} + +union RequestProcessor { + filter_query: FilterQueryRequestProcessor + neural_query_enricher: NeuralQueryEnricherRequestProcessor + script: SearchScriptRequestProcessor + oversample: OversampleRequestProcessor +} + +structure FilterQueryRequestProcessor with [IgnoreableProcessorBase] { + query: UserDefinedObjectStructure +} + +structure NeuralQueryEnricherRequestProcessor with [ProcessorBase] { + default_model_id: String + neural_field_default_id: NeuralFieldMap +} + +map NeuralFieldMap { + key: String + value: String +} + +structure SearchScriptRequestProcessor with [IgnoreableProcessorBase] { + @required + source: String + + lang: String +} + +structure OversampleRequestProcessor with [IgnoreableProcessorBase] { + @required + sample_factor: Float + + content_prefix: String +} + +// === Response Processors === + +list ResponseProcessorList { + member: RequestProcessor +} + +union ResponseProcessor { + personalize_search_ranking: PersonalizeSearchRankingResponseProcessor + retrieval_augmented_generation: RetrievalAugmentedGenerationResponseProcessor + rename_field: RenameFieldResponseProcessor + rerank: RerankResponseProcessor + collapse: CollapseResponseProcessor + truncate_hits: TruncateHitsResponseProcessor +} + +structure PersonalizeSearchRankingResponseProcessor with [IgnoreableProcessorBase] { + @required + campaign_arn: String + + @required + recipe: String + + @required + weight: Float + + item_id_field: String + + iam_role_arn: String +} + +structure RetrievalAugmentedGenerationResponseProcessor with [ProcessorBase] { + @required + model_id: String + + @required + context_field_list: ContextFieldList + + system_prompt: String + + user_instructions: String +} + +list ContextFieldList { + member: String +} + +structure RenameFieldResponseProcessor with [IgnoreableProcessorBase] { + @required + field: String + + @required + target_field: String +} + +structure RerankResponseProcessor with [IgnoreableProcessorBase] { + ml_opensearch: MLOpenSearchReranker + context: RerankContext +} + +structure MLOpenSearchReranker { + @required + model_id: String +} + +structure RerankContext { + @required + document_fields: DocumentFieldList +} + +list DocumentFieldList { + member: String +} + +structure CollapseResponseProcessor with [IgnoreableProcessorBase] { + @required + field: String + + context_prefix: String +} + +structure TruncateHitsResponseProcessor with [IgnoreableProcessorBase] { + target_size: Integer + context_prefix: String +} + +// === Phase Results Processors === + +list PhaseResultsProcessorList { + member: PhaseResultsProcessor +} + +union PhaseResultsProcessor { + @jsonName("normalization-processor") + normalization_processor: NormalizationPhaseResultsProcessor +} + +structure NormalizationPhaseResultsProcessor with [IgnoreableProcessorBase] { + normalization: ScoreNormalization + combination: ScoreCombination +} + +structure ScoreNormalization { + technique: ScoreNormalizationTechnique +} + +enum ScoreNormalizationTechnique { + MIN_MAX = "min_max" + L2 = "l2" +} + +structure ScoreCombination { + technique: ScoreCombinationTechnique + parameters: ScoreWeights +} + +enum ScoreCombinationTechnique { + ARITHMETIC_MEAN = "arithmetic_mean" + GEOMETRIC_MEAN = "geometric_mean" + HARMONIC_MEAN = "harmonic_mean" +} + +structure ScoreCombinationParameters { + weights: ScoreWeights +} + +list ScoreWeights { + member: Float +}