From 5bdfe022ac228a2a434b2de092dfc8990f0b883a Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 08:11:43 -0700 Subject: [PATCH 1/7] Update API specifications for k-NN 2.17 changes Signed-off-by: Ryan Bogan --- spec/namespaces/knn.yaml | 16 +++- spec/schemas/_common.mapping.yaml | 8 ++ spec/schemas/_common.yaml | 10 +++ .../default/_core/search/knn_disk_based.yaml | 71 ++++++++++++++++++ tests/default/knn/train_model.yaml | 74 +++++++++++++++++++ 5 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 tests/default/_core/search/knn_disk_based.yaml create mode 100644 tests/default/knn/train_model.yaml diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index d960f34a5..b6f0af6b2 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -268,11 +268,14 @@ components: format: int32 description: type: string + mode: + type: string + compression_level: + type: string method: type: string required: - dimension - - method - training_field - training_index required: true @@ -281,7 +284,16 @@ components: knn.get_model@200: {} knn.search_models@200: {} knn.stats@200: {} - knn.train_model@200: {} + knn.train_model@200: + content: + application/json: + schema: + type: object + properties: + model_id: + type: string + required: + - model_id knn.warmup@200: {} parameters: knn.delete_model::path.model_id: diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index a26c15e61..9b6677ed0 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -1141,6 +1141,14 @@ components: properties: dimension: type: number + space_type: + type : string + data_type: + type : string + mode: + type: string + compression_level: + type: string method: $ref: '#/components/schemas/KnnVectorMethod' required: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 140600787..4de17eea8 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -656,6 +656,16 @@ components: boost: description: Boost value to apply to kNN scores type: number + method_parameters: + type: object + x-version-added: '2.16' + additionalProperties: + type: number + rescore: + type: object + x-version-added: '2.17' + additionalProperties: + type: number required: - vector QueryVector: diff --git a/tests/default/_core/search/knn_disk_based.yaml b/tests/default/_core/search/knn_disk_based.yaml new file mode 100644 index 000000000..5373979c3 --- /dev/null +++ b/tests/default/_core/search/knn_disk_based.yaml @@ -0,0 +1,71 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with knn query. +version: '>= 2.17' + +prologues: + - method: PUT + path: /my-vector-index + request: + payload: + settings: + index: + knn: true + mappings: + properties: + my_vector_field: + type: knn_vector + dimension: 8 + space_type: innerproduct + data_type: float + mode: on_disk + compression_level: 16x + status: [200] + - method: POST + path: /_bulk + request: + content_type: application/x-ndjson + payload: + - { "index": { "_index": "my-vector-index", "_id": "1" } } + - { "my_vector_field": [ 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "price": 12.2 } + - { "index": { "_index": "my-vector-index", "_id": "2" } } + - { "my_vector_field": [ 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5 ], "price": 7.1 } + - { "index": { "_index": "my-vector-index", "_id": "3" } } + - { "my_vector_field": [ 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5 ], "price": 12.9 } + - { "index": { "_index": "my-vector-index", "_id": "4" } } + - { "my_vector_field": [ 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5 ], "price": 1.2 } + - { "index": { "_index": "my-vector-index", "_id": "5" } } + - { "my_vector_field": [ 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5 ], "price": 3.7 } + - { "index": { "_index": "my-vector-index", "_id": "6" } } + - { "my_vector_field": [ 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5 ], "price": 10.3 } + - { "index": { "_index": "my-vector-index", "_id": "7" } } + - { "my_vector_field": [ 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5 ], "price": 5.5 } + - { "index": { "_index": "my-vector-index", "_id": "8" } } + - { "my_vector_field": [ 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5 ], "price": 4.4 } + - { "index": { "_index": "my-vector-index", "_id": "9" } } + - { "my_vector_field": [ 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5 ], "price": 8.9 } + status: [200] +epilogues: + - path: /my-vector-index + method: DELETE + status: [ 200, 404 ] + +chapters: + - synopsis: Test k-NN disk-based search. + method: POST + path: /{index}/_search + parameters: + index: my-vector-index + request: + payload: + query: + knn: + my_vector_field: + vector: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] + k: 5 + method_parameters: + ef_search: 512 + rescore: + oversample_factor: 10.0 + response: + status: 200 \ No newline at end of file diff --git a/tests/default/knn/train_model.yaml b/tests/default/knn/train_model.yaml new file mode 100644 index 000000000..a62236a78 --- /dev/null +++ b/tests/default/knn/train_model.yaml @@ -0,0 +1,74 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test training k-NN model with disk-based parameters. +version: '>= 2.17' + +prologues: + - method: PUT + path: /my-vector-index + request: + payload: + settings: + index: + knn: true + mappings: + properties: + my_vector_field: + type: knn_vector + dimension: 8 + space_type: innerproduct + data_type: float + status: [200] + - method: POST + path: /_bulk + request: + content_type: application/x-ndjson + payload: + - { "index": { "_index": "my-vector-index", "_id": "1" } } + - { "my_vector_field": [ 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "price": 12.2 } + - { "index": { "_index": "my-vector-index", "_id": "2" } } + - { "my_vector_field": [ 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5 ], "price": 7.1 } + - { "index": { "_index": "my-vector-index", "_id": "3" } } + - { "my_vector_field": [ 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5 ], "price": 12.9 } + - { "index": { "_index": "my-vector-index", "_id": "4" } } + - { "my_vector_field": [ 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5 ], "price": 1.2 } + - { "index": { "_index": "my-vector-index", "_id": "5" } } + - { "my_vector_field": [ 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5 ], "price": 3.7 } + - { "index": { "_index": "my-vector-index", "_id": "6" } } + - { "my_vector_field": [ 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5 ], "price": 10.3 } + - { "index": { "_index": "my-vector-index", "_id": "7" } } + - { "my_vector_field": [ 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5 ], "price": 5.5 } + - { "index": { "_index": "my-vector-index", "_id": "8" } } + - { "my_vector_field": [ 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5 ], "price": 4.4 } + - { "index": { "_index": "my-vector-index", "_id": "9" } } + - { "my_vector_field": [ 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5 ], "price": 8.9 } + status: [200] +epilogues: + - path: /my-vector-index + method: DELETE + status: [ 200, 404 ] + - path: /_plugins/_knn/models/{model_id} + parameters: + model_id: ${train_model.test_model_id} + method: DELETE + status: [ 200, 404 ] + +chapters: + - synopsis: Test training a model with disk-based parameters. + id: train_model + method: POST + path: /_plugins/_knn/models/_train + request: + payload: + training_index: my-vector-index + training_field: my_vector_field + dimension: 8 + max_training_vector_count: 1200 + search_size: 100 + description: Test model + mode: on_disk + compression_level: 32x + response: + status: 200 + output: + test_model_id: "payload.model_id" \ No newline at end of file From 8a319b374b285cccea8c7653b5efe73247030142 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 08:15:48 -0700 Subject: [PATCH 2/7] Add changelog entry Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0129e440f..d6ffe37d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,6 +158,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) - Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) +- Update API specifications for k-NN 2.17 changes ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) ### Security From 45cdf3d90b1f7288c55a4b5f5c82defa54e43f45 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 10:13:56 -0700 Subject: [PATCH 3/7] Refactor k-NN tests, change naming to match theme, and update changelog message Signed-off-by: Ryan Bogan --- .github/workflows/test-spec.yml | 2 - CHANGELOG.md | 2 +- spec/schemas/_common.mapping.yaml | 4 +- tests/default/_core/search/knn/on_disk.yaml | 71 +++++++++++++++++++ .../search/{knn.yaml => knn/search.yaml} | 2 +- .../default/_core/search/knn_disk_based.yaml | 71 ------------------- tests/default/knn/train_model.yaml | 54 +++++++------- 7 files changed, 101 insertions(+), 105 deletions(-) create mode 100644 tests/default/_core/search/knn/on_disk.yaml rename tests/default/_core/search/{knn.yaml => knn/search.yaml} (98%) delete mode 100644 tests/default/_core/search/knn_disk_based.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index a202207de..9624ff91d 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,8 +37,6 @@ jobs: - version: 2.16.0 tests: snapshot - version: 2.17.0 - hub: opensearchstaging - ref: '@sha256:1273489ebbedcb470ea13563dae4c6dc6b2ed431e87e686ed931ae0733034b25' - version: 3.0.0 hub: opensearchstaging ref: '@sha256:06af2ba4037f8423dc1a4ed3cd29108a1912774e7c659e73f0fac09e1bb2b63d' diff --git a/CHANGELOG.md b/CHANGELOG.md index d6ffe37d9..ae3e0c37d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,7 +158,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) - Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) -- Update API specifications for k-NN 2.17 changes ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) +- Add mode and compression to k-NN index creation and search, and add rescore and oversample_factor to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) ### Security diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 9b6677ed0..2c26487fb 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -1142,9 +1142,9 @@ components: dimension: type: number space_type: - type : string + type: string data_type: - type : string + type: string mode: type: string compression_level: diff --git a/tests/default/_core/search/knn/on_disk.yaml b/tests/default/_core/search/knn/on_disk.yaml new file mode 100644 index 000000000..ac42d42f4 --- /dev/null +++ b/tests/default/_core/search/knn/on_disk.yaml @@ -0,0 +1,71 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with knn query. +version: '>= 2.17' + +prologues: + - method: PUT + path: /movies + request: + payload: + settings: + index: + knn: true + mappings: + properties: + recommendation_vector: + type: knn_vector + dimension: 8 + space_type: l2 + data_type: float + mode: on_disk + compression_level: 16x + status: [200] + - method: POST + path: /_bulk + request: + content_type: application/x-ndjson + payload: + - {index: {_index: movies, _id: '1'}} + - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} + - {index: {_index: movies, _id: '2'}} + - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} + - {index: {_index: movies, _id: '3'}} + - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 12.9} + - {index: {_index: movies, _id: '4'}} + - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 1.2} + - {index: {_index: movies, _id: '5'}} + - {recommendation_vector: [5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5], duration: 3.7} + - {index: {_index: movies, _id: '6'}} + - {recommendation_vector: [6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5], duration: 10.3} + - {index: {_index: movies, _id: '7'}} + - {recommendation_vector: [7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5], duration: 5.5} + - {index: {_index: movies, _id: '8'}} + - {recommendation_vector: [8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5], duration: 4.4} + - {index: {_index: movies, _id: '9'}} + - {recommendation_vector: [9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5], duration: 8.9} + status: [200] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Test k-NN disk-based search. + method: POST + path: /{index}/_search + parameters: + index: movies + request: + payload: + query: + knn: + recommendation_vector: + vector: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] + k: 5 + method_parameters: + ef_search: 512 + rescore: + oversample_factor: 10 + response: + status: 200 \ No newline at end of file diff --git a/tests/default/_core/search/knn.yaml b/tests/default/_core/search/knn/search.yaml similarity index 98% rename from tests/default/_core/search/knn.yaml rename to tests/default/_core/search/knn/search.yaml index 6fcfb12e6..ae4daedeb 100644 --- a/tests/default/_core/search/knn.yaml +++ b/tests/default/_core/search/knn/search.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with knn query. version: '>= 1.2' diff --git a/tests/default/_core/search/knn_disk_based.yaml b/tests/default/_core/search/knn_disk_based.yaml deleted file mode 100644 index 5373979c3..000000000 --- a/tests/default/_core/search/knn_disk_based.yaml +++ /dev/null @@ -1,71 +0,0 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml - -description: Test search endpoint with knn query. -version: '>= 2.17' - -prologues: - - method: PUT - path: /my-vector-index - request: - payload: - settings: - index: - knn: true - mappings: - properties: - my_vector_field: - type: knn_vector - dimension: 8 - space_type: innerproduct - data_type: float - mode: on_disk - compression_level: 16x - status: [200] - - method: POST - path: /_bulk - request: - content_type: application/x-ndjson - payload: - - { "index": { "_index": "my-vector-index", "_id": "1" } } - - { "my_vector_field": [ 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "price": 12.2 } - - { "index": { "_index": "my-vector-index", "_id": "2" } } - - { "my_vector_field": [ 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5 ], "price": 7.1 } - - { "index": { "_index": "my-vector-index", "_id": "3" } } - - { "my_vector_field": [ 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5 ], "price": 12.9 } - - { "index": { "_index": "my-vector-index", "_id": "4" } } - - { "my_vector_field": [ 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5 ], "price": 1.2 } - - { "index": { "_index": "my-vector-index", "_id": "5" } } - - { "my_vector_field": [ 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5 ], "price": 3.7 } - - { "index": { "_index": "my-vector-index", "_id": "6" } } - - { "my_vector_field": [ 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5 ], "price": 10.3 } - - { "index": { "_index": "my-vector-index", "_id": "7" } } - - { "my_vector_field": [ 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5 ], "price": 5.5 } - - { "index": { "_index": "my-vector-index", "_id": "8" } } - - { "my_vector_field": [ 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5 ], "price": 4.4 } - - { "index": { "_index": "my-vector-index", "_id": "9" } } - - { "my_vector_field": [ 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5 ], "price": 8.9 } - status: [200] -epilogues: - - path: /my-vector-index - method: DELETE - status: [ 200, 404 ] - -chapters: - - synopsis: Test k-NN disk-based search. - method: POST - path: /{index}/_search - parameters: - index: my-vector-index - request: - payload: - query: - knn: - my_vector_field: - vector: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] - k: 5 - method_parameters: - ef_search: 512 - rescore: - oversample_factor: 10.0 - response: - status: 200 \ No newline at end of file diff --git a/tests/default/knn/train_model.yaml b/tests/default/knn/train_model.yaml index a62236a78..3e37c6a07 100644 --- a/tests/default/knn/train_model.yaml +++ b/tests/default/knn/train_model.yaml @@ -5,7 +5,7 @@ version: '>= 2.17' prologues: - method: PUT - path: /my-vector-index + path: /movies request: payload: settings: @@ -13,45 +13,43 @@ prologues: knn: true mappings: properties: - my_vector_field: + recommendation_vector: type: knn_vector dimension: 8 - space_type: innerproduct - data_type: float status: [200] - method: POST path: /_bulk request: content_type: application/x-ndjson payload: - - { "index": { "_index": "my-vector-index", "_id": "1" } } - - { "my_vector_field": [ 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "price": 12.2 } - - { "index": { "_index": "my-vector-index", "_id": "2" } } - - { "my_vector_field": [ 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5 ], "price": 7.1 } - - { "index": { "_index": "my-vector-index", "_id": "3" } } - - { "my_vector_field": [ 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5 ], "price": 12.9 } - - { "index": { "_index": "my-vector-index", "_id": "4" } } - - { "my_vector_field": [ 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5 ], "price": 1.2 } - - { "index": { "_index": "my-vector-index", "_id": "5" } } - - { "my_vector_field": [ 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5 ], "price": 3.7 } - - { "index": { "_index": "my-vector-index", "_id": "6" } } - - { "my_vector_field": [ 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5 ], "price": 10.3 } - - { "index": { "_index": "my-vector-index", "_id": "7" } } - - { "my_vector_field": [ 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5 ], "price": 5.5 } - - { "index": { "_index": "my-vector-index", "_id": "8" } } - - { "my_vector_field": [ 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5 ], "price": 4.4 } - - { "index": { "_index": "my-vector-index", "_id": "9" } } - - { "my_vector_field": [ 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5 ], "price": 8.9 } + - {index: {_index: movies, _id: '1'}} + - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} + - {index: {_index: movies, _id: '2'}} + - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} + - {index: {_index: movies, _id: '3'}} + - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 12.9} + - {index: {_index: movies, _id: '4'}} + - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 1.2} + - {index: {_index: movies, _id: '5'}} + - {recommendation_vector: [5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5], duration: 3.7} + - {index: {_index: movies, _id: '6'}} + - {recommendation_vector: [6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5], duration: 10.3} + - {index: {_index: movies, _id: '7'}} + - {recommendation_vector: [7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5], duration: 5.5} + - {index: {_index: movies, _id: '8'}} + - {recommendation_vector: [8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5], duration: 4.4} + - {index: {_index: movies, _id: '9'}} + - {recommendation_vector: [9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5], duration: 8.9} status: [200] epilogues: - - path: /my-vector-index + - path: /movies method: DELETE - status: [ 200, 404 ] + status: [200, 404] - path: /_plugins/_knn/models/{model_id} parameters: model_id: ${train_model.test_model_id} method: DELETE - status: [ 200, 404 ] + status: [200, 404] chapters: - synopsis: Test training a model with disk-based parameters. @@ -60,8 +58,8 @@ chapters: path: /_plugins/_knn/models/_train request: payload: - training_index: my-vector-index - training_field: my_vector_field + training_index: movies + training_field: recommendation_vector dimension: 8 max_training_vector_count: 1200 search_size: 100 @@ -71,4 +69,4 @@ chapters: response: status: 200 output: - test_model_id: "payload.model_id" \ No newline at end of file + test_model_id: payload.model_id \ No newline at end of file From c7ddc45b8bb90b3eabe65d0318d6980c282947da Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 10:51:33 -0700 Subject: [PATCH 4/7] Add spaceType parameter to train model API Signed-off-by: Ryan Bogan --- spec/namespaces/knn.yaml | 2 ++ tests/default/knn/train_model.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index b6f0af6b2..c8cbe4ff3 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -274,6 +274,8 @@ components: type: string method: type: string + spaceType: + type: string required: - dimension - training_field diff --git a/tests/default/knn/train_model.yaml b/tests/default/knn/train_model.yaml index 3e37c6a07..b45ac5c6f 100644 --- a/tests/default/knn/train_model.yaml +++ b/tests/default/knn/train_model.yaml @@ -66,6 +66,7 @@ chapters: description: Test model mode: on_disk compression_level: 32x + spaceType: l2 response: status: 200 output: From 67c2c1f32b54059bc9f87e52e8d83f6c93a35a6c Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 12:08:35 -0700 Subject: [PATCH 5/7] Update sha references and remove version checks for asynchronous search Signed-off-by: Ryan Bogan --- .github/workflows/test-spec.yml | 5 ++++- tests/default/asynchronous_search/search.yaml | 3 --- tests/default/asynchronous_search/stats.yaml | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 9624ff91d..8f15414e3 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,9 +37,12 @@ jobs: - version: 2.16.0 tests: snapshot - version: 2.17.0 + - version: 2.18.0 + hub: opensearchstaging + ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:06af2ba4037f8423dc1a4ed3cd29108a1912774e7c659e73f0fac09e1bb2b63d' + ref: '@sha256:cf07c0ffa7d9e8a3e7fdb58041caae3bb81f1123260431b99d0ebf4a52c3d9a3' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/tests/default/asynchronous_search/search.yaml b/tests/default/asynchronous_search/search.yaml index 23e22d57c..8024f38c1 100644 --- a/tests/default/asynchronous_search/search.yaml +++ b/tests/default/asynchronous_search/search.yaml @@ -27,7 +27,6 @@ chapters: synopsis: Asynchronous Search. path: /_plugins/_asynchronous_search method: POST - version: < 3.0 parameters: index: books keep_on_completion: true @@ -44,7 +43,6 @@ chapters: - synopsis: Get partial response from asynchronous search. path: /_plugins/_asynchronous_search/{id} method: GET - version: < 3.0 parameters: id: ${async_search.id} response: @@ -52,7 +50,6 @@ chapters: - synopsis: Delete partial response from asynchronous search. path: /_plugins/_asynchronous_search/{id} method: DELETE - version: < 3.0 parameters: id: ${async_search.id} response: diff --git a/tests/default/asynchronous_search/stats.yaml b/tests/default/asynchronous_search/stats.yaml index 7854b156e..b5f113006 100644 --- a/tests/default/asynchronous_search/stats.yaml +++ b/tests/default/asynchronous_search/stats.yaml @@ -6,6 +6,5 @@ chapters: - synopsis: Get stats. path: /_plugins/_asynchronous_search/stats method: GET - version: < 3.0 response: status: 200 From 8453596b21851940077b7b6d09a5888e494d3c4d Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 12:12:17 -0700 Subject: [PATCH 6/7] Add version checks back for asynchronous search Signed-off-by: Ryan Bogan --- tests/default/asynchronous_search/search.yaml | 3 +++ tests/default/asynchronous_search/stats.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/default/asynchronous_search/search.yaml b/tests/default/asynchronous_search/search.yaml index 8024f38c1..23e22d57c 100644 --- a/tests/default/asynchronous_search/search.yaml +++ b/tests/default/asynchronous_search/search.yaml @@ -27,6 +27,7 @@ chapters: synopsis: Asynchronous Search. path: /_plugins/_asynchronous_search method: POST + version: < 3.0 parameters: index: books keep_on_completion: true @@ -43,6 +44,7 @@ chapters: - synopsis: Get partial response from asynchronous search. path: /_plugins/_asynchronous_search/{id} method: GET + version: < 3.0 parameters: id: ${async_search.id} response: @@ -50,6 +52,7 @@ chapters: - synopsis: Delete partial response from asynchronous search. path: /_plugins/_asynchronous_search/{id} method: DELETE + version: < 3.0 parameters: id: ${async_search.id} response: diff --git a/tests/default/asynchronous_search/stats.yaml b/tests/default/asynchronous_search/stats.yaml index b5f113006..7854b156e 100644 --- a/tests/default/asynchronous_search/stats.yaml +++ b/tests/default/asynchronous_search/stats.yaml @@ -6,5 +6,6 @@ chapters: - synopsis: Get stats. path: /_plugins/_asynchronous_search/stats method: GET + version: < 3.0 response: status: 200 From 6f8c43d0779ce9a89285ecfb8d8cdeb70d8e1598 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 12:13:37 -0700 Subject: [PATCH 7/7] Update changelog style for entry Signed-off-by: Ryan Bogan --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae3e0c37d..4facf49a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,7 +158,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) - Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) -- Add mode and compression to k-NN index creation and search, and add rescore and oversample_factor to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) +- Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) ### Security