Added support for testing multiple distributions.

Signed-off-by: dblock <dblock@amazon.com>
opensearch-project · Aug 8, 2024 · 6def08d · 6def08d
1 parent e377553
commit 6def08d
Show file tree

Hide file tree

Showing 34 changed files with 258 additions and 88 deletions.
diff --git a/.cspell b/.cspell
@@ -130,7 +130,6 @@ readingform
 rebalance
 Rebalance
 recoverysource
-Refn
 reindex
 Reindex
 relo

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -67,6 +67,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465))
 - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471))
 - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476))
+- Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483))
 
 ### Changed
 

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -146,6 +146,7 @@ This repository includes several OpenAPI Specification Extensions to fill in any
 - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them.
 - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml).
 - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema.
+- `x-distributions`: Contains a list of distributions known to include the API. Use `opensearch.org` for the official distribution, `aos` for Amazon Managed OpenSearch, and `aoss` for Amazon OpenSearch Serverless. 
 
 ## Writing Spec Tests
 

diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md
@@ -10,6 +10,7 @@
     - [Simple Test Story](#simple-test-story)
     - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
     - [Managing Versions](#managing-versions)
+    - [Managing Distributions](#managing-distributions)
     - [Waiting for Tasks](#waiting-for-tasks)
     - [Warnings](#warnings)
       - [multiple-paths-detected](#multiple-paths-detected)
@@ -205,7 +206,38 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne
     status: 200
 ```
 
-The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests.
+The test tool will fetch the server version when it starts and use it automatically. The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests.
+
+### Managing Distributions
+
+OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not.
+
+```yaml
+/:
+  get:
+    operationId: info.0
+    x-distributions:
+      - opensearch.org
+      - aos
+    description: Returns basic information about the cluster.
+```
+
+Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it.
+
+```yaml
+description: Test root endpoint.
+distributions:
+  - opensearch.org
+  - aos
+chapters:
+  - synopsis: Get server info.
+    path: /
+    method: GET
+    response:
+      status: 200
+```
+
+To test a particular distribution pass `--opensearch-distribution` to the test tool.
 
 ### Waiting for Tasks
 

diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml
@@ -21,6 +21,8 @@ properties:
       $ref: '#/definitions/Chapter'
   version:
     $ref: '#/definitions/Version'
+  distributions:
+    $ref: '#/definitions/Distributions'
 required: [chapters,description]
 additionalProperties: false
 
@@ -85,6 +87,8 @@ definitions:
         $ref: '#/definitions/Output'
       version:
         $ref: '#/definitions/Version'
+      distributions:
+        $ref: '#/definitions/Distributions'
       retry:
         $ref: '#/definitions/Retry'
     required: [method, path]
@@ -106,6 +110,13 @@ definitions:
       The semver range to execute the story or chapter against.
     type: string
 
+  Distributions:
+    description: |
+      The list of distributions that support this API.
+    type: array
+    items:
+      type: string
+
   Retry:
     description: |
       Number of times to retry on error.

diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml
@@ -9,6 +9,9 @@ paths:
       operationId: info.0
       x-operation-group: info
       x-version-added: '1.0'
+      x-distributions:
+        - aos
+        - opensearch.org
       description: Returns basic information about the cluster.
       externalDocs:
         url: https://opensearch.org/docs/latest

diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml
@@ -562,6 +562,14 @@ components:
         write_operations:
           description: The total number of write operations for the device completed since starting OpenSearch.
           type: number
+        read_time:
+          type: number
+        write_time:
+          type: number
+        queue_size:
+          type: number
+        io_time_in_millis:
+          $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis'
     Jvm:
       type: object
       properties:
@@ -1068,8 +1076,10 @@ components:
           type: boolean
         enforced:
           type: boolean
+        total_rejections_breakup:
+          $ref: '#/components/schemas/TotalRejectionsBreakup'
         total_rejections_breakup_shadow_mode:
-          $ref: '#/components/schemas/TotalRejectionsBreakupShadowMode'
+          $ref: '#/components/schemas/TotalRejectionsBreakup'
     ShardSearchBackpressureStats:
       type: object
       properties:
@@ -1140,7 +1150,11 @@ components:
           type: number
         cancellation_limit_reached_count:
           type: number
-    TotalRejectionsBreakupShadowMode:
+        cancelled_task_percentage:
+          type: number
+        current_cancellation_eligible_tasks_count:
+          type: number
+    TotalRejectionsBreakup:
       type: object
       properties:
         node_limits:

diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml
@@ -102,14 +102,10 @@ components:
         search_pipelines:
           $ref: '#/components/schemas/NodeInfoSearchPipelines'
       required:
-        - attributes
         - build_hash
         - build_type
-        - host
-        - ip
         - name
         - roles
-        - transport_address
         - version
     NodeInfoHttp:
       type: object
@@ -156,7 +152,7 @@ components:
         bundled_jdk:
           type: boolean
         using_bundled_jdk:
-          type: boolean
+          type: [boolean, 'null']
         using_compressed_ordinary_object_pointers:
           oneOf:
             - type: boolean
@@ -167,16 +163,9 @@ components:
             type: string
       required:
         - bundled_jdk
-        - gc_collectors
-        - input_arguments
         - mem
-        - memory_pools
         - pid
         - start_time_in_millis
-        - version
-        - vm_name
-        - vm_vendor
-        - vm_version
     NodeInfoJvmMemory:
       type: object
       properties:
@@ -256,12 +245,8 @@ components:
         swap:
           $ref: '#/components/schemas/NodeInfoMemory'
       required:
-        - arch
         - available_processors
-        - name
-        - pretty_name
         - refresh_interval_in_millis
-        - version
     NodeInfoOSCPU:
       type: object
       properties:

diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml
@@ -2,6 +2,9 @@
 $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test root endpoint.
+distributions:
+  - aos
+  - opensearch.org
 chapters:
   - synopsis: Get server info.
     path: /

diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml
@@ -12,16 +12,6 @@ prologues:
         title: Moneyball
         year: 2011
     status: [201]
-  - path: /movies/_doc
-    method: POST
-    parameters:
-      refresh: true
-    request:
-      payload:
-        director: Nicolas Winding Refn
-        title: Drive
-        year: 2011
-    status: [201]
 epilogues:
   - path: /movies
     method: DELETE
@@ -42,7 +32,7 @@ chapters:
       payload:
         timed_out: false
         hits:
-          total: 2
+          total: 1
           max_score: 1
           hits:
             - _index: movies
@@ -51,12 +41,6 @@ chapters:
                 director: Bennett Miller
                 title: Moneyball
                 year: 2011
-            - _index: movies
-              _score: 1
-              _source:
-                director: Nicolas Winding Refn
-                title: Drive
-                year: 2011
   - synopsis: Search with rest_total_hits_as_int=false.
     path: /{index}/_search
     parameters:
@@ -73,7 +57,7 @@ chapters:
         timed_out: false
         hits:
           total:
-            value: 2
+            value: 1
             relation: eq
           max_score: 1
           hits:
@@ -83,12 +67,6 @@ chapters:
                 director: Bennett Miller
                 title: Moneyball
                 year: 2011
-            - _index: movies
-              _score: 1
-              _source:
-                director: Nicolas Winding Refn
-                title: Drive
-                year: 2011
   - synopsis: Search with rest_total_hits_as_int=false track_total_hits=1.
     path: /{index}/_search
     parameters:
@@ -107,7 +85,7 @@ chapters:
         hits:
           total:
             value: 1
-            relation: gte
+            relation: eq
           max_score: 1
           hits:
             - _index: movies
@@ -116,10 +94,3 @@ chapters:
                 director: Bennett Miller
                 title: Moneyball
                 year: 2011
-            - _index: movies
-              _score: 1
-              _source:
-                director: Nicolas Winding Refn
-                title: Drive
-                year: 2011
-
diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml
@@ -53,7 +53,6 @@ chapters:
       content_type: application/json
       payload:
         - node.total: '1'
-          status: yellow
           node.data: '1'
   - synopsis: Cat with master response (format=json).
     method: GET
@@ -66,7 +65,6 @@ chapters:
       content_type: application/json
       payload:
         - node.total: '1'
-          status: yellow
           node.data: '1'
           discovered_master: 'true'
   - synopsis: Cat with cluster_manager response (format=json).
@@ -80,7 +78,6 @@ chapters:
       content_type: application/json
       payload:
         - node.total: '1'
-          status: yellow
           node.data: '1'
           discovered_cluster_manager: 'true'
   - synopsis: Cat in different formats (format=yaml).
@@ -93,9 +90,10 @@ chapters:
       content_type: application/yaml
       payload:
         - node.total: '1'
-          status: yellow
           node.data: '1'
   - synopsis: Cat in different formats (format=cbor).
+    distributions:
+      - opensearch.org
     method: GET
     path: /_cat/health
     parameters:
@@ -105,9 +103,10 @@ chapters:
       content_type: application/cbor
       payload:
         - node.total: '1'
-          status: yellow
           node.data: '1'
   - synopsis: Cat in different formats (format=smile).
+    distributions:
+      - opensearch.org
     method: GET
     path: /_cat/health
     parameters:
@@ -117,5 +116,4 @@ chapters:
       content_type: application/smile
       payload:
         - node.total: '1'
-          status: yellow
           node.data: '1'
diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml
@@ -71,6 +71,8 @@ chapters:
       status: 200
       content_type: application/yaml
   - synopsis: Cat in different formats (format=cbor).
+    distributions:
+      - opensearch.org
     method: GET
     path: /_cat/indices
     parameters:
@@ -79,6 +81,8 @@ chapters:
       status: 200
       content_type: application/cbor
   - synopsis: Cat in different formats (format=smile).
+    distributions:
+      - opensearch.org
     method: GET
     path: /_cat/indices
     parameters:

diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml
@@ -3,6 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 description: Test cat/nodeattrs endpoints.
 chapters:
   - synopsis: Cat with a json response.
+    distributions:
+      - opensearch.org
     path: /_cat/nodeattrs
     method: GET
     parameters:

diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml
@@ -1,6 +1,8 @@
 $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test index clear cache.
+distributions:
+  - opensearch.org
 prologues:
   - path: /movies
     method: PUT

diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml
@@ -1,6 +1,8 @@
 $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test dangling indexes.
+distributions:
+  - opensearch.org
 chapters:
   - synopsis: Get dangling indexes.
     path: /_dangling