Added testing to the generated OpenAPI.

.github/workflows/coverage-gather.yml → .github/workflows/coverage-api.yml

@@ -45,7 +45,7 @@ jobs:
           total=`jq -r '.paths | keys | length' build/local-openapi.json`
           percent=$((current * 100 / total))
           echo "API specs implemented for $current/$total ($percent%) APIs."
-          cat >>"coverage.json" <<EOL
+          cat >>"coverage-api.json" <<EOL
           {
             "pull_request":${{ github.event.number }},
             "current":$current,
@@ -55,5 +55,5 @@ jobs:
           EOL
       - uses: actions/upload-artifact@v4
         with:
-          name: coverage
-          path: coverage.json
+          name: coverage-api
+          path: coverage-api.json

.github/workflows/coverage-comment.yml

@@ -2,7 +2,7 @@ name: Comment with API Coverage
 
 on:
   workflow_run:
-    workflows: ["Gather API Coverage"]
+    workflows: ["Gather API Coverage", "Test Spec"]
     types:
       - completed
 
@@ -17,7 +17,8 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          name: coverage
+          pattern: coverage-*
+          merge-multiple: true
           run-id: ${{ github.event.workflow_run.id }}
 
       - name: 'Comment on PR'
@@ -26,11 +27,13 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const fs = require('fs');
-            const data = JSON.parse(fs.readFileSync('./coverage.json'));
-            console.log(data);
+            const coverage_api = JSON.parse(fs.readFileSync('./coverage-api.json'));
+            console.log(coverage_api);
+            const coverage_tests = JSON.parse(fs.readFileSync('./coverage-tests.json'));
+            console.log(coverage_tests);
             await github.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: data.pull_request,
-              body: `API specs implemented for ${data.current}/${data.total} (${data.percent}%) APIs.`
+              issue_number: coverage_api.pull_request,
+              body: `API specs implemented for ${coverage_api.current}/${coverage_api.total} (${coverage_api.percent}%) APIs, with ${coverage_tests.current}/${coverage_tests.total} (${coverage_tests.percent}%) APIs tested.`
             });

.github/workflows/tests-spec.yml

@@ -0,0 +1,63 @@
+name: Test Spec
+
+on: [push, pull_request]
+
+env:
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: BobgG7YrtsdKf9M
+
+jobs:
+  tests-spec:
+    name: Build and Test Spec
+    runs-on: ubuntu-latest
+    permissions: write-all
+
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v3
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '20'
+
+      - name: Build
+        run: |-
+          mkdir -p ./build
+          npm install
+          npm run merge -- --source ./spec --output ./build/opensearch-openapi.yaml
+          npm run merge -- --source ./spec --output ./build/opensearch-openapi-tested.yaml --x-include=x-tested
+
+      - name: Build and Run Docker Container
+        run: |
+          docker build coverage --tag opensearch-with-api-plugin
+          docker run -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_INITIAL_ADMIN_PASSWORD" opensearch-with-api-plugin
+          sleep 15
+
+      - name: Display OpenSearch Info
+        run: |
+          curl -ks -u "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" https://localhost:9200/ | jq
+
+      - name: Test
+        run: |-
+          npm run dredd -- --user "admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD" ./build/opensearch-openapi-tested.yaml https://localhost:9200
+
+      - name: Gather Test Coverage
+        id: test-coverage
+        shell: bash
+        run: |
+          current=`docker run --rm -i mikefarah/yq:latest -r '.paths | keys | length' < build/opensearch-openapi-tested.yaml`
+          total=`docker run --rm -i mikefarah/yq:latest -r '.paths | keys | length' < build/opensearch-openapi.yaml`
+          percent=$((current * 100 / total))
+          echo "$current/$total ($percent%) APIs tested."
+          cat >>"coverage-tests.json" <<EOL
+          {
+            "pull_request":${{ github.event.number }},
+            "current":$current,
+            "total":$total,
+            "percent":$percent
+          }
+          EOL
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-tests
+          path: coverage-tests.json

.github/workflows/tools.yml → .github/workflows/tests-tools.yml

@@ -1,4 +1,4 @@
-name: Tools Testing
+name: Test Tools
 
 on:
   push:
@@ -11,7 +11,7 @@ on:
       - 'tools/**'
 
 jobs:
-  tools-tests:
+  tests-tools:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

DEVELOPER_GUIDE.md

@@ -7,6 +7,9 @@
   - [Global Parameters](#global-parameters)
   - [OpenAPI Extensions](#openapi-extensions)
   - [Tools](#tools)
+    - [Merger](#merger)
+    - [Linter](#linter)
+    - [Validator](#validator)
 
 # Developer Guide
 
@@ -115,6 +118,7 @@ This repository includes several OpenAPI Specification Extensions to fill in any
 - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them.
 - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml).
 - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema.
+- `x-tested`: Marks APIs as tested (or testable) using [test automation](./tests/). 
 
 ## Tools
 
@@ -127,3 +131,7 @@ The spec merger "builds", aka combines all `.yaml` files in a spec folder into a
 ### Linter
 
 The spec linter that validates every `.yaml` file in the `./spec` folder to assure that they follow the guidelines we have set. Check out the [Linter README](tools/README.md#spec-linter) for more information on how to run it locally. Make sure to run the linter before submitting a PR.
+
+### Validator
+
+The project uses [dredd](https://dredd.org) to test the spec against a live instance OpenSearch. See [tests](tests/README.md) for details.

coverage/README.md

@@ -2,4 +2,4 @@
 
 Builds the OpenAPI spec, and uses the [opensearch-api plugin](https://github.com/dblock/opensearch-api) and [openapi-diff](https://github.com/OpenAPITools/openapi-diff) to show the differences.
 
-API coverage is run on all pull requests via the [coverage workflow](../.github/workflows/coverage-gather.yml).
+API coverage is run on all pull requests via the [coverage workflow](../.github/workflows/coverage-api.yml).

eslint.config.mjs

@@ -48,10 +48,17 @@ export default [
           allowRuleToRunWithoutStrictNullChecksIKnowWhatIAmDoing: false
         }
       ],
+      '@typescript-eslint/no-var-requires': 'error',
       'array-callback-return': 'off',
       'new-cap': 'off',
       'no-return-assign': 'error',
       'object-shorthand': 'error'
     }
+  },
+  {
+    files: ['**/*.js'],
+    rules: {
+      '@typescript-eslint/no-var-requires': 'off'
+    }
   }
 ]