From 04005f411c48136af4df1a9bec9a0ef9baac1cc6 Mon Sep 17 00:00:00 2001 From: dblock Date: Fri, 21 Jun 2024 10:04:26 -0400 Subject: [PATCH] Added support for application/x-ndjson and tests for _bulk. Signed-off-by: dblock --- .cspell | 1 + CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 3 +- spec/schemas/_core.bulk.yaml | 2 +- tests/_core/bulk.yaml | 17 +++++ tools/src/helpers.ts | 4 ++ tools/src/tester/ChapterReader.ts | 20 ++++-- tools/src/tester/SchemaValidator.ts | 1 + tools/tests/helpers.test.ts | 13 +++- tools/tests/tester/ChapterReader.test.ts | 90 ++++++++++++++++++++++++ 10 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 tests/_core/bulk.yaml create mode 100644 tools/tests/tester/ChapterReader.test.ts diff --git a/.cspell b/.cspell index 3acd43642..07bbfa27f 100644 --- a/.cspell +++ b/.cspell @@ -20,6 +20,7 @@ Lovins Lucene Millis Multisearch +Moneyball Nanos Nori ONNX diff --git a/CHANGELOG.md b/CHANGELOG.md index 53beddde3..232779597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `created_time` and `last_updated_time` to `ml.get_model_group@200` ([#342](https://github.com/opensearch-project/opensearch-api-specification/pull/342)) - Added spellcheck linter ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341)) - Added tests for response payload ([#347](https://github.com/opensearch-project/opensearch-api-specification/pull/347)) +- Added support for testing `application/x-ndjson` payloads ([#355](https://github.com/opensearch-project/opensearch-api-specification/pull/355)) ### Changed diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index a2c35ef50..fc634a5e0 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2120,7 +2120,8 @@ components: schema: type: array items: - oneOf: + type: object + anyOf: - $ref: '../schemas/_core.bulk.yaml#/components/schemas/OperationContainer' - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction' - type: object diff --git a/spec/schemas/_core.bulk.yaml b/spec/schemas/_core.bulk.yaml index 97e5583d0..d4f725b21 100644 --- a/spec/schemas/_core.bulk.yaml +++ b/spec/schemas/_core.bulk.yaml @@ -112,7 +112,7 @@ components: properties: _id: description: The document ID associated with the operation. - oneOf: + anyOf: - type: string - nullable: true type: string diff --git a/tests/_core/bulk.yaml b/tests/_core/bulk.yaml new file mode 100644 index 000000000..9efa20a14 --- /dev/null +++ b/tests/_core/bulk.yaml @@ -0,0 +1,17 @@ +$schema: ../../json_schemas/test_story.schema.yaml + +skip: false +description: Test bulk endpoint. +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create an index. + path: /_bulk + method: POST + request_body: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} diff --git a/tools/src/helpers.ts b/tools/src/helpers.ts index bb95fd253..361495a92 100644 --- a/tools/src/helpers.ts +++ b/tools/src/helpers.ts @@ -97,6 +97,10 @@ export function to_json(content: any, replacer?: (this: any, key: string, value: return JSON.stringify(content, replacer, 2) } +export function to_ndjson(content: any[]): string { + return _.join(_.map(content, JSON.stringify), "\n") + "\n" +} + export function write_json (file_path: string, content: any, replacer?: (this: any, key: string, value: any) => any): void { write_text(file_path, to_json(content, replacer)) } diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index 04ff18b1d..eb5a3aba5 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -11,9 +11,8 @@ import { type ChapterRequest, type ActualResponse, type Parameter } from './type import { type OpenSearchHttpClient } from '../OpenSearchHttpClient' import { type StoryOutputs } from './StoryOutputs' import { Logger } from 'Logger' -import { to_json } from '../helpers' +import { to_json, to_ndjson } from '../helpers' -// A lightweight client for testing the API export default class ChapterReader { private readonly _client: OpenSearchHttpClient private readonly logger: Logger @@ -27,11 +26,16 @@ export default class ChapterReader { const response: Record = {} const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {}) const [url_path, params] = this.#parse_url(chapter.path, resolved_params) - const request_data = chapter.request_body?.payload !== undefined ? story_outputs.resolve_value(chapter.request_body.payload) : undefined - this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) | ${to_json(request_data)}`) + const content_type = chapter.request_body?.content_type ?? 'application/json' + const request_data = chapter.request_body?.payload !== undefined ? this.#serialize_payload( + story_outputs.resolve_value(chapter.request_body.payload), + content_type + ) : undefined + this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] | ${to_json(request_data)}`) await this._client.request({ url: url_path, method: chapter.method, + headers: { 'Content-Type' : content_type }, params, data: request_data }).then(r => { @@ -54,6 +58,14 @@ export default class ChapterReader { return response as ActualResponse } + #serialize_payload(payload: any, content_type: string): any { + if (payload === undefined) return undefined + switch(content_type) { + case 'application/x-ndjson': return to_ndjson(payload as [any]) + default: return payload + } + } + #parse_url (path: string, parameters: Record): [string, Record] { const path_params = new Set() const parsed_path = path.replace(/{(\w+)}/g, (_, key) => { diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index 7d13320f9..25fe3d5d1 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -35,6 +35,7 @@ export default class SchemaValidator { if (! valid) { this.logger.info(`# ${to_json(schema)}`) this.logger.info(`* ${to_json(data)}`) + this.logger.info(`& ${to_json(validate.errors)}`) } return { result: valid ? Result.PASSED : Result.FAILED, diff --git a/tools/tests/helpers.test.ts b/tools/tests/helpers.test.ts index 29ae225a5..c11ea7d76 100644 --- a/tools/tests/helpers.test.ts +++ b/tools/tests/helpers.test.ts @@ -7,7 +7,7 @@ * compatible open source license. */ -import { sort_array_by_keys } from '../src/helpers' +import { sort_array_by_keys, to_json, to_ndjson } from '../src/helpers' describe('helpers', () => { describe('sort_array_by_keys', () => { @@ -25,4 +25,15 @@ describe('helpers', () => { expect(arr).toEqual(['GET', 'POST']) }) }) + + test('to_json', () => { + expect(to_json({})).toEqual("{}") + expect(to_json({x: 1})).toEqual("{\n \"x\": 1\n}") + }) + + test('to_ndjson', () => { + expect(to_ndjson([])).toEqual("\n") + expect(to_ndjson([{x: 1}])).toEqual("{\"x\":1}\n") + expect(to_ndjson([{x: 1}, {y: 'z'}])).toEqual("{\"x\":1}\n{\"y\":\"z\"}\n") + }) }) diff --git a/tools/tests/tester/ChapterReader.test.ts b/tools/tests/tester/ChapterReader.test.ts new file mode 100644 index 000000000..3889c4dd8 --- /dev/null +++ b/tools/tests/tester/ChapterReader.test.ts @@ -0,0 +1,90 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { Logger } from 'Logger'; +import { OpenSearchHttpClient } from 'OpenSearchHttpClient'; +import axios from 'axios'; +import ChapterReader from 'tester/ChapterReader'; +import { StoryOutputs } from 'tester/StoryOutputs'; + +jest.mock('axios'); +const mocked_axios = axios as jest.Mocked; + +describe('ChapterReader', () => { + var reader: ChapterReader + + beforeEach(() => { + mocked_axios.create.mockReturnThis() + + mocked_axios.request.mockResolvedValue({ + status: 200, + headers: { + 'content-type': 'application/json' + } + }); + + reader = new ChapterReader(new OpenSearchHttpClient(), new Logger()) + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + it('sends a GET request', async () => { + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'GET', + parameters: undefined, + request_body: undefined, + output: undefined + }, new StoryOutputs()) + + expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(mocked_axios.request.mock.calls).toEqual([ + [{ url: 'path', method: 'GET', headers: { 'Content-Type': 'application/json' }, params: {}, data: undefined }] + ]) + }) + + it('sends a POST request', async () => { + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'POST', + parameters: { 'x': 1 }, + request_body: { payload: { "body": "present" } }, + output: undefined + }, new StoryOutputs()) + + expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(mocked_axios.request.mock.calls).toEqual([ + [{ url: 'path', method: 'POST', headers: { 'Content-Type': 'application/json' }, params: { 'x': 1 }, data: { 'body': 'present' } }] + ]) + }) + + it('sends an nd-json POST request', async () => { + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'POST', + parameters: { 'x': 1 }, + request_body: { + content_type: 'application/x-ndjson', + payload: [{ "body": "present" }] + }, + output: undefined + }, new StoryOutputs()) + + expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(mocked_axios.request.mock.calls).toEqual([ + [{ url: 'path', method: 'POST', headers: { 'Content-Type': 'application/x-ndjson' }, params: { 'x': 1 }, data: "{\"body\":\"present\"}\n"}] + ]) + }) +}) +