Skip to content

Commit

Permalink
Added support for application/x-ndjson and tests for _bulk.
Browse files Browse the repository at this point in the history
Signed-off-by: dblock <dblock@amazon.com>
  • Loading branch information
dblock committed Jun 21, 2024
1 parent 33aa91a commit 04005f4
Show file tree
Hide file tree
Showing 10 changed files with 145 additions and 7 deletions.
1 change: 1 addition & 0 deletions .cspell
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ Lovins
Lucene
Millis
Multisearch
Moneyball
Nanos
Nori
ONNX
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added `created_time` and `last_updated_time` to `ml.get_model_group@200` ([#342](https://github.com/opensearch-project/opensearch-api-specification/pull/342))
- Added spellcheck linter ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
- Added tests for response payload ([#347](https://github.com/opensearch-project/opensearch-api-specification/pull/347))
- Added support for testing `application/x-ndjson` payloads ([#355](https://github.com/opensearch-project/opensearch-api-specification/pull/355))

### Changed

Expand Down
3 changes: 2 additions & 1 deletion spec/namespaces/_core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2120,7 +2120,8 @@ components:
schema:
type: array
items:
oneOf:
type: object
anyOf:
- $ref: '../schemas/_core.bulk.yaml#/components/schemas/OperationContainer'
- $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction'
- type: object
Expand Down
2 changes: 1 addition & 1 deletion spec/schemas/_core.bulk.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ components:
properties:
_id:
description: The document ID associated with the operation.
oneOf:
anyOf:
- type: string
- nullable: true
type: string
Expand Down
17 changes: 17 additions & 0 deletions tests/_core/bulk.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
$schema: ../../json_schemas/test_story.schema.yaml

skip: false
description: Test bulk endpoint.
epilogues:
- path: /movies
method: DELETE
status: [200, 404]
chapters:
- synopsis: Create an index.
path: /_bulk
method: POST
request_body:
content_type: application/x-ndjson
payload:
- {create: {_index: movies}}
- {director: Bennett Miller, title: Moneyball, year: 2011}
4 changes: 4 additions & 0 deletions tools/src/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ export function to_json(content: any, replacer?: (this: any, key: string, value:
return JSON.stringify(content, replacer, 2)
}

export function to_ndjson(content: any[]): string {
return _.join(_.map(content, JSON.stringify), "\n") + "\n"
}

export function write_json (file_path: string, content: any, replacer?: (this: any, key: string, value: any) => any): void {
write_text(file_path, to_json(content, replacer))
}
Expand Down
20 changes: 16 additions & 4 deletions tools/src/tester/ChapterReader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ import { type ChapterRequest, type ActualResponse, type Parameter } from './type
import { type OpenSearchHttpClient } from '../OpenSearchHttpClient'
import { type StoryOutputs } from './StoryOutputs'
import { Logger } from 'Logger'
import { to_json } from '../helpers'
import { to_json, to_ndjson } from '../helpers'

// A lightweight client for testing the API
export default class ChapterReader {
private readonly _client: OpenSearchHttpClient
private readonly logger: Logger
Expand All @@ -27,11 +26,16 @@ export default class ChapterReader {
const response: Record<string, any> = {}
const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {})
const [url_path, params] = this.#parse_url(chapter.path, resolved_params)
const request_data = chapter.request_body?.payload !== undefined ? story_outputs.resolve_value(chapter.request_body.payload) : undefined
this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) | ${to_json(request_data)}`)
const content_type = chapter.request_body?.content_type ?? 'application/json'
const request_data = chapter.request_body?.payload !== undefined ? this.#serialize_payload(
story_outputs.resolve_value(chapter.request_body.payload),
content_type
) : undefined
this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] | ${to_json(request_data)}`)
await this._client.request({
url: url_path,
method: chapter.method,
headers: { 'Content-Type' : content_type },
params,
data: request_data
}).then(r => {
Expand All @@ -54,6 +58,14 @@ export default class ChapterReader {
return response as ActualResponse
}

#serialize_payload(payload: any, content_type: string): any {
if (payload === undefined) return undefined
switch(content_type) {
case 'application/x-ndjson': return to_ndjson(payload as [any])
default: return payload
}
}

#parse_url (path: string, parameters: Record<string, Parameter>): [string, Record<string, Parameter>] {
const path_params = new Set<string>()
const parsed_path = path.replace(/{(\w+)}/g, (_, key) => {
Expand Down
1 change: 1 addition & 0 deletions tools/src/tester/SchemaValidator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export default class SchemaValidator {
if (! valid) {
this.logger.info(`# ${to_json(schema)}`)
this.logger.info(`* ${to_json(data)}`)
this.logger.info(`& ${to_json(validate.errors)}`)
}
return {
result: valid ? Result.PASSED : Result.FAILED,
Expand Down
13 changes: 12 additions & 1 deletion tools/tests/helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* compatible open source license.
*/

import { sort_array_by_keys } from '../src/helpers'
import { sort_array_by_keys, to_json, to_ndjson } from '../src/helpers'

describe('helpers', () => {
describe('sort_array_by_keys', () => {
Expand All @@ -25,4 +25,15 @@ describe('helpers', () => {
expect(arr).toEqual(['GET', 'POST'])
})
})

test('to_json', () => {
expect(to_json({})).toEqual("{}")
expect(to_json({x: 1})).toEqual("{\n \"x\": 1\n}")
})

test('to_ndjson', () => {
expect(to_ndjson([])).toEqual("\n")
expect(to_ndjson([{x: 1}])).toEqual("{\"x\":1}\n")
expect(to_ndjson([{x: 1}, {y: 'z'}])).toEqual("{\"x\":1}\n{\"y\":\"z\"}\n")
})
})
90 changes: 90 additions & 0 deletions tools/tests/tester/ChapterReader.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import { Logger } from 'Logger';
import { OpenSearchHttpClient } from 'OpenSearchHttpClient';
import axios from 'axios';
import ChapterReader from 'tester/ChapterReader';
import { StoryOutputs } from 'tester/StoryOutputs';

jest.mock('axios');
const mocked_axios = axios as jest.Mocked<typeof axios>;

describe('ChapterReader', () => {
var reader: ChapterReader

beforeEach(() => {
mocked_axios.create.mockReturnThis()

mocked_axios.request.mockResolvedValue({
status: 200,
headers: {
'content-type': 'application/json'
}
});

reader = new ChapterReader(new OpenSearchHttpClient(), new Logger())
})

afterEach(() => {
jest.clearAllMocks()
})

it('sends a GET request', async () => {
const result = await reader.read({
id: 'id',
path: 'path',
method: 'GET',
parameters: undefined,
request_body: undefined,
output: undefined
}, new StoryOutputs())

expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
expect(mocked_axios.request.mock.calls).toEqual([
[{ url: 'path', method: 'GET', headers: { 'Content-Type': 'application/json' }, params: {}, data: undefined }]
])
})

it('sends a POST request', async () => {
const result = await reader.read({
id: 'id',
path: 'path',
method: 'POST',
parameters: { 'x': 1 },
request_body: { payload: { "body": "present" } },
output: undefined
}, new StoryOutputs())

expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
expect(mocked_axios.request.mock.calls).toEqual([
[{ url: 'path', method: 'POST', headers: { 'Content-Type': 'application/json' }, params: { 'x': 1 }, data: { 'body': 'present' } }]
])
})

it('sends an nd-json POST request', async () => {
const result = await reader.read({
id: 'id',
path: 'path',
method: 'POST',
parameters: { 'x': 1 },
request_body: {
content_type: 'application/x-ndjson',
payload: [{ "body": "present" }]
},
output: undefined
}, new StoryOutputs())

expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
expect(mocked_axios.request.mock.calls).toEqual([
[{ url: 'path', method: 'POST', headers: { 'Content-Type': 'application/x-ndjson' }, params: { 'x': 1 }, data: "{\"body\":\"present\"}\n"}]
])
})
})

0 comments on commit 04005f4

Please sign in to comment.