From f1b8207359a8aeb82b3e1d142eae9dd3b2fb11b4 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 7 Aug 2024 11:54:25 +0200 Subject: [PATCH 1/6] fix(specs): ingestion docker task input --- specs/ingestion/common/schemas/task.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index a383b825ee..10f9c6e69f 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -460,10 +460,33 @@ DockerStreamsInput: type: object properties: streams: - type: object + type: array + items: '#/DockerStreams' required: - streams +DockerStreams: + type: object + properties: + name: + type: string + description: The name of the stream to fetch the data from (e.g. table name). + properties: + type: array + description: The properties of the stream to select (e.g. column). + items: + type: string + syncMode: + $ref: '#/DockerStreamsSyncMode' + required: + - name + - syncMode + +DockerStreamsSyncMode: + type: string + description: The strategy to use to fetch the data. + enum: [incremental, fullTable] + failureThreshold: type: integer minimum: 0 From cc899667ade6d6d9993a0c6ad21d0e93b3f1abb6 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 7 Aug 2024 11:56:36 +0200 Subject: [PATCH 2/6] chore: add test --- tests/CTS/requests/ingestion/createTask.json | 35 +++++++++++++++++ .../CTS/requests/ingestion/createTaskV1.json | 39 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/tests/CTS/requests/ingestion/createTask.json b/tests/CTS/requests/ingestion/createTask.json index 4c20362ed0..3daa050891 100644 --- a/tests/CTS/requests/ingestion/createTask.json +++ b/tests/CTS/requests/ingestion/createTask.json @@ -34,5 +34,40 @@ "action": "replace" } } + }, + { + "testName": "task shopify", + "parameters": { + "sourceID": "search", + "destinationID": "destinationName", + "cron": "* * * * *", + "action": "replace", + "input": { + "streams": [ + { + "name": "foo", + "syncMode": "incremental" + } + ] + } + }, + "request": { + "path": "/2/tasks", + "method": "POST", + "body": { + "sourceID": "search", + "destinationID": "destinationName", + "cron": "* * * * *", + "action": "replace", + "input": { + "streams": [ + { + "name": "foo", + "syncMode": "incremental" + } + ] + } + } + } } ] diff --git a/tests/CTS/requests/ingestion/createTaskV1.json b/tests/CTS/requests/ingestion/createTaskV1.json index 81c5c1e8ff..9535d78b4c 100644 --- a/tests/CTS/requests/ingestion/createTaskV1.json +++ b/tests/CTS/requests/ingestion/createTaskV1.json @@ -69,5 +69,44 @@ "action": "replace" } } + }, + { + "testName": "task shopify", + "parameters": { + "sourceID": "search", + "destinationID": "destinationName", + "trigger": { + "type": "onDemand" + }, + "action": "replace", + "input": { + "streams": [ + { + "name": "foo", + "syncMode": "incremental" + } + ] + } + }, + "request": { + "path": "/2/tasks", + "method": "POST", + "body": { + "sourceID": "search", + "destinationID": "destinationName", + "trigger": { + "type": "onDemand" + }, + "action": "replace", + "input": { + "streams": [ + { + "name": "foo", + "syncMode": "incremental" + } + ] + } + } + } } ] From 5560e26fa39f89fdbc2964d20dd71126216bf3b6 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 7 Aug 2024 11:59:59 +0200 Subject: [PATCH 3/6] fix: ref --- specs/ingestion/common/schemas/task.yml | 3 ++- specs/ingestion/paths/transformations/transformationsTry.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index 10f9c6e69f..0992d6a5d7 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -461,7 +461,8 @@ DockerStreamsInput: properties: streams: type: array - items: '#/DockerStreams' + items: + $ref: '#/DockerStreams' required: - streams diff --git a/specs/ingestion/paths/transformations/transformationsTry.yml b/specs/ingestion/paths/transformations/transformationsTry.yml index 2e2567bbf6..2f4c61ef24 100644 --- a/specs/ingestion/paths/transformations/transformationsTry.yml +++ b/specs/ingestion/paths/transformations/transformationsTry.yml @@ -3,7 +3,7 @@ post: - transformations summary: Try a transformation description: Try a transformation. - operationId: tryTransformations + operationId: tryTransformation x-acl: - addObject - deleteIndex From 24056859b27aeef9bd1f75ce079c39996960125f Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 7 Aug 2024 12:07:07 +0200 Subject: [PATCH 4/6] chore: naming --- .../ingestion/{tryTransformations.json => tryTransformation.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/CTS/requests/ingestion/{tryTransformations.json => tryTransformation.json} (100%) diff --git a/tests/CTS/requests/ingestion/tryTransformations.json b/tests/CTS/requests/ingestion/tryTransformation.json similarity index 100% rename from tests/CTS/requests/ingestion/tryTransformations.json rename to tests/CTS/requests/ingestion/tryTransformation.json From 4e5875a00457866e9252b56ba8f4471865ca4ef8 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 7 Aug 2024 12:15:28 +0200 Subject: [PATCH 5/6] fix: test --- tests/CTS/requests/ingestion/createTask.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CTS/requests/ingestion/createTask.json b/tests/CTS/requests/ingestion/createTask.json index 3daa050891..e8df4eb1eb 100644 --- a/tests/CTS/requests/ingestion/createTask.json +++ b/tests/CTS/requests/ingestion/createTask.json @@ -52,7 +52,7 @@ } }, "request": { - "path": "/2/tasks", + "path": "/1/tasks", "method": "POST", "body": { "sourceID": "search", From 6782f028133349c70caa83756e14078c53daf0dc Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 7 Aug 2024 12:24:37 +0200 Subject: [PATCH 6/6] chore: karma --- tests/CTS/requests/ingestion/createTask.json | 2 +- tests/CTS/requests/ingestion/createTaskV1.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CTS/requests/ingestion/createTask.json b/tests/CTS/requests/ingestion/createTask.json index e8df4eb1eb..3daa050891 100644 --- a/tests/CTS/requests/ingestion/createTask.json +++ b/tests/CTS/requests/ingestion/createTask.json @@ -52,7 +52,7 @@ } }, "request": { - "path": "/1/tasks", + "path": "/2/tasks", "method": "POST", "body": { "sourceID": "search", diff --git a/tests/CTS/requests/ingestion/createTaskV1.json b/tests/CTS/requests/ingestion/createTaskV1.json index 9535d78b4c..091fa6bfee 100644 --- a/tests/CTS/requests/ingestion/createTaskV1.json +++ b/tests/CTS/requests/ingestion/createTaskV1.json @@ -89,7 +89,7 @@ } }, "request": { - "path": "/2/tasks", + "path": "/1/tasks", "method": "POST", "body": { "sourceID": "search",