diff --git a/specs/abtesting/common/schemas/ABTest.yml b/specs/abtesting/common/schemas/ABTest.yml index 6ef7f73ba4..a769d098a0 100644 --- a/specs/abtesting/common/schemas/ABTest.yml +++ b/specs/abtesting/common/schemas/ABTest.yml @@ -154,10 +154,13 @@ MinimumDetectableEffect: description: | Smallest difference in an observable metric between variants. For example, to detect a 10% difference between variants, set this value to 0.1. - effect: - $ref: '#/Effect' + metric: + $ref: '#/EffectMetric' + required: + - size + - metric -Effect: +EffectMetric: type: string description: Metric for which you want to detect the smallest relative difference. enum: diff --git a/specs/abtesting/common/schemas/EstimateABTestResponse.yml b/specs/abtesting/common/schemas/EstimateABTestResponse.yml new file mode 100644 index 0000000000..cc8d8244dd --- /dev/null +++ b/specs/abtesting/common/schemas/EstimateABTestResponse.yml @@ -0,0 +1,18 @@ +EstimateABTestResponse: + type: object + properties: + durationDays: + type: integer + format: int64 + description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. + example: 21 + controlSampleSize: + type: integer + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the control variant. + example: 23415 + experimentSampleSize: + type: integer + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. + example: 23415 diff --git a/specs/abtesting/paths/estimate.yml b/specs/abtesting/paths/estimate.yml new file mode 100644 index 0000000000..fa29a90a08 --- /dev/null +++ b/specs/abtesting/paths/estimate.yml @@ -0,0 +1,62 @@ +post: + tags: + - abtest + operationId: estimateABTest + x-acl: + - analytics + summary: Estimate the sample size and duration of an A/B test + description: Given the traffic percentage and the expected effect size, this endpoint estimates the sample size and duration of an A/B test based on historical traffic. + requestBody: + required: true + content: + application/json: + schema: + title: estimateABTestRequest + type: object + additionalProperties: false + properties: + configuration: + title: estimateConfiguration + type: object + description: A/B test configuration for estimating the sample size and duration using minimum detectable effect. + properties: + outliers: + $ref: '../common/schemas/ABTest.yml#/Outliers' + emptySearch: + $ref: '../common/schemas/ABTest.yml#/EmptySearch' + minimumDetectableEffect: + $ref: '../common/schemas/ABTest.yml#/MinimumDetectableEffect' + required: + - minimumDetectableEffect + variants: + type: array + description: A/B test variants. + minItems: 2 + maxItems: 2 + items: + $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' + required: + - configuration + - variants + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/EstimateABTestResponse.yml#/EstimateABTestResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' diff --git a/specs/abtesting/spec.yml b/specs/abtesting/spec.yml index 512fdc2065..9f61c2804e 100644 --- a/specs/abtesting/spec.yml +++ b/specs/abtesting/spec.yml @@ -79,7 +79,7 @@ tags: description: | Manage A/B tests. - A/B tests are configurations of two indices, usually your production index and an index with different settings that you want to test. + A/B tests are configurations one or more indices, usually your production index and an index with different settings that you want to test. x-tagGroups: - name: General tags: @@ -99,6 +99,8 @@ paths: $ref: 'paths/stopABTest.yml' /2/abtests/schedule: $ref: 'paths/scheduleABTest.yml' + /2/abtests/estimate: + $ref: 'paths/estimate.yml' # ############### # ### Helpers ### diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json new file mode 100644 index 0000000000..fe8bc548f2 --- /dev/null +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -0,0 +1,51 @@ +[ + { + "testName": "estimate AB Test sample size", + "parameters": { + "configuration": { + "emptySearch": { + "exclude": true + }, + "minimumDetectableEffect": { + "size": 0.03, + "metric": "conversionRate" + } + }, + "variants": [ + { + "index": "AB_TEST_1", + "trafficPercentage": 50 + }, + { + "index": "AB_TEST_2", + "trafficPercentage": 50 + } + ] + }, + "request": { + "path": "/2/abtests/estimate", + "method": "POST", + "body": { + "configuration": { + "emptySearch": { + "exclude": true + }, + "minimumDetectableEffect": { + "size": 0.03, + "metric": "conversionRate" + } + }, + "variants": [ + { + "index": "AB_TEST_1", + "trafficPercentage": 50 + }, + { + "index": "AB_TEST_2", + "trafficPercentage": 50 + } + ] + } + } + } +] \ No newline at end of file