From efa672a7ccdd9f92d6274ec542e644f21f245142 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:00:23 -0400 Subject: [PATCH 01/13] feat: add estimate path and responses --- specs/abtesting/common/parameters.yml | 19 +++++++ .../common/schemas/EstimateResponse.yml | 2 + specs/abtesting/paths/estimate.yml | 51 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 specs/abtesting/common/schemas/EstimateResponse.yml create mode 100644 specs/abtesting/paths/estimate.yml diff --git a/specs/abtesting/common/parameters.yml b/specs/abtesting/common/parameters.yml index f58bd7e47b8..23198c94a33 100644 --- a/specs/abtesting/common/parameters.yml +++ b/specs/abtesting/common/parameters.yml @@ -137,3 +137,22 @@ filterEffects: type: integer description: Number of tracked searches removed from the A/B test. example: 237 + +minimumDetectableEffect: + type: object + properties: + durationDays: + type: number + format: int64 + description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. + example: 21 + controlSampleSize: + type: number + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the control variant. + example: 23415 + experimentSampleSize: + type: number + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. + example: 23415 diff --git a/specs/abtesting/common/schemas/EstimateResponse.yml b/specs/abtesting/common/schemas/EstimateResponse.yml new file mode 100644 index 00000000000..5712e8ec88a --- /dev/null +++ b/specs/abtesting/common/schemas/EstimateResponse.yml @@ -0,0 +1,2 @@ +EstimateResponse: + $ref: '../parameters.yml#/minimumDetectableEffect' diff --git a/specs/abtesting/paths/estimate.yml b/specs/abtesting/paths/estimate.yml new file mode 100644 index 00000000000..5e3b20c68a7 --- /dev/null +++ b/specs/abtesting/paths/estimate.yml @@ -0,0 +1,51 @@ +post: + tags: + - abtest + operationId: estimateABTest + x-acl: + - analytics + summary: Estimate the sample size and duration of an A/B test + description: Given the traffic percentage and the expected effect size, this endpoint estimates the sample size and duration of an A/B test based on historical traffic. + requestBody: + required: true + content: + application/json: + schema: + title: estimateABTestRequest + type: object + additionalProperties: false + properties: + configuration: + $ref: '../common/schemas/ABTest.yml#/ABTestConfiguration' + variants: + type: array + description: A/B test variants. + minItems: 2 + maxItems: 2 + items: + $ref: '../common/schemas/AddABTestsVariant.yml#/AddABTestsVariant' + required: + - configuration + - variants + responses: + '200': + description: OK + headers: + x-ratelimit-limit: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-limit' + x-ratelimit-remaining: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-remaining' + x-ratelimit-reset: + $ref: '../../common/responses/rateLimit.yml#/x-ratelimit-reset' + content: + application/json: + schema: + $ref: '../common/schemas/EstimateResponse.yml#/EstimateResponse' + '400': + $ref: '../../common/responses/BadRequest.yml' + '402': + $ref: '../../common/responses/FeatureNotEnabled.yml' + '403': + $ref: '../../common/responses/MethodNotAllowed.yml' + '404': + $ref: '../../common/responses/IndexNotFound.yml' From 3b52e850cc045c03ac032200b685952d367ce39c Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:20:39 -0500 Subject: [PATCH 02/13] fix: reference --- specs/abtesting/spec.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/abtesting/spec.yml b/specs/abtesting/spec.yml index 512fdc2065c..9f61c2804e6 100644 --- a/specs/abtesting/spec.yml +++ b/specs/abtesting/spec.yml @@ -79,7 +79,7 @@ tags: description: | Manage A/B tests. - A/B tests are configurations of two indices, usually your production index and an index with different settings that you want to test. + A/B tests are configurations one or more indices, usually your production index and an index with different settings that you want to test. x-tagGroups: - name: General tags: @@ -99,6 +99,8 @@ paths: $ref: 'paths/stopABTest.yml' /2/abtests/schedule: $ref: 'paths/scheduleABTest.yml' + /2/abtests/estimate: + $ref: 'paths/estimate.yml' # ############### # ### Helpers ### From 3ba7922e1e7612a96614a989758770eb0cf285a8 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:35:26 -0500 Subject: [PATCH 03/13] feat: add test --- .../requests/abtesting/estimateABTest.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/CTS/requests/abtesting/estimateABTest.json diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json new file mode 100644 index 00000000000..f92cec303ef --- /dev/null +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -0,0 +1,23 @@ +[ + { + "configuration": { + "emptySearch": { + "exclude": true + }, + "minimumDetectableEffect": { + "size": 0.03, + "metric": "conversionrate" + } + }, + "variants": [ + { + "index": "AB_TEST_1", + "trafficPercentage": 50 + }, + { + "index": "AB_TEST_2", + "trafficPercentage": 50 + } + ] + } +] \ No newline at end of file From 2732ff3c0ba501e757d607b6d89c26edad437359 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:09:29 -0500 Subject: [PATCH 04/13] fix: test --- .../requests/abtesting/estimateABTest.json | 55 +++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json index f92cec303ef..7ea84c7cb74 100644 --- a/tests/CTS/requests/abtesting/estimateABTest.json +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -1,23 +1,44 @@ [ { - "configuration": { - "emptySearch": { - "exclude": true - }, - "minimumDetectableEffect": { - "size": 0.03, - "metric": "conversionrate" - } + "testName": "estimate AB Test sample size", + "parameters": { + "endAt": "2022-12-31T00:00:00.000Z", + "name": "myABTest", + "variants": [ + { + "index": "AB_TEST_1", + "trafficPercentage": 30 + }, + { + "index": "AB_TEST_2", + "trafficPercentage": 50 + } + ] }, - "variants": [ - { - "index": "AB_TEST_1", - "trafficPercentage": 50 - }, - { - "index": "AB_TEST_2", - "trafficPercentage": 50 + "request": { + "path": "/2/abtests/estimate", + "method": "POST", + "body": { + "configuration": { + "emptySearch": { + "exclude": true + }, + "minimumDetectableEffect": { + "size": 0.03, + "metric": "conversionrate" + } + }, + "variants": [ + { + "index": "AB_TEST_1", + "trafficPercentage": 50 + }, + { + "index": "AB_TEST_2", + "trafficPercentage": 50 + } + ] } - ] + } } ] \ No newline at end of file From 2aa6807f4e2fed553898902e10318821ed9db59b Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:20:20 -0500 Subject: [PATCH 05/13] fix: test --- tests/CTS/requests/abtesting/estimateABTest.json | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json index 7ea84c7cb74..b6319a7fb27 100644 --- a/tests/CTS/requests/abtesting/estimateABTest.json +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -2,12 +2,19 @@ { "testName": "estimate AB Test sample size", "parameters": { - "endAt": "2022-12-31T00:00:00.000Z", - "name": "myABTest", + "configuration": { + "emptySearch": { + "exclude": true + }, + "minimumDetectableEffect": { + "size": 0.03, + "metric": "conversionrate" + } + }, "variants": [ { "index": "AB_TEST_1", - "trafficPercentage": 30 + "trafficPercentage": 50 }, { "index": "AB_TEST_2", From f9c803890033394976858adae70628e652547040 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:51:16 -0500 Subject: [PATCH 06/13] chore: tests --- specs/abtesting/common/schemas/ABTest.yml | 2 +- .../requests/abtesting/estimateABTest.json | 21 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/specs/abtesting/common/schemas/ABTest.yml b/specs/abtesting/common/schemas/ABTest.yml index 6ef7f73ba42..bc6623d0adc 100644 --- a/specs/abtesting/common/schemas/ABTest.yml +++ b/specs/abtesting/common/schemas/ABTest.yml @@ -154,7 +154,7 @@ MinimumDetectableEffect: description: | Smallest difference in an observable metric between variants. For example, to detect a 10% difference between variants, set this value to 0.1. - effect: + metric: $ref: '#/Effect' Effect: diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json index b6319a7fb27..348808f72e9 100644 --- a/tests/CTS/requests/abtesting/estimateABTest.json +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -1,27 +1,6 @@ [ { "testName": "estimate AB Test sample size", - "parameters": { - "configuration": { - "emptySearch": { - "exclude": true - }, - "minimumDetectableEffect": { - "size": 0.03, - "metric": "conversionrate" - } - }, - "variants": [ - { - "index": "AB_TEST_1", - "trafficPercentage": 50 - }, - { - "index": "AB_TEST_2", - "trafficPercentage": 50 - } - ] - }, "request": { "path": "/2/abtests/estimate", "method": "POST", From 2746d95974f0d1386fa087c452f8fb3595b029e4 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:48:06 -0500 Subject: [PATCH 07/13] chore: adjust parameters again --- tests/CTS/requests/abtesting/estimateABTest.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json index 348808f72e9..9f21ce5cb4f 100644 --- a/tests/CTS/requests/abtesting/estimateABTest.json +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -1,6 +1,7 @@ [ { "testName": "estimate AB Test sample size", + "parameters": {}, "request": { "path": "/2/abtests/estimate", "method": "POST", From be5035f26e67e89dd5827161758e6c65f771c450 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:53:54 -0500 Subject: [PATCH 08/13] chore: additional changes --- specs/abtesting/common/schemas/ABTest.yml | 7 ++++-- .../common/schemas/EstimateResponse.yml | 22 ++++++++++++++++++- specs/abtesting/paths/estimate.yml | 15 +++++++++++-- .../requests/abtesting/estimateABTest.json | 22 ++++++++++++++++++- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/specs/abtesting/common/schemas/ABTest.yml b/specs/abtesting/common/schemas/ABTest.yml index bc6623d0adc..ea4d51ea95b 100644 --- a/specs/abtesting/common/schemas/ABTest.yml +++ b/specs/abtesting/common/schemas/ABTest.yml @@ -155,9 +155,12 @@ MinimumDetectableEffect: Smallest difference in an observable metric between variants. For example, to detect a 10% difference between variants, set this value to 0.1. metric: - $ref: '#/Effect' + $ref: '#/Metric' + required: + - size + - metric -Effect: +Metric: type: string description: Metric for which you want to detect the smallest relative difference. enum: diff --git a/specs/abtesting/common/schemas/EstimateResponse.yml b/specs/abtesting/common/schemas/EstimateResponse.yml index 5712e8ec88a..b6dc563478b 100644 --- a/specs/abtesting/common/schemas/EstimateResponse.yml +++ b/specs/abtesting/common/schemas/EstimateResponse.yml @@ -1,2 +1,22 @@ EstimateResponse: - $ref: '../parameters.yml#/minimumDetectableEffect' + type: object + properties: + durationDays: + type: number + format: int64 + description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. + example: 21 + controlSampleSize: + type: number + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the control variant. + example: 23415 + experimentSampleSize: + type: number + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. + example: 23415 + required: + - durationDays + - controlSampleSize + - experimentSampleSize diff --git a/specs/abtesting/paths/estimate.yml b/specs/abtesting/paths/estimate.yml index 5e3b20c68a7..61229b0f2c1 100644 --- a/specs/abtesting/paths/estimate.yml +++ b/specs/abtesting/paths/estimate.yml @@ -11,12 +11,23 @@ post: content: application/json: schema: - title: estimateABTestRequest + title: estimateRequest type: object additionalProperties: false properties: configuration: - $ref: '../common/schemas/ABTest.yml#/ABTestConfiguration' + title: configuration + type: object + description: A/B test configuration. + properties: + outliers: + $ref: '../common/schemas/ABTest.yml#/Outliers' + emptySearch: + $ref: '../common/schemas/ABTest.yml#/EmptySearch' + minimumDetectableEffect: + $ref: '../common/schemas/ABTest.yml#/MinimumDetectableEffect' + required: + - minimumDetectableEffect variants: type: array description: A/B test variants. diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json index 9f21ce5cb4f..b6319a7fb27 100644 --- a/tests/CTS/requests/abtesting/estimateABTest.json +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -1,7 +1,27 @@ [ { "testName": "estimate AB Test sample size", - "parameters": {}, + "parameters": { + "configuration": { + "emptySearch": { + "exclude": true + }, + "minimumDetectableEffect": { + "size": 0.03, + "metric": "conversionrate" + } + }, + "variants": [ + { + "index": "AB_TEST_1", + "trafficPercentage": 50 + }, + { + "index": "AB_TEST_2", + "trafficPercentage": 50 + } + ] + }, "request": { "path": "/2/abtests/estimate", "method": "POST", From 30ed903a398398e7a3d787217e014ef45fac4db5 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:04:34 -0500 Subject: [PATCH 09/13] chore: change estimate --- .../common/schemas/EstimateResponse.yml | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/specs/abtesting/common/schemas/EstimateResponse.yml b/specs/abtesting/common/schemas/EstimateResponse.yml index b6dc563478b..60e3af4ec8b 100644 --- a/specs/abtesting/common/schemas/EstimateResponse.yml +++ b/specs/abtesting/common/schemas/EstimateResponse.yml @@ -1,22 +1,2 @@ -EstimateResponse: - type: object - properties: - durationDays: - type: number - format: int64 - description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. - example: 21 - controlSampleSize: - type: number - format: int64 - description: Number of tracked searches needed to be able to detect the configured effect for the control variant. - example: 23415 - experimentSampleSize: - type: number - format: int64 - description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. - example: 23415 - required: - - durationDays - - controlSampleSize - - experimentSampleSize +EstimateABTestResponse: + $ref: '../parameters.yml#/minimumDetectableEffect' From 7f199a8c115581e952fa9a5b90be4bbd0efa2f3f Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:12:30 -0500 Subject: [PATCH 10/13] chore: rename and change spec --- specs/abtesting/common/parameters.yml | 19 ------------------- .../common/schemas/EstimateABTestResponse.yml | 18 ++++++++++++++++++ .../common/schemas/EstimateResponse.yml | 2 -- specs/abtesting/paths/estimate.yml | 8 ++++---- 4 files changed, 22 insertions(+), 25 deletions(-) create mode 100644 specs/abtesting/common/schemas/EstimateABTestResponse.yml delete mode 100644 specs/abtesting/common/schemas/EstimateResponse.yml diff --git a/specs/abtesting/common/parameters.yml b/specs/abtesting/common/parameters.yml index 23198c94a33..f58bd7e47b8 100644 --- a/specs/abtesting/common/parameters.yml +++ b/specs/abtesting/common/parameters.yml @@ -137,22 +137,3 @@ filterEffects: type: integer description: Number of tracked searches removed from the A/B test. example: 237 - -minimumDetectableEffect: - type: object - properties: - durationDays: - type: number - format: int64 - description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. - example: 21 - controlSampleSize: - type: number - format: int64 - description: Number of tracked searches needed to be able to detect the configured effect for the control variant. - example: 23415 - experimentSampleSize: - type: number - format: int64 - description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. - example: 23415 diff --git a/specs/abtesting/common/schemas/EstimateABTestResponse.yml b/specs/abtesting/common/schemas/EstimateABTestResponse.yml new file mode 100644 index 00000000000..e036f184de5 --- /dev/null +++ b/specs/abtesting/common/schemas/EstimateABTestResponse.yml @@ -0,0 +1,18 @@ +EstimateABTestResponse: + type: object + properties: + durationDays: + type: number + format: int64 + description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. + example: 21 + controlSampleSize: + type: number + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the control variant. + example: 23415 + experimentSampleSize: + type: number + format: int64 + description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. + example: 23415 diff --git a/specs/abtesting/common/schemas/EstimateResponse.yml b/specs/abtesting/common/schemas/EstimateResponse.yml deleted file mode 100644 index 60e3af4ec8b..00000000000 --- a/specs/abtesting/common/schemas/EstimateResponse.yml +++ /dev/null @@ -1,2 +0,0 @@ -EstimateABTestResponse: - $ref: '../parameters.yml#/minimumDetectableEffect' diff --git a/specs/abtesting/paths/estimate.yml b/specs/abtesting/paths/estimate.yml index 61229b0f2c1..fa29a90a080 100644 --- a/specs/abtesting/paths/estimate.yml +++ b/specs/abtesting/paths/estimate.yml @@ -11,14 +11,14 @@ post: content: application/json: schema: - title: estimateRequest + title: estimateABTestRequest type: object additionalProperties: false properties: configuration: - title: configuration + title: estimateConfiguration type: object - description: A/B test configuration. + description: A/B test configuration for estimating the sample size and duration using minimum detectable effect. properties: outliers: $ref: '../common/schemas/ABTest.yml#/Outliers' @@ -51,7 +51,7 @@ post: content: application/json: schema: - $ref: '../common/schemas/EstimateResponse.yml#/EstimateResponse' + $ref: '../common/schemas/EstimateABTestResponse.yml#/EstimateABTestResponse' '400': $ref: '../../common/responses/BadRequest.yml' '402': From e06f159d288c5d4e515732e6ec474aca2ae660f8 Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:37:11 -0500 Subject: [PATCH 11/13] chore: rename metric to effectmetric --- specs/abtesting/common/schemas/ABTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/abtesting/common/schemas/ABTest.yml b/specs/abtesting/common/schemas/ABTest.yml index ea4d51ea95b..a769d098a06 100644 --- a/specs/abtesting/common/schemas/ABTest.yml +++ b/specs/abtesting/common/schemas/ABTest.yml @@ -155,12 +155,12 @@ MinimumDetectableEffect: Smallest difference in an observable metric between variants. For example, to detect a 10% difference between variants, set this value to 0.1. metric: - $ref: '#/Metric' + $ref: '#/EffectMetric' required: - size - metric -Metric: +EffectMetric: type: string description: Metric for which you want to detect the smallest relative difference. enum: From ed7d93cf37caac8bae2e4e078aa893ccbaf4c47e Mon Sep 17 00:00:00 2001 From: Christopher Hawke <69921547+cdhawke@users.noreply.github.com> Date: Wed, 6 Nov 2024 12:06:16 -0500 Subject: [PATCH 12/13] fix: cts enum syntax.... --- tests/CTS/requests/abtesting/estimateABTest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CTS/requests/abtesting/estimateABTest.json b/tests/CTS/requests/abtesting/estimateABTest.json index b6319a7fb27..fe8bc548f2d 100644 --- a/tests/CTS/requests/abtesting/estimateABTest.json +++ b/tests/CTS/requests/abtesting/estimateABTest.json @@ -8,7 +8,7 @@ }, "minimumDetectableEffect": { "size": 0.03, - "metric": "conversionrate" + "metric": "conversionRate" } }, "variants": [ @@ -32,7 +32,7 @@ }, "minimumDetectableEffect": { "size": 0.03, - "metric": "conversionrate" + "metric": "conversionRate" } }, "variants": [ From c75b8478e9368acf6de297aaac1549a08820ffa5 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 7 Nov 2024 13:53:30 +0100 Subject: [PATCH 13/13] fix number --- specs/abtesting/common/schemas/EstimateABTestResponse.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/abtesting/common/schemas/EstimateABTestResponse.yml b/specs/abtesting/common/schemas/EstimateABTestResponse.yml index e036f184de5..cc8d8244dd6 100644 --- a/specs/abtesting/common/schemas/EstimateABTestResponse.yml +++ b/specs/abtesting/common/schemas/EstimateABTestResponse.yml @@ -2,17 +2,17 @@ EstimateABTestResponse: type: object properties: durationDays: - type: number + type: integer format: int64 description: Estimated number of days needed to reach the sample sizes required for detecting the configured effect. This value is based on historical traffic. example: 21 controlSampleSize: - type: number + type: integer format: int64 description: Number of tracked searches needed to be able to detect the configured effect for the control variant. example: 23415 experimentSampleSize: - type: number + type: integer format: int64 description: Number of tracked searches needed to be able to detect the configured effect for the experiment variant. example: 23415