fix(pagination): [nan-1958] per page offset pagination (#2890)

## Describe your changes Offset right now only adds the response data length but many APIs just need the page to be incremented by one. As seen in NangoHQ/integration-templates#72 ## Issue ticket number and link NAN-1958 ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [ ] I added tests, otherwise the reason is: - [ ] I added observability, otherwise the reason is: - [ ] I added analytics, otherwise the reason is:
NangoHQ · Oct 25, 2024 · 9e4fe49 · 9e4fe49
1 parent a70a276
commit 9e4fe49
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 2 deletions.
diff --git a/docs-v2/reference/api-configuration.mdx b/docs-v2/reference/api-configuration.mdx
@@ -323,6 +323,9 @@ slack:
                 <ResponseField name="offset_name_in_request" type="string">
                     The name of the offset parameter in the request.
                 </ResponseField>
+                <ResponseField name="offset_calculation_method" type="string">
+                    The method to calculate the offset in the request. Must be one of: "per-page" or "by-response-size". Optional parameter that defaults to "by-response-size".
+                </ResponseField>
                 <ResponseField name="offset_start_value" type="number">
                     The starting value for the offset.
                 </ResponseField>

diff --git a/packages/cli/lib/services/__snapshots__/model.service.unit.test.ts.snap b/packages/cli/lib/services/__snapshots__/model.service.unit.test.ts.snap
@@ -102,6 +102,7 @@ interface LinkPagination extends Pagination {
 interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: 'per-page' | 'by-response-size';
 }
 interface RetryHeaderConfig {
     at?: string;

diff --git a/packages/shared/lib/models/Proxy.ts b/packages/shared/lib/models/Proxy.ts
@@ -83,7 +83,10 @@ export interface LinkPagination extends Pagination {
     link_path_in_response_body?: string;
 }
 
+export type OffsetCalculationMethod = 'per-page' | 'by-response-size';
+
 export interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: OffsetCalculationMethod;
 }
diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts
@@ -130,6 +130,7 @@ interface LinkPagination extends Pagination {
 interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: 'per-page' | 'by-response-size';
 }
 
 interface RetryHeaderConfig {

diff --git a/packages/shared/lib/services/paginate.service.ts b/packages/shared/lib/services/paginate.service.ts
@@ -1,7 +1,14 @@
 import type { AxiosResponse } from 'axios';
 import parseLinksHeader from 'parse-link-header';
 import get from 'lodash-es/get.js';
-import type { Pagination, UserProvidedProxyConfiguration, CursorPagination, OffsetPagination, LinkPagination } from '../models/Proxy.js';
+import type {
+    Pagination,
+    UserProvidedProxyConfiguration,
+    CursorPagination,
+    OffsetPagination,
+    LinkPagination,
+    OffsetCalculationMethod
+} from '../models/Proxy.js';
 import { PaginationType } from '../models/Proxy.js';
 import { isValidHttpUrl } from '../utils/utils.js';
 
@@ -124,6 +131,7 @@ class PaginationService {
     ): AsyncGenerator<T[], undefined, void> {
         const offsetPagination: OffsetPagination = paginationConfig;
         const offsetParameterName: string = offsetPagination.offset_name_in_request;
+        const offsetCalculatioMethod: OffsetCalculationMethod = offsetPagination.offset_calculation_method || 'by-response-size';
         let offset = offsetPagination.offset_start_value || 0;
 
         while (true) {
@@ -149,7 +157,11 @@ class PaginationService {
                 return;
             }
 
-            offset += responseData.length;
+            if (offsetCalculatioMethod === 'per-page') {
+                offset++;
+            } else {
+                offset += responseData.length;
+            }
         }
     }
 

diff --git a/packages/types/lib/proxy/api.ts b/packages/types/lib/proxy/api.ts
@@ -70,7 +70,10 @@ export interface LinkPagination extends Pagination {
     link_path_in_response_body?: string;
 }
 
+export type OffsetCalculationMethod = 'per-page' | 'by-response-size';
+
 export interface OffsetPagination extends Pagination {
     offset_name_in_request: string;
     offset_start_value?: number;
+    offset_calculation_method?: OffsetCalculationMethod;
 }
diff --git a/scripts/validation/providers/schema.json b/scripts/validation/providers/schema.json
@@ -159,6 +159,10 @@
                                 "offset_start_value": {
                                     "type": "number"
                                 },
+                                "offset_calculation_method": {
+                                    "type": "string",
+                                    "enum": ["per-page", "by-response-size"]
+                                },
                                 "response_path": {
                                     "type": "string"
                                 },