Added withFetchAll option (#72)

* feat: added fetchAll flag * changed way how to skip pagination and fetch all records * updated README and added max limit * lint * small fixes * version increment
crowdin · Jul 10, 2020 · 55fffa4 · 55fffa4
1 parent ac64ebb
commit 55fffa4
Show file tree

Hide file tree

Showing 24 changed files with 181 additions and 135 deletions.
diff --git a/README.md b/README.md
@@ -160,6 +160,34 @@ const projectsGroupsApi = new ProjectsGroups(credentials, {
 
 Or even pass your own http client as `httpClient` property which should implement `HttpClient` interface.
 
+### Fetch all records
+
+It is possible to fetch all records from paginatable methods (where we have limit and offset in arguments).
+
+```javascript
+import { ProjectsGroups } from '@crowdin/crowdin-api-client';
+
+// initialization of ProjectsGroups
+const projectsGroupsApi = new ProjectsGroups({
+  token: 'personalAccessToken',
+  organization: 'organizationName' // optional
+});
+
+// get all projects
+projectsGroupsApi
+  .withFetchAll()
+  .listProjects()
+  .then(projects => console.log(projects))
+  .catch(error => console.error(error));
+
+// get projects but not more than 1000
+projectsGroupsApi
+  .withFetchAll(1000)
+  .listProjects()
+  .then(projects => console.log(projects))
+  .catch(error => console.error(error));
+```
+
 ### Retry configuration
 
 There is a possibility to configure client invoke http calls with retry mechanism.

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@crowdin/crowdin-api-client",
-    "version": "1.8.11",
+    "version": "1.8.12",
     "description": "JavaScript library for Crowdin API v2.",
     "main": "out/index.js",
     "types": "out/index.d.ts",

diff --git a/src/core/index.ts b/src/core/index.ts
@@ -123,6 +123,9 @@ export abstract class CrowdinApi {
     readonly config: ClientConfig | undefined;
     readonly retryService: RetryService;
 
+    protected fetchAllFlag = false;
+    protected maxLimit: number | undefined;
+
     /**
      * @param credentials credentials
      * @param config optional configuration of the client
@@ -200,6 +203,67 @@ export abstract class CrowdinApi {
         return CrowdinApi.AXIOS_INSTANCE;
     }
 
+    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+    public withFetchAll(maxLimit?: number) {
+        this.fetchAllFlag = true;
+        this.maxLimit = maxLimit;
+        return this;
+    }
+
+    protected async getList<T = any>(
+        url: string,
+        limit?: number,
+        offset?: number,
+        config?: { headers: any },
+    ): Promise<ResponseList<T>> {
+        const conf = config || this.defaultConfig();
+        if (this.fetchAllFlag) {
+            this.fetchAllFlag = false;
+            const maxAmount = this.maxLimit;
+            this.maxLimit = undefined;
+            return await this.fetchAll(url, conf, maxAmount);
+        } else {
+            url = this.addQueryParam(url, 'limit', limit);
+            url = this.addQueryParam(url, 'offset', offset);
+            return this.get(url, conf);
+        }
+    }
+
+    protected async fetchAll<T = any>(
+        url: string,
+        config: { headers: any },
+        maxAmount?: number,
+    ): Promise<ResponseList<T>> {
+        let limit = 500;
+        if (!!maxAmount && maxAmount < limit) {
+            limit = maxAmount;
+        }
+        let offset = 0;
+        let resp: ResponseList<T> | undefined;
+        for (;;) {
+            let urlWithPagination = this.addQueryParam(url, 'limit', limit);
+            urlWithPagination = this.addQueryParam(urlWithPagination, 'offset', offset);
+            const e: ResponseList<T> = await this.get(urlWithPagination, config);
+            if (!resp) {
+                resp = e;
+            } else {
+                resp.data = resp.data.concat(e.data);
+                resp.pagination.limit += e.data.length;
+            }
+            if (e.data.length < limit || (!!maxAmount && resp.data.length >= maxAmount)) {
+                break;
+            } else {
+                offset += limit;
+            }
+            if (!!maxAmount) {
+                if (maxAmount < resp.data.length + limit) {
+                    limit = maxAmount - resp.data.length;
+                }
+            }
+        }
+        return resp;
+    }
+
     //Http overrides
 
     protected get<T = any>(url: string, config?: { headers: any }): Promise<T> {

diff --git a/src/distributions/index.ts b/src/distributions/index.ts
@@ -11,10 +11,8 @@ export class Distributions extends CrowdinApi {
         limit?: number,
         offset?: number,
     ): Promise<ResponseList<DistributionsModel.Distribution>> {
-        let url = `${this.url}/projects/${projectId}/distributions`;
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        const url = `${this.url}/projects/${projectId}/distributions`;
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/glossaries/index.ts b/src/glossaries/index.ts
@@ -9,9 +9,7 @@ export class Glossaries extends CrowdinApi {
     listGlossaries(groupId: number, limit?: number, offset?: number): Promise<ResponseList<GlossariesModel.Glossary>> {
         let url = `${this.url}/glossaries`;
         url = this.addQueryParam(url, 'groupId', groupId);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**
@@ -122,9 +120,7 @@ export class Glossaries extends CrowdinApi {
     ): Promise<ResponseList<GlossariesModel.Term>> {
         let url = `${this.url}/glossaries/${glossaryId}/terms`;
         url = this.addQueryParam(url, 'userId', userId);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/issues/index.ts b/src/issues/index.ts
@@ -1,4 +1,4 @@
-import { CrowdinApi, ResponseList, ResponseObject, PatchRequest } from '../core';
+import { CrowdinApi, PatchRequest, ResponseList, ResponseObject } from '../core';
 
 export class Issues extends CrowdinApi {
     /**
@@ -16,11 +16,9 @@ export class Issues extends CrowdinApi {
         status?: IssuesModel.Status,
     ): Promise<ResponseList<IssuesModel.Issue>> {
         let url = `${this.url}/projects/${projectId}/issues`;
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
         url = this.addQueryParam(url, 'type', type);
         url = this.addQueryParam(url, 'status', status);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/languages/index.ts b/src/languages/index.ts
@@ -1,15 +1,13 @@
-import { CrowdinApi, ResponseList, ResponseObject, PatchRequest } from '../core';
+import { CrowdinApi, PatchRequest, ResponseList, ResponseObject } from '../core';
 
 export class Languages extends CrowdinApi {
     /**
      * @param limit maximum number of items to retrieve (default 25)
      * @param offset starting offset in the collection (default 0)
      */
     listSupportedLanguages(limit?: number, offset?: number): Promise<ResponseList<LanguagesModel.Language>> {
-        let url = `${this.url}/languages`;
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        const url = `${this.url}/languages`;
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/machineTranslation/index.ts b/src/machineTranslation/index.ts
@@ -1,4 +1,4 @@
-import { CrowdinApi, ResponseList, ResponseObject, PatchRequest } from '../core';
+import { CrowdinApi, PatchRequest, ResponseList, ResponseObject } from '../core';
 
 export class MachineTranslation extends CrowdinApi {
     /**
@@ -13,9 +13,7 @@ export class MachineTranslation extends CrowdinApi {
     ): Promise<ResponseList<MachineTranslationModel.MachineTranslation>> {
         let url = `${this.url}/mts`;
         url = this.addQueryParam(url, 'groupId', groupId);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/projectsGroups/index.ts b/src/projectsGroups/index.ts
@@ -5,13 +5,18 @@ export class ProjectsGroups extends CrowdinApi {
      * @param parentId parent group identifier
      * @param offset starting offset in the collection (default 0)
      * @param userId get user own projects
+     * @param limit maximum number of items to retrieve (default 25)
      */
-    listGroups(parentId?: number, offset?: number, userId?: number): Promise<ResponseList<ProjectsGroupsModel.Group>> {
+    listGroups(
+        parentId?: number,
+        offset?: number,
+        userId?: number,
+        limit?: number,
+    ): Promise<ResponseList<ProjectsGroupsModel.Group>> {
         let url = `${this.url}/groups`;
         url = this.addQueryParam(url, 'parentId', parentId);
-        url = this.addQueryParam(url, 'offset', offset);
         url = this.addQueryParam(url, 'userId', userId);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**
@@ -62,9 +67,7 @@ export class ProjectsGroups extends CrowdinApi {
         let url = `${this.url}/projects`;
         url = this.addQueryParam(url, 'groupId', groupId);
         url = this.addQueryParam(url, 'hasManagerAccess', hasManagerAccess);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/screenshots/index.ts b/src/screenshots/index.ts
@@ -11,10 +11,8 @@ export class Screenshots extends CrowdinApi {
         limit?: number,
         offset?: number,
     ): Promise<ResponseList<ScreenshotsModel.Screenshot>> {
-        let url = `${this.url}/projects/${projectId}/screenshots`;
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        const url = `${this.url}/projects/${projectId}/screenshots`;
+        return this.getList(url, limit, offset);
     }
 
     /**
@@ -87,10 +85,8 @@ export class Screenshots extends CrowdinApi {
         limit?: number,
         offset?: number,
     ): Promise<ResponseList<ScreenshotsModel.Tag>> {
-        let url = `${this.url}/projects/${projectId}/screenshots/${screenshotId}/tags`;
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        const url = `${this.url}/projects/${projectId}/screenshots/${screenshotId}/tags`;
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/sourceFiles/index.ts b/src/sourceFiles/index.ts
@@ -15,9 +15,7 @@ export class SourceFiles extends CrowdinApi {
     ): Promise<ResponseList<SourceFilesModel.Branch>> {
         let url = `${this.url}/projects/${projectId}/branches`;
         url = this.addQueryParam(url, 'name', name);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**
@@ -81,9 +79,7 @@ export class SourceFiles extends CrowdinApi {
         let url = `${this.url}/projects/${projectId}/directories`;
         url = this.addQueryParam(url, 'branchId', branchId);
         url = this.addQueryParam(url, 'directoryId', directoryId);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**
@@ -170,10 +166,8 @@ export class SourceFiles extends CrowdinApi {
         }
         url = this.addQueryParam(url, 'branchId', request.branchId);
         url = this.addQueryParam(url, 'directoryId', request.directoryId);
-        url = this.addQueryParam(url, 'limit', request.limit);
-        url = this.addQueryParam(url, 'offset', request.offset);
         url = this.addQueryParam(url, 'recursion', request.recursion);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, request.limit, request.offset);
     }
 
     /**
@@ -255,10 +249,8 @@ export class SourceFiles extends CrowdinApi {
         limit?: number,
         offset?: number,
     ): Promise<ResponseList<SourceFilesModel.FileRevision>> {
-        let url = `${this.url}/projects/${projectId}/files/${fileId}/revisions`;
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        const url = `${this.url}/projects/${projectId}/files/${fileId}/revisions`;
+        return this.getList(url, limit, offset);
     }
 
     /**
@@ -289,9 +281,7 @@ export class SourceFiles extends CrowdinApi {
     ): Promise<ResponseList<SourceFilesModel.ReviewedSourceFilesBuild>> {
         let url = `${this.url}/projects/${projectId}/strings/reviewed-builds`;
         url = this.addQueryParam(url, 'branchId', branchId);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**

diff --git a/src/sourceStrings/index.ts b/src/sourceStrings/index.ts
@@ -17,10 +17,8 @@ export class SourceStrings extends CrowdinApi {
     ): Promise<ResponseList<SourceStringsModel.String>> {
         let url = `${this.url}/projects/${projectId}/strings`;
         url = this.addQueryParam(url, 'fileId', fileId);
-        url = this.addQueryParam(url, 'limit', limit);
-        url = this.addQueryParam(url, 'offset', offset);
         url = this.addQueryParam(url, 'filter', filter);
-        return this.get(url, this.defaultConfig());
+        return this.getList(url, limit, offset);
     }
 
     /**