fix: skip duplicated commits

nya1 · Mar 18, 2023 · 2468abf · 2468abf
1 parent 0b86c70
commit 2468abf
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 3 deletions.
diff --git a/src/integrations/github.ts b/src/integrations/github.ts
@@ -242,6 +242,7 @@ export class GithubIntegration extends IntegrationBase {
       let page = 1
 
       let commitList: GithubCommitItemData[] = []
+      const commitShaList = new Set<string>()
 
       while (page > 0) {
         logger.debug(`github integration working on ${page}`)
@@ -260,7 +261,14 @@ export class GithubIntegration extends IntegrationBase {
         }
 
         // eslint-disable-next-line unicorn/prefer-spread
-        commitList = commitList.concat(commits.items)
+        commitList = commitList.concat(
+          // remove duplicate items
+          commits.items.filter(c => !commitShaList.has(c.sha)),
+        )
+        for (const c of commits.items) {
+          commitShaList.add(c.sha)
+        }
+
         logger.debug(`github integration commitList ${commitList.length} (added ${commits.items.length} commits)`)
 
         page += 1

diff --git a/src/integrations/gitlab.ts b/src/integrations/gitlab.ts
@@ -158,6 +158,7 @@ export class GitlabIntegration extends IntegrationBase {
       let eventList: GitlabRawEventData[] = []
 
       const projectIds = new Set<number>()
+      const commitShaList = new Set<string>()
 
       while (page > 0) {
         logger.debug(`gitlab integration working on ${page}`)
@@ -175,12 +176,17 @@ export class GitlabIntegration extends IntegrationBase {
           break
         }
 
+        // eslint-disable-next-line unicorn/prefer-spread
+        eventList = eventList.concat(
+          // remove duplicated commits
+          events.filter(e => !commitShaList.has(e.push_data.commit_to)),
+        )
+
         for (const e of events) {
+          commitShaList.add(e.push_data.commit_to)
           projectIds.add(e.project_id)
         }
 
-        // eslint-disable-next-line unicorn/prefer-spread
-        eventList = eventList.concat(events)
         logger.debug(`gitlab integration eventList ${eventList.length} (added ${events.length} events)`)
 
         page += 1