Merge branch 'main' into ts/repocop-memory

guardian · Dec 4, 2024 · 000c115 · 000c115
2 parents 116a24d + 962cac0
commit 000c115
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 6 deletions.
diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap
@@ -14252,6 +14252,9 @@ spec:
   version: v11.11.1
   tables:
     - github_issues
+  skip_tables:
+    - github_issue_timeline_events
+    - github_issue_pullrequest_reviews
   destinations:
     - postgresql
   spec:
@@ -14295,7 +14298,7 @@ spec:
             "Environment": [
               {
                 "Name": "GOMEMLIMIT",
-                "Value": "819MiB",
+                "Value": "1638MiB",
               },
             ],
             "Essential": true,
@@ -14602,7 +14605,7 @@ spec:
           ],
         },
         "Family": "ServiceCatalogueCloudquerySourceGitHubIssuesTaskDefinition750BB414",
-        "Memory": "1024",
+        "Memory": "2048",
         "NetworkMode": "awsvpc",
         "RequiresCompatibilities": [
           "FARGATE",
@@ -15428,7 +15431,7 @@ spec:
             "Environment": [
               {
                 "Name": "GOMEMLIMIT",
-                "Value": "819MiB",
+                "Value": "1638MiB",
               },
             ],
             "Essential": true,
@@ -15735,7 +15738,7 @@ spec:
           ],
         },
         "Family": "ServiceCatalogueCloudquerySourceGitHubRepositoriesTaskDefinition13A7DF48",
-        "Memory": "1024",
+        "Memory": "2048",
         "NetworkMode": "awsvpc",
         "RequiresCompatibilities": [
           "FARGATE",

diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts
@@ -406,7 +406,7 @@ export function addCloudqueryEcsCluster(
 			}),
 			secrets: githubSecrets,
 			additionalCommands: additionalGithubCommands,
-			memoryLimitMiB: 1024,
+			memoryLimitMiB: 2048,
 		},
 		{
 			name: 'GitHubTeams',
@@ -447,10 +447,20 @@ export function addCloudqueryEcsCluster(
 			config: githubSourceConfig({
 				org: gitHubOrgName,
 				tables: ['github_issues'],
+				skipTables: [
+					/*
+          These tables are children of github_issues.
+          ServiceCatalogue collects child tables automatically.
+          We don't use them as they take a long time to collect, so skip them.
+          See https://www.cloudquery.io/docs/advanced-topics/performance-tuning#improve-performance-by-skipping-relations
+           */
+					'github_issue_timeline_events',
+					'github_issue_pullrequest_reviews',
+				],
 			}),
 			secrets: githubSecrets,
 			additionalCommands: additionalGithubCommands,
-			memoryLimitMiB: 1024,
+			memoryLimitMiB: 2048,
 		},
 	];