fix: skip go finding output (#305)

* fix: ignore space * deploy: limit golangci-lint concurrency to avoid OOM * fix: ignore go finding output * deploy: limit golangci-lit
qiniu · Sep 1, 2024 · f4a0b59 · f4a0b59
1 parent c6a9a2b
commit f4a0b59
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 15 deletions.
diff --git a/config/linters-config/.golangci.yml b/config/linters-config/.golangci.yml
@@ -1,5 +1,9 @@
 # 这是我们当前内部使用的配置文件，会基于实际的变化和我们的认知而迭代，仅供参考
 
+run:
+  # control the resource usage of golangci-lint to avoid OOM
+  concurrency: 4
+
 linters-settings:
   paralleltest:
     # Ignore missing calls to `t.Parallel()` and only report incorrect uses of it.

diff --git a/deploy/reviewbot.yaml b/deploy/reviewbot.yaml
@@ -45,8 +45,8 @@ spec:
           imagePullPolicy: Always
           resources:
             requests:
-              memory: "8Gi"
-              cpu: "4"
+              memory: "10Gi"
+              cpu: "8"
           name: reviewbot
           ports:
             - containerPort: 8888

diff --git a/internal/linters/go/golangci_lint/golangci_lint.go b/internal/linters/go/golangci_lint/golangci_lint.go
@@ -102,7 +102,7 @@ func parser(log *xlog.Logger, output []byte) (map[string][]linters.LinterOutput,
 		}
 
 		// skip the go download log
-		if strings.Contains(ex, "go: downloading") {
+		if strings.Contains(ex, "go: downloading") || strings.Contains(ex, "go: finding") {
 			continue
 		}
 
@@ -134,11 +134,12 @@ func argsApply(log *xlog.Logger, a linters.Agent) linters.Agent {
 	newArgs := []string{"run"}
 
 	var (
-		timeoutFlag   bool
-		parallelFlag  bool
-		outFormatFlag bool
-		printFlag     bool
-		configFlag    bool
+		timeoutFlag     bool
+		parallelFlag    bool
+		outFormatFlag   bool
+		printFlag       bool
+		configFlag      bool
+		concurrencyFlag bool
 	)
 
 	for _, arg := range legacyArgs {
@@ -154,13 +155,15 @@ func argsApply(log *xlog.Logger, a linters.Agent) linters.Agent {
 			printFlag = true
 		case strings.HasPrefix(arg, "--config"):
 			configFlag = true
+		case strings.HasPrefix(arg, "--concurrency"):
+			concurrencyFlag = true
 		}
 
 		newArgs = append(newArgs, arg)
 	}
 
 	if !timeoutFlag {
-		newArgs = append(newArgs, "--timeout=5m0s")
+		newArgs = append(newArgs, "--timeout=15m0s")
 	}
 	if !parallelFlag {
 		newArgs = append(newArgs, "--allow-parallel-runners=true")
@@ -171,6 +174,9 @@ func argsApply(log *xlog.Logger, a linters.Agent) linters.Agent {
 	if !printFlag {
 		newArgs = append(newArgs, "--print-issued-lines=false")
 	}
+	if !concurrencyFlag {
+		newArgs = append(newArgs, "--concurrency=8")
+	}
 	if !configFlag && config.ConfigPath != "" {
 		config.ConfigPath = configApply(log, a)
 		newArgs = append(newArgs, "--config", config.ConfigPath)

diff --git a/internal/linters/go/golangci_lint/golangci_lint_test.go b/internal/linters/go/golangci_lint/golangci_lint_test.go
@@ -118,7 +118,7 @@ func TestArgs(t *testing.T) {
 				LinterConfig: config.Linter{
 					Enable:  &tp,
 					Command: []string{"golangci-lint"},
-					Args:    []string{"run", "--timeout=5m0s", "--allow-parallel-runners=true", "--out-format=line-number", "--print-issued-lines=false"},
+					Args:    []string{"run", "--timeout=15m0s", "--allow-parallel-runners=true", "--out-format=line-number", "--print-issued-lines=false", "--concurrency=8"},
 				},
 			},
 		},
@@ -135,7 +135,7 @@ func TestArgs(t *testing.T) {
 				LinterConfig: config.Linter{
 					Enable:  &tp,
 					Command: []string{"golangci-lint"},
-					Args:    []string{"run", "--timeout=10m", "--out-format=tab", "--config", "golangci-lint.yml", "--allow-parallel-runners=true", "--print-issued-lines=false"},
+					Args:    []string{"run", "--timeout=10m", "--out-format=tab", "--config", "golangci-lint.yml", "--allow-parallel-runners=true", "--print-issued-lines=false", "--concurrency=8"},
 				},
 			},
 		},
@@ -198,7 +198,7 @@ func TestArgs(t *testing.T) {
 				LinterConfig: config.Linter{
 					Enable:     &tp,
 					Command:    []string{"golangci-lint"},
-					Args:       []string{"run", "--timeout=5m0s", "--allow-parallel-runners=true", "--out-format=line-number", "--print-issued-lines=false", "--config", "config/golangci-lint.yml"},
+					Args:       []string{"run", "--timeout=15m0s", "--allow-parallel-runners=true", "--out-format=line-number", "--print-issued-lines=false", "--concurrency=8", "--config", "config/golangci-lint.yml"},
 					ConfigPath: "config/golangci-lint.yml",
 				},
 			},

diff --git a/internal/linters/linters.go b/internal/linters/linters.go
@@ -149,9 +149,11 @@ func GeneralHandler(log *xlog.Logger, a Agent, execRun func(a Agent) ([]byte, er
 	lintResults, unexpected := linterParser(log, output)
 	if len(unexpected) > 0 {
 		msg := lintersutil.LimitJoin(unexpected, 1000)
-		// just log the unexpected lines and notify the webhook, no need to return error
-		log.Warnf("unexpected lines: %v", msg)
-		metric.NotifyWebhookByText(ConstructUnknownMsg(linterName, a.PullRequestEvent.Repo.GetFullName(), a.PullRequestEvent.PullRequest.GetHTMLURL(), log.ReqId, msg))
+		if msg != "" {
+			// just log the unexpected lines and notify the webhook, no need to return error
+			log.Warnf("unexpected lines: %v", msg)
+			metric.NotifyWebhookByText(ConstructUnknownMsg(linterName, a.PullRequestEvent.Repo.GetFullName(), a.PullRequestEvent.PullRequest.GetHTMLURL(), log.ReqId, msg))
+		}
 	}
 
 	return Report(log, a, lintResults)