Fix tcp stacking and concurrency handling

.github/workflows/main.yml → .github/workflows/goreleaser.yml

@@ -15,22 +15,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: set up go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.18
-
-      - name: cache go modules
-        uses: actions/cache@v1
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: ">=1.20"
 
       - name: tests modules
         run: |
@@ -51,13 +43,13 @@ jobs:
           go test -v ./...
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: run goreleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v4
         if: success() && startsWith(github.ref, 'refs/tags/')
         with:
           version: latest

.github/workflows/linter.yml

@@ -0,0 +1,22 @@
+name: linter
+
+on: push
+
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: set up go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ">=1.20"
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --config .golangci.yml

.github/workflows/stale-issue-pr.yml

@@ -0,0 +1,26 @@
+name: stale-issue-pr
+
+on:
+  schedule:
+    - cron: '30 1 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Comment or this will be closed in 5 days.'
+          stale-pr-message: 'This PR is stale because it has been open 60 days with no activity. Comment or this will be closed in 5 days.'
+          close-issue-message: 'This issue was automatically closed because it has been stalled for 5 days with no activity.'
+          close-pr-message: 'This PR was automatically closed because it has been stalled for 5 days with no activity.'
+          days-before-stale: 60
+          days-before-close: 5
+          exempt-all-pr-assignees: true
+          operations-per-run: 60

.github/workflows/tests.yml

@@ -0,0 +1,22 @@
+name: tests
+
+
+on: push
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: set up go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ">=1.20"
+
+      - name: running unit-tests
+        run: |
+          go test -v ./...

.golangci.yml

@@ -0,0 +1,35 @@
+run:
+  timeout: 30m
+output:
+  format: line-number
+linters:
+  disable-all: false
+  enable:
+  - bodyclose
+  - depguard
+  - dogsled
+  #- dupl
+  - errcheck
+  - exportloopref
+  #- funlen
+  - gocognit
+  - goconst
+  - gocritic
+  - godox
+  - gofmt
+  - goimports
+  - gosec
+  - gosimple
+  - govet
+  - ineffassign
+  - misspell
+  - nakedret
+  - revive
+  - staticcheck
+  - stylecheck
+  - typecheck
+  - unconvert
+  - unparam
+  - unused
+  - whitespace
+  - reassign

Makefile

@@ -0,0 +1,12 @@
+all: test
+
+test:
+	@go test -v ./...
+
+check:
+	@golangci-lint run --config .golangci.yml
+
+coverage:
+	@go test -cover -coverprofile cover.out -v ./...
+	@go tool cover -func=cover.out
+	@rm -f cover.out

collectors/applications.go

@@ -10,11 +10,6 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-const (
-	concurrentOrganizationsGoroutines = 10
-	concurrentSpacesGoroutines        = 10
-)
-
 type ApplicationsCollector struct {
 	namespace                                   string
 	environment                                 string
@@ -160,12 +155,11 @@ func NewApplicationsCollector(
 
 func (c ApplicationsCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
 	errorMetric := float64(0)
-	err := objs.Error
 	if objs.Error != nil {
 		errorMetric = float64(1)
 		c.applicationsScrapeErrorsTotalMetric.Inc()
 	} else {
-		err = c.reportApplicationsMetrics(objs, ch)
+		err := c.reportApplicationsMetrics(objs, ch)
 		if err != nil {
 			log.Error(err)
 			errorMetric = float64(1)
@@ -198,12 +192,12 @@ func (c ApplicationsCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 // reportApplicationsMetrics
-// 1. empty detected buildpacks for apps running on droplet
-//    staged with a buildpack that is no mot available
-//    fallback to buildpack field for compatibility with v0
-// 2. symmetrically in some corner cases, buildpack is null but
-//    detected_buildpack is available. Use detected_buildpack
-//    for compatibility with v0
+//  1. empty detected buildpacks for apps running on droplet
+//     staged with a buildpack that is no mot available
+//     fallback to buildpack field for compatibility with v0
+//  2. symmetrically in some corner cases, buildpack is null but
+//     detected_buildpack is available. Use detected_buildpack
+//     for compatibility with v0
 func (c ApplicationsCollector) reportApplicationsMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) error {
 	c.applicationInfoMetric.Reset()
 	c.applicationInstancesMetric.Reset()
@@ -223,21 +217,21 @@ func (c ApplicationsCollector) reportApplicationsMetrics(objs *models.CFObjects,
 			}
 		}
 
-		spaceGuid, ok := application.Relationships[constant.RelationshipTypeSpace]
+		spaceRel, ok := application.Relationships[constant.RelationshipTypeSpace]
 		if !ok {
 			return fmt.Errorf("could not find space relation in application '%s'", application.GUID)
 		}
-		space, ok := objs.Spaces[spaceGuid.GUID]
+		space, ok := objs.Spaces[spaceRel.GUID]
 		if !ok {
-			return fmt.Errorf("could not find space with guid '%s'", spaceGuid.GUID)
+			return fmt.Errorf("could not find space with guid '%s'", spaceRel.GUID)
 		}
-		orgGuid, ok := space.Relationships[constant.RelationshipTypeOrganization]
+		orgRel, ok := space.Relationships[constant.RelationshipTypeOrganization]
 		if !ok {
 			return fmt.Errorf("could not find org relation in space '%s'", space.GUID)
 		}
-		organization, ok := objs.Orgs[orgGuid.GUID]
+		organization, ok := objs.Orgs[orgRel.GUID]
 		if !ok {
-			return fmt.Errorf("could not find org with guid '%s'", orgGuid.GUID)
+			return fmt.Errorf("could not find org with guid '%s'", orgRel.GUID)
 		}
 		appSum, ok := objs.AppSummaries[application.GUID]
 		if !ok {

collectors/buildpacks.go

@@ -2,8 +2,9 @@ package collectors
 
 import (
 	"time"
-	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/bosh-prometheus/cf_exporter/models"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 type BuildpacksCollector struct {

collectors/collectors.go

@@ -1,9 +1,9 @@
 package collectors
 
 import (
-	"github.com/bosh-prometheus/cf_exporter/models"
-	"github.com/bosh-prometheus/cf_exporter/filters"
 	"github.com/bosh-prometheus/cf_exporter/fetcher"
+	"github.com/bosh-prometheus/cf_exporter/filters"
+	"github.com/bosh-prometheus/cf_exporter/models"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -13,22 +13,24 @@ type ObjectCollector interface {
 }
 
 type Collector struct {
-	fetcher *fetcher.Fetcher
-	filter *filters.Filter
+	workers    int
+	config     *fetcher.CFConfig
+	filter     *filters.Filter
 	collectors []ObjectCollector
 }
 
 func NewCollector(
 	namespace string,
 	environment string,
 	deployment string,
-	fetcher *fetcher.Fetcher,
+	workers int,
+	config *fetcher.CFConfig,
 	filter *filters.Filter,
 ) (*Collector, error) {
-
 	res := &Collector{
-		fetcher: fetcher,
-		filter: filter,
+		workers:    workers,
+		config:     config,
+		filter:     filter,
 		collectors: []ObjectCollector{},
 	}
 
@@ -100,9 +102,9 @@ func NewCollector(
 	return res, nil
 }
 
-
 func (c *Collector) Collect(ch chan<- prometheus.Metric) {
-	objs := c.fetcher.GetObjects()
+	fetcher := fetcher.NewFetcher(c.workers, c.config, c.filter)
+	objs := fetcher.GetObjects()
 	for _, collector := range c.collectors {
 		collector.Collect(objs, ch)
 	}

collectors/events.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/bosh-prometheus/cf_exporter/models"
 	"github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
 )
 
 type EventsCollector struct {
@@ -18,8 +17,8 @@ type EventsCollector struct {
 	lastEventsScrapeErrorMetric           prometheus.Gauge
 	lastEventsScrapeTimestampMetric       prometheus.Gauge
 	lastEventsScrapeDurationSecondsMetric prometheus.Gauge
-	lastCheckFilter time.Time
-	timeLocation    *time.Location
+	lastCheckFilter                       time.Time
+	timeLocation                          *time.Location
 }
 
 func NewEventsCollector(
@@ -108,17 +107,11 @@ func NewEventsCollector(
 
 func (c *EventsCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
 	errorMetric := float64(0)
-	err := objs.Error
 	if objs.Error != nil {
 		errorMetric = float64(1)
 		c.eventsScrapeErrorsTotalMetric.Inc()
 	} else {
-		err = c.reportEventsMetrics(objs, ch)
-		if err != nil {
-			log.Error(err)
-			errorMetric = float64(1)
-			c.eventsScrapeErrorsTotalMetric.Inc()
-		}
+		c.reportEventsMetrics(objs, ch)
 	}
 
 	c.eventsScrapeErrorsTotalMetric.Collect(ch)
@@ -143,7 +136,7 @@ func (c *EventsCollector) Describe(ch chan<- *prometheus.Desc) {
 
 // reportEventsMetrics
 // 1. find user's username in user map if available
-func (c *EventsCollector) reportEventsMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) error {
+func (c *EventsCollector) reportEventsMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) {
 	c.eventsInfoMetric.Reset()
 
 	for _, event := range objs.Events {
@@ -174,5 +167,4 @@ func (c *EventsCollector) reportEventsMetrics(objs *models.CFObjects, ch chan<-
 	timeLocation, _ := time.LoadLocation("UTC")
 	c.lastCheckFilter = time.Now().In(timeLocation)
 	c.eventsInfoMetric.Collect(ch)
-	return nil
 }

collectors/isolation_segments.go

@@ -2,8 +2,9 @@ package collectors
 
 import (
 	"time"
-	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/bosh-prometheus/cf_exporter/models"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 type IsolationSegmentsCollector struct {

collectors/organizations.go

@@ -232,12 +232,11 @@ func NewOrganizationsCollector(
 
 func (c OrganizationsCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
 	errorMetric := float64(0)
-	err := objs.Error
 	if objs.Error != nil {
 		errorMetric = float64(1)
 		c.organizationsScrapeErrorsTotalMetric.Inc()
 	} else {
-		err = c.reportOrganizationsMetrics(objs, ch)
+		err := c.reportOrganizationsMetrics(objs, ch)
 		if err != nil {
 			log.Error(err)
 			errorMetric = float64(1)