Merge branch 'master' of github.com:rudderlabs/rudder-server into cho…

…re.sec-scan

.github/workflows/builds.yml

@@ -107,7 +107,32 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+      - name: Build image (no-push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.build-config.platform }}
+          push: false
+          tags: ${{ matrix.build-config.tags }}
+          labels: ${{ matrix.build-config.labels }}
+          build-args: |
+            BUILD_DATE=${{ needs.docker-oss-meta.outputs.build-date }}
+            VERSION=${{ needs.docker-oss-meta.outputs.version }}
+            COMMIT_HASH=${{ github.sha }}
+            REVISION=${{ needs.docker-oss-meta.outputs.revision }}
+          outputs: type=docker,dest=${{ runner.temp }}/local-rudder-server.tar
+          cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp
+      - name: Run TruffleHog
+        run: |
+          docker pull trufflesecurity/trufflehog:latest
+          docker run --rm -v ${{ runner.temp }}:/tmp \
+            trufflesecurity/trufflehog:latest \
+            docker --image file:///tmp/local-rudder-server.tar \
+            --github-actions \
+            --no-verification \
+            --fail
       - name: Build and push
+        if: success()
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -120,7 +145,7 @@ jobs:
             VERSION=${{ needs.docker-oss-meta.outputs.version }}
             COMMIT_HASH=${{ github.sha }}
             REVISION=${{ needs.docker-oss-meta.outputs.revision }}
-
+          cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp
   docker-ent-meta:
     runs-on: ubuntu-latest
     outputs:
@@ -181,7 +206,33 @@ jobs:
         with:
           username: rudderlabs
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build image (no-push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.build-config.platform }}
+          push: false
+          tags: ${{ matrix.build-config.tags }}
+          labels: ${{ matrix.build-config.labels }}
+          build-args: |
+            BUILD_DATE=${{ needs.docker-ent-meta.outputs.build-date }}
+            VERSION=${{ needs.docker-ent-meta.outputs.version }}
+            COMMIT_HASH=${{ github.sha }}
+            REVISION=${{ needs.docker-ent-meta.outputs.revision }}
+            ENTERPRISE_TOKEN=${{ secrets.ENTERPRISE_TOKEN }}
+          outputs: type=docker,dest=${{ runner.temp }}/local-rudder-server.tar
+          cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp
+      - name: Run TruffleHog
+        run: |
+          docker pull trufflesecurity/trufflehog:latest
+          docker run --rm -v ${{ runner.temp }}:/tmp \
+            trufflesecurity/trufflehog:latest \
+            docker --image file:///tmp/local-rudder-server.tar \
+            --github-actions \
+            --no-verification \
+            --fail
       - name: Build and push
+        if: success()
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -195,6 +246,7 @@ jobs:
             COMMIT_HASH=${{ github.sha }}
             REVISION=${{ needs.docker-ent-meta.outputs.revision }}
             ENTERPRISE_TOKEN=${{ secrets.ENTERPRISE_TOKEN }}
+          cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp
   docker-sbsvc-meta:
     runs-on: ubuntu-latest
     outputs:
@@ -255,7 +307,33 @@ jobs:
         with:
           username: rudderlabs
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build image (no-push)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.build-config.platform }}
+          file: ./suppression-backup-service/Dockerfile
+          push: false
+          tags: ${{ matrix.build-config.tags }}
+          labels: ${{ matrix.build-config.labels }}
+          build-args: |
+            BUILD_DATE=${{ needs.docker-sbsvc-meta.outputs.build-date }}
+            VERSION=${{ needs.docker-sbsvc-meta.outputs.version }}
+            COMMIT_HASH=${{ github.sha }}
+            REVISION=${{ needs.docker-sbsvc-meta.outputs.revision }}
+          outputs: type=docker,dest=${{ runner.temp }}/suppression-backup-service.tar
+          cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp
+      - name: Run TruffleHog
+        run: |
+          docker pull trufflesecurity/trufflehog:latest
+          docker run --rm -v ${{ runner.temp }}:/tmp \
+            trufflesecurity/trufflehog:latest \
+            docker --image file:///tmp/suppression-backup-service.tar \
+            --github-actions \
+            --no-verification \
+            --fail
       - name: Build and push
+        if: success()
         uses: docker/build-push-action@v6
         with:
           context: .
@@ -269,6 +347,7 @@ jobs:
             VERSION=${{ needs.docker-sbsvc-meta.outputs.version }}
             COMMIT_HASH=${{ github.sha }}
             REVISION=${{ needs.docker-sbsvc-meta.outputs.revision }}
+          cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp
   create-manifest-docker-oss:
     runs-on: ubuntu-latest
     needs: [ docker-oss, docker-oss-meta ]

.github/workflows/verify.yml

@@ -77,5 +77,5 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.57.1
+          version: v1.60.3
           args: -v

Dockerfile

@@ -2,7 +2,7 @@
 # syntax=docker/dockerfile:1
 
 # GO_VERSION is updated automatically to match go.mod, see Makefile
-ARG GO_VERSION=1.22.5
+ARG GO_VERSION=1.23.0
 ARG ALPINE_VERSION=3.20
 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS builder
 ARG VERSION

Makefile

@@ -6,7 +6,7 @@ TESTFILE=_testok
 MOUNT_PATH=/local
 
 # go tools versions
-GOLANGCI=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.57.1
+GOLANGCI=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3
 gofumpt=mvdan.cc/gofumpt@latest
 govulncheck=golang.org/x/vuln/cmd/govulncheck@latest
 goimports=golang.org/x/tools/cmd/goimports@latest
@@ -16,7 +16,6 @@ protoc-gen-go=google.golang.org/protobuf/cmd/protoc-gen-go@v1.33.0
 protoc-gen-go-grpc=google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
 actionlint=github.com/rhysd/actionlint/cmd/actionlint@latest
 gitleaks=github.com/zricethezav/gitleaks/v8@v8.18.4
-govulncheck=golang.org/x/vuln/cmd/govulncheck@latest
 
 default: build
 
@@ -98,11 +97,12 @@ install-tools:
 	$(GO) install $(protoc-gen-go)
 	$(GO) install $(protoc-gen-go-grpc)
 	$(GO) install $(gotestsum)
+
 .PHONY: lint
-lint: fmt sec ## Run linters on all go files
+lint: fmt  ## Run linters on all go files
 	$(GO) run $(GOLANGCI) run -v
-	$(GO) run $(govulncheck) ./...
 	$(GO) run $(actionlint)
+	@$(MAKE) sec
 
 .PHONY: fmt
 fmt: install-tools ## Formats all go files
@@ -134,3 +134,8 @@ generate-openapi-spec: install-tools
 	  -g html2 \
 	  -o ${MOUNT_PATH}/gateway/openapi
 
+.PHONY: sec
+sec: ## Run security checks
+	$(GO) run $(govulncheck) ./...
+	./build/scan_docker.sh
+	./build/scan_docker.sh -f ./suppression-backup-service/Dockerfile

app/apphandlers/embeddedAppHandler.go

@@ -35,6 +35,7 @@ import (
 	"github.com/rudderlabs/rudder-server/services/fileuploader"
 	"github.com/rudderlabs/rudder-server/services/transformer"
 	"github.com/rudderlabs/rudder-server/services/transientsource"
+	"github.com/rudderlabs/rudder-server/utils/crash"
 	"github.com/rudderlabs/rudder-server/utils/misc"
 	"github.com/rudderlabs/rudder-server/utils/payload"
 	"github.com/rudderlabs/rudder-server/utils/types"
@@ -324,10 +325,10 @@ func (a *embeddedApp) StartRudderCore(ctx context.Context, options *app.Options)
 		return fmt.Errorf("drain config manager setup: %v", err)
 	}
 	defer drainConfigManager.Stop()
-	g.Go(misc.WithBugsnag(func() (err error) {
+	g.Go(crash.Wrapper(func() (err error) {
 		return drainConfigManager.DrainConfigRoutine(ctx)
 	}))
-	g.Go(misc.WithBugsnag(func() (err error) {
+	g.Go(crash.Wrapper(func() (err error) {
 		return drainConfigManager.CleanupRoutine(ctx)
 	}))
 	streamMsgValidator := stream.NewMessageValidator()

app/apphandlers/processorAppHandler.go

@@ -11,8 +11,6 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
-	"github.com/bugsnag/bugsnag-go/v2"
-
 	"github.com/rudderlabs/rudder-go-kit/config"
 	kithttputil "github.com/rudderlabs/rudder-go-kit/httputil"
 	"github.com/rudderlabs/rudder-go-kit/logger"
@@ -35,6 +33,7 @@ import (
 	"github.com/rudderlabs/rudder-server/services/fileuploader"
 	"github.com/rudderlabs/rudder-server/services/transformer"
 	"github.com/rudderlabs/rudder-server/services/transientsource"
+	"github.com/rudderlabs/rudder-server/utils/crash"
 	"github.com/rudderlabs/rudder-server/utils/misc"
 	"github.com/rudderlabs/rudder-server/utils/payload"
 	"github.com/rudderlabs/rudder-server/utils/types"
@@ -115,7 +114,7 @@ func (a *processorApp) StartRudderCore(ctx context.Context, options *app.Options
 	reporting := a.app.Features().Reporting.Setup(ctx, backendconfig.DefaultBackendConfig)
 	defer reporting.Stop()
 	syncer := reporting.DatabaseSyncer(types.SyncerConfig{ConnInfo: misc.GetConnectionString(config, "reporting")})
-	g.Go(misc.WithBugsnag(func() error {
+	g.Go(crash.Wrapper(func() error {
 		syncer()
 		return nil
 	}))
@@ -240,10 +239,10 @@ func (a *processorApp) StartRudderCore(ctx context.Context, options *app.Options
 		return fmt.Errorf("drain config manager setup: %v", err)
 	}
 	defer drainConfigManager.Stop()
-	g.Go(misc.WithBugsnag(func() (err error) {
+	g.Go(crash.Wrapper(func() (err error) {
 		return drainConfigManager.DrainConfigRoutine(ctx)
 	}))
-	g.Go(misc.WithBugsnag(func() (err error) {
+	g.Go(crash.Wrapper(func() (err error) {
 		return drainConfigManager.CleanupRoutine(ctx)
 	}))
 
@@ -351,7 +350,7 @@ func (a *processorApp) startHealthWebHandler(ctx context.Context, db *jobsdb.Han
 	srvMux.HandleFunc("/", app.LivenessHandler(db))
 	srv := &http.Server{
 		Addr:              ":" + strconv.Itoa(a.config.http.webPort),
-		Handler:           bugsnag.Handler(srvMux),
+		Handler:           crash.Handler(srvMux),
 		ReadTimeout:       a.config.http.ReadTimeout,
 		ReadHeaderTimeout: a.config.http.ReadHeaderTimeout,
 		WriteTimeout:      a.config.http.WriteTimeout,

backend-config/backend_config_test.go

@@ -61,6 +61,48 @@ var sampleBackendConfig = ConfigT{
 	},
 }
 
+var sampleConfigWithConnection = ConfigT{
+	WorkspaceID: sampleWorkspaceID,
+	Sources: []SourceT{
+		{
+			ID:       "1",
+			WriteKey: "d",
+			Enabled:  false,
+		}, {
+			ID:       "2",
+			WriteKey: "d2",
+			Enabled:  false,
+			Destinations: []DestinationT{
+				{
+					ID:                 "d1",
+					Name:               "processor Disabled",
+					IsProcessorEnabled: false,
+				}, {
+					ID:                 "d2",
+					Name:               "processor Enabled",
+					IsProcessorEnabled: true,
+				},
+			},
+		},
+	},
+	Connections: map[string]Connection{
+		"1": {
+			SourceID:         "2",
+			DestinationID:    "d1",
+			Enabled:          true,
+			Config:           map[string]interface{}{"key": "value"},
+			ProcessorEnabled: false,
+		},
+		"2": {
+			SourceID:         "2",
+			DestinationID:    "d2",
+			Enabled:          true,
+			Config:           map[string]interface{}{"key2": "value2"},
+			ProcessorEnabled: true,
+		},
+	},
+}
+
 // This configuration is assumed by all gateway tests and, is returned on Subscribe of mocked backend config
 var sampleFilteredSources = ConfigT{
 	Sources: []SourceT{
@@ -264,6 +306,37 @@ func TestConfigUpdate(t *testing.T) {
 		require.Equal(t, (<-chProcess).Data, map[string]ConfigT{workspaces: sampleFilteredSources})
 		require.Equal(t, (<-chBackend).Data, map[string]ConfigT{workspaces: sampleBackendConfig})
 	})
+
+	t.Run("new config with connections", func(t *testing.T) {
+		var (
+			ctrl        = gomock.NewController(t)
+			ctx, cancel = context.WithCancel(context.Background())
+			workspaces  = "foo"
+			cacheStore  = cache.NewMockCache(ctrl)
+		)
+		defer ctrl.Finish()
+		defer cancel()
+
+		wc := NewMockworkspaceConfig(ctrl)
+		wc.EXPECT().Get(gomock.Eq(ctx)).Return(map[string]ConfigT{workspaces: sampleConfigWithConnection}, nil).Times(1)
+
+		var pubSub pubsub.PublishSubscriber
+		bc := &backendConfigImpl{
+			eb:              &pubSub,
+			workspaceConfig: wc,
+			cache:           cacheStore,
+		}
+		bc.curSourceJSON = map[string]ConfigT{workspaces: sampleBackendConfig2}
+
+		chProcess := pubSub.Subscribe(ctx, string(TopicProcessConfig))
+		chBackend := pubSub.Subscribe(ctx, string(TopicBackendConfig))
+
+		bc.configUpdate(ctx)
+		require.True(t, bc.initialized)
+		require.Equal(t, (<-chProcess).Data, map[string]ConfigT{workspaces: sampleFilteredSources})
+		require.Equal(t, (<-chBackend).Data, map[string]ConfigT{workspaces: sampleConfigWithConnection})
+		require.Equal(t, bc.curSourceJSON[workspaces].Connections, sampleConfigWithConnection.Connections)
+	})
 }
 
 func TestFilterProcessorEnabledDestinations(t *testing.T) {

backend-config/types.go

@@ -94,6 +94,15 @@ type ConfigT struct {
 	Settings        Settings                     `json:"settings"`
 	UpdatedAt       time.Time                    `json:"updatedAt"`
 	Credentials     map[string]Credential        `json:"credentials"`
+	Connections     map[string]Connection        `json:"connections"`
+}
+
+type Connection struct {
+	SourceID         string                 `json:"sourceId"`
+	DestinationID    string                 `json:"destinationId"`
+	Enabled          bool                   `json:"enabled"`
+	Config           map[string]interface{} `json:"config"`
+	ProcessorEnabled bool                   `json:"processorEnabled"`
 }
 
 func (c *ConfigT) SourcesMap() map[string]*SourceT {