Secrets cache

.github/workflows/build.yml

@@ -0,0 +1,27 @@
+name: Go
+
+on:
+  push:
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.22'
+
+      - name: Compile
+        run: |
+          cd collector
+          go build -v ./...
+
+      - name: Test
+        run: |
+          cd collector
+          go test -v ./...
+

collector/build/Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.22-alpine as builder
+
+WORKDIR /app
+COPY go.mod ./
+RUN go mod download
+COPY . ./
+RUN go build cmd/collector/database-collector.go
+
+
+FROM debian:stable-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY --from=builder /app/database-collector /app/database-collector
+ENTRYPOINT ["/app/database-collector"]

collector/cmd/collector/database-collector.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/promlog"
+	"github.com/robfig/cron/v3"
+	"github.com/truemark/database-collector/exporters/mysql"
+	"github.com/truemark/database-collector/exporters/oracle"
+	"github.com/truemark/database-collector/exporters/postgres"
+	"github.com/truemark/database-collector/internal/aws"
+	"github.com/truemark/database-collector/internal/utils"
+)
+
+var registry = prometheus.NewRegistry()
+
+func collectMetrics(secretValueMap map[string]interface{}, engine string, logger log.Logger, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	registry := prometheus.NewRegistry()
+	switch engine {
+	case "mysql":
+		mysql.RegisterMySQLCollector(registry, secretValueMap, logger)
+	case "postgres":
+		postgres.RegisterPostgresCollector(registry, secretValueMap, logger)
+	case "oracle":
+		oracle.RegisterOracleDBCollector(registry, secretValueMap, logger)
+	}
+
+	metricFamilies, err := registry.Gather()
+	if err != nil {
+		level.Error(logger).Log("msg", "Error gathering metrics", "err", err)
+		return
+	}
+
+	response, err := utils.ConvertMetricFamilyToTimeSeries(metricFamilies, secretValueMap["host"].(string))
+	if err != nil {
+		fmt.Println("Failed to send metrics to APS", err)
+	} else {
+		fmt.Println("Successfully sent metrics to APS ", response)
+	}
+}
+
+func HandleRequest() {
+	promlogConfig := &promlog.Config{
+		Level: &promlog.AllowedLevel{},
+	}
+	if err := promlogConfig.Level.Set("info"); err != nil {
+		fmt.Println("Error setting log level:", err)
+		return
+	}
+
+	logger := promlog.New(promlogConfig)
+
+	level.Info(logger).Log("msg", "Starting database collector")
+
+	listSecretsResult := aws.ListSecrets()
+	var wg sync.WaitGroup
+
+	for i := 0; i < len(listSecretsResult.SecretList); i++ {
+		secretValue := aws.GetSecretsValue(*listSecretsResult.SecretList[i].Name)
+		secretValueMap := map[string]interface{}{}
+		err := json.Unmarshal([]byte(secretValue), &secretValueMap)
+		if err != nil {
+			fmt.Println(err)
+			continue
+		}
+
+		engine := secretValueMap["engine"].(string)
+		wg.Add(1)
+		go collectMetrics(secretValueMap, engine, logger, &wg)
+	}
+
+	wg.Wait()
+}
+
+func main() {
+	mode := os.Getenv("RUN_MODE")
+	if mode == "LAMBDA" {
+		// Run as AWS Lambda function
+		lambda.Start(HandleRequest)
+	} else if mode == "CRON" {
+		fmt.Println("Starting in CRON mode...")
+
+		// Run as internal cron job
+		c := cron.New()
+		_, err := c.AddFunc("@every 1m", HandleRequest) // Run CronJob every minute
+		if err != nil {
+			fmt.Println("Error setting up cron job:", err)
+			return
+		}
+		c.Start()
+
+		// Keep the program running
+		select {}
+	} else {
+		fmt.Println("Invalid RUN_MODE. Set RUN_MODE to either 'LAMBDA' or 'CRON'")
+	}
+}

collector/cmd/events-collector/events-collector.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/aws/aws-lambda-go/events"
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/truemark/database-collector/internal/utils"
+)
+
+type Event struct {
+	Detail    json.RawMessage `json:"Detail"`
+	AccountID string          `json:"AccountID"`
+}
+
+type RdsEventMessage struct {
+	EventCategories  []string `json:"EventCategories"`
+	SourceType       string   `json:"SourceType"`
+	SourceArn        string   `json:"SourceArn"`
+	Date             string   `json:"Date"`
+	SourceIdentifier string   `json:"SourceIdentifier"`
+	Message          string   `json:"Message"`
+	EventID          string   `json:"EventID"`
+}
+
+var EventsCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "rds_service_events",
+		Help: "This metric indicates on whats happening on various aws services, e.g RDS",
+	},
+	[]string{"event_id", "event_message", "event_source"},
+)
+
+func handler(ctx context.Context, e events.CloudWatchEvent) {
+	registry := prometheus.NewRegistry()
+	registry.Unregister(prometheus.NewGoCollector())
+	registry.Unregister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+	registry.MustRegister(EventsCounter)
+	event := RdsEventMessage{}
+	print(string(e.Detail))
+	err := json.Unmarshal(e.Detail, &event)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(event.EventID)
+	if len(event.EventID) == 1 {
+		EventsCounter.WithLabelValues("none", event.Message, event.SourceIdentifier).Inc()
+	} else {
+		EventsCounter.WithLabelValues(event.EventID, event.Message, event.SourceIdentifier).Inc()
+	}
+	print(EventsCounter)
+	gatherers := prometheus.Gatherers{
+		registry,
+	}
+	metricFamilies, err := gatherers.Gather()
+	response, err := utils.ConvertMetricFamilyToTimeSeries(metricFamilies, event.EventID)
+	if err != nil {
+		fmt.Println(err, "Failed to convert metric family to time series")
+	} else {
+		fmt.Println("Successfully sent metrics to APS", response)
+	}
+}
+
+func main() {
+	lambda.Start(handler)
+}