From 32cbfffa0c0c2b3840fe1045e3c21805872947a7 Mon Sep 17 00:00:00 2001 From: Jon Date: Thu, 16 Nov 2023 22:51:46 -0500 Subject: [PATCH] [receiver/mongodbatlasreceiver] add metrics project config (#28866) **Description:** This feature adds a Project Config for the metrics to filter by Project name and or clusters. **Link to tracking Issue:** #28865 **Testing:** - Added test for cluster filtering - Tested project name alone, project name with IncludeClusters and project name with ExcludeClusters on a live environment with success. **Documentation:** Added optional project config fields to README --------- Co-authored-by: Daniel Jaglowski --- ...asreceiver-add-metrics-project-config.yaml | 27 ++++ receiver/mongodbatlasreceiver/README.md | 5 + receiver/mongodbatlasreceiver/config.go | 7 ++ receiver/mongodbatlasreceiver/config_test.go | 41 +++++++ receiver/mongodbatlasreceiver/receiver.go | 116 ++++++++++++++---- .../mongodbatlasreceiver/receiver_test.go | 66 ++++++++++ 6 files changed, 239 insertions(+), 23 deletions(-) create mode 100755 .chloggen/mongodbatlasreceiver-add-metrics-project-config.yaml diff --git a/.chloggen/mongodbatlasreceiver-add-metrics-project-config.yaml b/.chloggen/mongodbatlasreceiver-add-metrics-project-config.yaml new file mode 100755 index 000000000000..0e8208e25909 --- /dev/null +++ b/.chloggen/mongodbatlasreceiver-add-metrics-project-config.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: receiver/mongodbatlasreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: adds project config to mongodbatlas metrics to filter by project name and clusters. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [28865] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/mongodbatlasreceiver/README.md b/receiver/mongodbatlasreceiver/README.md index 93f675553254..29118d287b6c 100644 --- a/receiver/mongodbatlasreceiver/README.md +++ b/receiver/mongodbatlasreceiver/README.md @@ -42,6 +42,11 @@ MongoDB Atlas [Documentation](https://www.mongodb.com/docs/atlas/reference/api/l - `granularity` (default `PT1M` - See [MongoDB Atlas Documentation](https://docs.atlas.mongodb.com/reference/api/process-measurements/)) - `collection_interval` (default `3m`) This receiver collects metrics on an interval. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. - `storage` (optional) The component ID of a storage extension which can be used when polling for `alerts` or `events` . The storage extension prevents duplication of data after a collector restart by remembering which data were previously collected. +- `projects` (optional for metrics) a slice of projects this receiver collects metrics from instead of all projects in an organization + - `name` Name of the project to discover metrics from + - `include_clusters` (default empty, exclusive with `exclude_clusters`) + - `exclude_clusters` (default empty, exclusive with `include_clusters`) + - If both `include_clusters` and `exclude_clusters` are empty, then all clusters in the project will be included - `retry_on_failure` - `enabled` (default true) - `initial_interval` (default 5s) diff --git a/receiver/mongodbatlasreceiver/config.go b/receiver/mongodbatlasreceiver/config.go index b2d6d3c81010..ae159d65044a 100644 --- a/receiver/mongodbatlasreceiver/config.go +++ b/receiver/mongodbatlasreceiver/config.go @@ -28,6 +28,7 @@ type Config struct { PrivateKey configopaque.String `mapstructure:"private_key"` Granularity string `mapstructure:"granularity"` MetricsBuilderConfig metadata.MetricsBuilderConfig `mapstructure:",squash"` + Projects []*ProjectConfig `mapstructure:"projects"` Alerts AlertConfig `mapstructure:"alerts"` Events *EventsConfig `mapstructure:"events"` Logs LogConfig `mapstructure:"logs"` @@ -133,6 +134,12 @@ var ( func (c *Config) Validate() error { var errs error + for _, project := range c.Projects { + if len(project.ExcludeClusters) != 0 && len(project.IncludeClusters) != 0 { + errs = multierr.Append(errs, errClusterConfig) + } + } + errs = multierr.Append(errs, c.Alerts.validate()) errs = multierr.Append(errs, c.Logs.validate()) if c.Events != nil { diff --git a/receiver/mongodbatlasreceiver/config_test.go b/receiver/mongodbatlasreceiver/config_test.go index 9148e7357ac8..183d58290053 100644 --- a/receiver/mongodbatlasreceiver/config_test.go +++ b/receiver/mongodbatlasreceiver/config_test.go @@ -116,6 +116,47 @@ func TestValidate(t *testing.T) { }, expectedErr: errNoCert.Error(), }, + { + name: "Valid Metrics Config", + input: Config{ + Projects: []*ProjectConfig{ + { + Name: "Project1", + }, + }, + ScraperControllerSettings: scraperhelper.NewDefaultScraperControllerSettings(metadata.Type), + }, + }, + { + name: "Valid Metrics Config with multiple projects with an inclusion or exclusion", + input: Config{ + Projects: []*ProjectConfig{ + { + Name: "Project1", + IncludeClusters: []string{"Cluster1"}, + }, + { + Name: "Project2", + ExcludeClusters: []string{"Cluster1"}, + }, + }, + ScraperControllerSettings: scraperhelper.NewDefaultScraperControllerSettings(metadata.Type), + }, + }, + { + name: "invalid Metrics Config", + input: Config{ + Projects: []*ProjectConfig{ + { + Name: "Project1", + IncludeClusters: []string{"Cluster1"}, + ExcludeClusters: []string{"Cluster2"}, + }, + }, + ScraperControllerSettings: scraperhelper.NewDefaultScraperControllerSettings(metadata.Type), + }, + expectedErr: errClusterConfig.Error(), + }, { name: "Valid Logs Config", input: Config{ diff --git a/receiver/mongodbatlasreceiver/receiver.go b/receiver/mongodbatlasreceiver/receiver.go index 72723aa552b0..f6a97bae439f 100644 --- a/receiver/mongodbatlasreceiver/receiver.go +++ b/receiver/mongodbatlasreceiver/receiver.go @@ -37,6 +37,10 @@ type timeconstraints struct { func newMongoDBAtlasReceiver(settings receiver.CreateSettings, cfg *Config) *mongodbatlasreceiver { client := internal.NewMongoDBAtlasClient(cfg.PublicKey, string(cfg.PrivateKey), cfg.RetrySettings, settings.Logger) + for _, p := range cfg.Projects { + p.populateIncludesAndExcludes() + } + return &mongodbatlasreceiver{ log: settings.Logger, cfg: cfg, @@ -77,47 +81,113 @@ func (s *mongodbatlasreceiver) shutdown(context.Context) error { return s.client.Shutdown() } +// poll decides whether to poll all projects or a specific project based on the configuration. func (s *mongodbatlasreceiver) poll(ctx context.Context, time timeconstraints) error { + if len(s.cfg.Projects) == 0 { + return s.pollAllProjects(ctx, time) + } + return s.pollProjects(ctx, time) +} + +// pollAllProjects handles polling across all projects within the organizations. +func (s *mongodbatlasreceiver) pollAllProjects(ctx context.Context, time timeconstraints) error { orgs, err := s.client.Organizations(ctx) if err != nil { return fmt.Errorf("error retrieving organizations: %w", err) } for _, org := range orgs { - projects, err := s.client.Projects(ctx, org.ID) + proj, err := s.client.Projects(ctx, org.ID) if err != nil { - return fmt.Errorf("error retrieving projects: %w", err) + s.log.Error("error retrieving projects", zap.String("orgID", org.ID), zap.Error(err)) + continue } - for _, project := range projects { - nodeClusterMap, providerMap, err := s.getNodeClusterNameMap(ctx, project.ID) - if err != nil { - return fmt.Errorf("error collecting clusters from project %s: %w", project.ID, err) + for _, project := range proj { + // Since there is no specific ProjectConfig for these projects, pass nil. + if err := s.processProject(ctx, time, org.Name, project, nil); err != nil { + s.log.Error("error processing project", zap.String("projectID", project.ID), zap.Error(err)) } + } + } + return nil +} - processes, err := s.client.Processes(ctx, project.ID) - if err != nil { - return fmt.Errorf("error retrieving MongoDB Atlas processes for project %s: %w", project.ID, err) - } - for _, process := range processes { - clusterName := nodeClusterMap[process.UserAlias] - providerValues := providerMap[clusterName] +// pollProject handles polling for specific projects as configured. +func (s *mongodbatlasreceiver) pollProjects(ctx context.Context, time timeconstraints) error { + for _, projectCfg := range s.cfg.Projects { + project, err := s.client.GetProject(ctx, projectCfg.Name) + if err != nil { + s.log.Error("error retrieving project", zap.String("projectName", projectCfg.Name), zap.Error(err)) + continue + } - if err := s.extractProcessMetrics(ctx, time, org.Name, project, process, clusterName, providerValues); err != nil { - return fmt.Errorf("error when polling process metrics from MongoDB Atlas for process %s: %w", process.ID, err) - } + org, err := s.client.GetOrganization(ctx, project.OrgID) + if err != nil { + s.log.Error("error retrieving organization from project", zap.String("projectName", projectCfg.Name), zap.Error(err)) + continue + } - if err := s.extractProcessDatabaseMetrics(ctx, time, org.Name, project, process, clusterName, providerValues); err != nil { - return fmt.Errorf("error when polling process database metrics from MongoDB Atlas for process %s: %w", process.ID, err) - } + if err := s.processProject(ctx, time, org.Name, project, projectCfg); err != nil { + s.log.Error("error processing project", zap.String("projectID", project.ID), zap.Error(err)) + } + } + return nil +} - if err := s.extractProcessDiskMetrics(ctx, time, org.Name, project, process, clusterName, providerValues); err != nil { - return fmt.Errorf("error when polling process disk metrics from MongoDB Atlas for process %s: %w", process.ID, err) - } - } +func (s *mongodbatlasreceiver) processProject(ctx context.Context, time timeconstraints, orgName string, project *mongodbatlas.Project, projectCfg *ProjectConfig) error { + nodeClusterMap, providerMap, err := s.getNodeClusterNameMap(ctx, project.ID) + if err != nil { + return fmt.Errorf("error collecting clusters from project %s: %w", project.ID, err) + } + + processes, err := s.client.Processes(ctx, project.ID) + if err != nil { + return fmt.Errorf("error retrieving MongoDB Atlas processes for project %s: %w", project.ID, err) + } + + for _, process := range processes { + clusterName := nodeClusterMap[process.UserAlias] + providerValues := providerMap[clusterName] + + if !shouldProcessCluster(projectCfg, clusterName) { + // Skip processing for this cluster + continue + } + + if err := s.extractProcessMetrics(ctx, time, orgName, project, process, clusterName, providerValues); err != nil { + return fmt.Errorf("error when polling process metrics from MongoDB Atlas for process %s: %w", process.ID, err) + } + + if err := s.extractProcessDatabaseMetrics(ctx, time, orgName, project, process, clusterName, providerValues); err != nil { + return fmt.Errorf("error when polling process database metrics from MongoDB Atlas for process %s: %w", process.ID, err) + } + + if err := s.extractProcessDiskMetrics(ctx, time, orgName, project, process, clusterName, providerValues); err != nil { + return fmt.Errorf("error when polling process disk metrics from MongoDB Atlas for process %s: %w", process.ID, err) } } + return nil } +// shouldProcessCluster checks whether a given cluster should be processed based on the project configuration. +func shouldProcessCluster(projectCfg *ProjectConfig, clusterName string) bool { + if projectCfg == nil { + // If there is no project config, process all clusters. + return true + } + + _, isIncluded := projectCfg.includesByClusterName[clusterName] + _, isExcluded := projectCfg.excludesByClusterName[clusterName] + + // Return false immediately if the cluster is excluded. + if isExcluded { + return false + } + + // If IncludeClusters is empty, or the cluster is explicitly included, return true. + return len(projectCfg.IncludeClusters) == 0 || isIncluded +} + type providerValues struct { RegionName string ProviderName string diff --git a/receiver/mongodbatlasreceiver/receiver_test.go b/receiver/mongodbatlasreceiver/receiver_test.go index 88d9d05a7a93..c592890f3984 100644 --- a/receiver/mongodbatlasreceiver/receiver_test.go +++ b/receiver/mongodbatlasreceiver/receiver_test.go @@ -71,3 +71,69 @@ func TestTimeConstraints(t *testing.T) { t.Run(testCase.name, testCase.run) } } + +func TestShouldProcessCluster(t *testing.T) { + tests := []struct { + name string + projectCfg *ProjectConfig + clusterName string + want bool + }{ + { + name: "included cluster should be processed", + projectCfg: &ProjectConfig{ + IncludeClusters: []string{"Cluster1"}, + }, + clusterName: "Cluster1", + want: true, + }, + { + name: "cluster not included should not be processed", + projectCfg: &ProjectConfig{ + IncludeClusters: []string{"Cluster1"}, + }, + clusterName: "Cluster2", + want: false, + }, + { + name: "excluded cluster should not be processed", + projectCfg: &ProjectConfig{ + ExcludeClusters: []string{"Cluster2"}, + }, + clusterName: "Cluster2", + want: false, + }, + { + name: "cluster not excluded should processed assuming it exists in the project", + projectCfg: &ProjectConfig{ + ExcludeClusters: []string{"Cluster1"}, + }, + clusterName: "Cluster2", + want: true, + }, + { + name: "cluster should be processed when no includes or excludes are set", + projectCfg: &ProjectConfig{}, + clusterName: "Cluster1", + want: true, + }, + { + name: "cluster should be processed when no includes or excludes are set and cluster name is empty", + projectCfg: nil, + clusterName: "Cluster1", + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.projectCfg != nil { + tt.projectCfg.populateIncludesAndExcludes() + } + + if got := shouldProcessCluster(tt.projectCfg, tt.clusterName); got != tt.want { + t.Errorf("shouldProcessCluster() = %v, want %v", got, tt.want) + } + }) + } +}