update automation tests

integration-tests/benchmark/automation_test.go

@@ -3,6 +3,7 @@ package benchmark
 import (
 	"fmt"
 	"math/big"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -334,6 +335,10 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.Automation
 		return testEnvironment, testNetwork
 	}
 
+	if *keeperTestConfig.GetPyroscopeConfig().Enabled {
+		keeperTestConfig.GetPyroscopeConfig().Environment = &testEnvironment.Cfg.Namespace
+	}
+
 	// separate RPC urls per CL node
 	internalWsURLs := make([]string, 0)
 	internalHttpURLs := make([]string, 0)
@@ -360,22 +365,27 @@ func SetupAutomationBenchmarkEnv(t *testing.T, keeperTestConfig types.Automation
 	l.Debug().Strs("internalWsURLs", internalWsURLs).Strs("internalHttpURLs", internalHttpURLs).Msg("internalURLs")
 
 	for i := 0; i < numberOfNodes; i++ {
+		config := keeperTestConfig
+		if *config.GetPyroscopeConfig().Enabled {
+			name := testEnvironment.Cfg.Namespace + "-" + strconv.Itoa(i)
+			config.GetPyroscopeConfig().Environment = &name
+		}
 		testNetwork.HTTPURLs = []string{internalHttpURLs[i]}
 		testNetwork.URLs = []string{internalWsURLs[i]}
 
 		var overrideFn = func(_ interface{}, target interface{}) {
-			ctfconfig.MustConfigOverrideChainlinkVersion(keeperTestConfig.GetChainlinkImageConfig(), target)
-			ctfconfig.MightConfigOverridePyroscopeKey(keeperTestConfig.GetPyroscopeConfig(), target)
+			ctfconfig.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target)
+			ctfconfig.MightConfigOverridePyroscopeKey(config.GetPyroscopeConfig(), target)
 		}
 
-		tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(keeperTestConfig, testNetwork)
+		tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(config, testNetwork)
 		require.NoError(t, err, "Error building TOML config")
 
 		cd := chainlink.NewWithOverride(i, map[string]any{
 			"toml":      tomlConfig,
 			"chainlink": chainlinkResources,
 			"db":        dbResources,
-		}, keeperTestConfig.GetChainlinkImageConfig(), overrideFn)
+		}, config.GetChainlinkImageConfig(), overrideFn)
 
 		testEnvironment.AddHelm(cd)
 	}

integration-tests/load/automationv2_1/automationv2_1_test.go

@@ -277,24 +277,29 @@ Load Config:
 		secretsTOML = ""
 	}
 
-	numberOfUpkeeps := *loadedTestConfig.Automation.General.NumberOfNodes
+	numberOfNodes := *loadedTestConfig.Automation.General.NumberOfNodes
 
-	for i := 0; i < numberOfUpkeeps+1; i++ { // +1 for the OCR boot node
+	for i := 0; i < numberOfNodes+1; i++ { // +1 for the OCR boot node
+		config := loadedTestConfig
+		if *config.Pyroscope.Enabled {
+			name := testEnvironment.Cfg.Namespace + "-" + strconv.Itoa(i)
+			config.Pyroscope.Environment = &name
+		}
 		var overrideFn = func(_ interface{}, target interface{}) {
-			ctfconfig.MustConfigOverrideChainlinkVersion(loadedTestConfig.GetChainlinkImageConfig(), target)
-			ctfconfig.MightConfigOverridePyroscopeKey(loadedTestConfig.GetPyroscopeConfig(), target)
+			ctfconfig.MustConfigOverrideChainlinkVersion(config.GetChainlinkImageConfig(), target)
+			ctfconfig.MightConfigOverridePyroscopeKey(config.GetPyroscopeConfig(), target)
 		}
 
-		tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&loadedTestConfig, testNetwork)
+		tomlConfig, err := actions.BuildTOMLNodeConfigForK8s(&config, testNetwork)
 		require.NoError(t, err, "Error building TOML config")
 
 		cd := chainlink.NewWithOverride(i, map[string]any{
 			"toml":        tomlConfig,
 			"chainlink":   nodeSpec,
 			"db":          dbSpec,
-			"prometheus":  *loadedTestConfig.Automation.General.UsePrometheus,
+			"prometheus":  *config.Automation.General.UsePrometheus,
 			"secretsToml": secretsTOML,
-		}, loadedTestConfig.ChainlinkImage, overrideFn)
+		}, config.ChainlinkImage, overrideFn)
 		testEnvironment.AddHelm(cd)
 	}
 
@@ -318,7 +323,7 @@ Load Config:
 	a.PublicConfig = actions.ReadPublicConfig(loadedTestConfig)
 	a.RegistrarSettings = contracts.KeeperRegistrarSettings{
 		AutoApproveConfigType: uint8(2),
-		AutoApproveMaxAllowed: 1000,
+		AutoApproveMaxAllowed: math.MaxUint16,
 		MinLinkJuels:          big.NewInt(0),
 	}
 

integration-tests/load/automationv2_1/helpers.go

@@ -52,7 +52,7 @@ func sendSlackNotification(header string, l zerolog.Logger, config *tc.TestConfi
 	notificationBlocks = append(notificationBlocks, slack.NewDividerBlock())
 	if *config.Pyroscope.Enabled {
 		pyroscopeServer := *config.Pyroscope.ServerUrl
-		pyroscopeEnvironment := *config.Pyroscope.Environment
+		pyroscopeEnvironment := *config.Pyroscope.Environment + "-1"
 
 		formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%s&to=%s", pyroscopeServer, pyroscopeEnvironment, startingTime, endingTime)
 		l.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL")

integration-tests/testconfig/automation/automation.toml

@@ -321,6 +321,7 @@ contract_call_interval = "4s"
 
 [Benchmark.Seth]
 # keeper benchmark running on simulated network requires 100k per node
+ephemeral_addresses_number = 100
 root_key_funds_buffer = 1_000_000
 
 [Benchmark.Automation]

integration-tests/testconfig/automation/overrides/benchmark/10000Upkeeps-24h-2_3.toml

@@ -0,0 +1,18 @@
+[ChainlinkImage]
+version="2.16.0"
+
+[Benchmark.Automation.Benchmark]
+registry_to_test = "2_3"
+number_of_registries = 1
+number_of_nodes = 6
+number_of_upkeeps = 10000
+upkeep_gas_limit = 1500000
+check_gas_to_burn = 10000
+perform_gas_to_burn = 1000
+block_range = 86400
+block_interval = 60
+forces_single_tx_key = false
+delete_jobs_on_end = true
+
+[Pyroscope]
+enabled=true

integration-tests/testconfig/automation/overrides/load/50Upkeeps-1x-1000Upkeeps-0x-24h.toml

@@ -0,0 +1,55 @@
+[ChainlinkImage]
+version="2.16.0"
+
+[Load.Seth]
+root_key_funds_buffer = 1_000_000
+
+[Load.Common]
+chainlink_node_funding = 1000
+
+[Load.Automation.AutomationConfig]
+use_log_buffer_v1=false
+
+[Load.Automation.AutomationConfig.PluginConfig.LogProviderConfig]
+block_rate=1
+log_limit=2
+
+[Load.Automation]
+[Load.Automation.General]
+number_of_nodes=6
+duration=86400
+block_time=1
+spec_type="recommended"
+chainlink_node_log_level="debug"
+use_prometheus=true
+remove_namespace = true
+
+[Load.Automation.DataStreams]
+enabled=false
+
+[[Load.Automation.Load]]
+number_of_upkeeps=50
+number_of_events = 1
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[[Load.Automation.Load]]
+number_of_upkeeps=1000
+number_of_events = 0
+number_of_spam_matching_events = 0
+number_of_spam_non_matching_events = 0
+check_burn_amount = 0
+perform_burn_amount = 0
+upkeep_gas_limit = 1000000
+shared_trigger = false
+is_streams_lookup = false
+feeds = []
+
+[Pyroscope]
+enabled=true

integration-tests/testreporters/keeper_benchmark.go

@@ -11,6 +11,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/smartcontractkit/chainlink/integration-tests/testconfig"
+
 	"github.com/rs/zerolog/log"
 	"github.com/slack-go/slack"
 
@@ -26,6 +28,7 @@ type KeeperBenchmarkTestReporter struct {
 	NumRevertedUpkeeps             int64
 	NumStaleUpkeepReports          int64
 	Summary                        KeeperBenchmarkTestSummary `json:"summary"`
+	SlackTs                        string
 
 	namespace                 string
 	keeperReportFile          string
@@ -49,8 +52,9 @@ type KeeperBenchmarkTestLoad struct {
 }
 
 type KeeperBenchmarkTestConfig struct {
-	Chainlink map[string]map[string]string `json:"chainlink"`
-	Geth      map[string]map[string]string `json:"geth"`
+	Chainlink  map[string]map[string]string `json:"chainlink"`
+	Geth       map[string]map[string]string `json:"geth"`
+	TestConfig testconfig.TestConfig
 }
 
 type KeeperBenchmarkTestMetrics struct {
@@ -259,7 +263,7 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC
 	messageBlocks := testreporters.CommonSlackNotificationBlocks(
 		headerText, k.namespace, k.keeperReportFile,
 	)
-	ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...))
+	ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...), slack.MsgOptionTS(k.SlackTs))
 	if err != nil {
 		return err
 	}
@@ -277,12 +281,24 @@ func (k *KeeperBenchmarkTestReporter) SendSlackNotification(t *testing.T, slackC
 	formattedDashboardUrl := fmt.Sprintf("%s%s?from=%d&to=%d&var-namespace=%s&var-cl_node=chainlink-0-0", grafanaUrl, dashboardUrl, k.Summary.StartTime, k.Summary.EndTime, k.namespace)
 	log.Info().Str("Dashboard", formattedDashboardUrl).Msg("Dashboard URL")
 
+	notificationText := fmt.Sprintf("Automation Benchmark Test Summary %s.\n<%s|Test Dashboard> ", k.namespace, formattedDashboardUrl)
+
+	config := k.Summary.Config.TestConfig
+	if *config.GetPyroscopeConfig().Enabled {
+		pyroscopeServer := *config.GetPyroscopeConfig().ServerUrl
+		pyroscopeEnvironment := *config.GetPyroscopeConfig().Environment + "-1"
+
+		formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%d&to=%d", pyroscopeServer, pyroscopeEnvironment, k.Summary.StartTime, k.Summary.EndTime)
+		log.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL")
+		notificationText += fmt.Sprintf("\n<%s|Pyroscope>", formattedPyroscopeUrl)
+	}
+
 	if err := testreporters.UploadSlackFile(slackClient, slack.FileUploadParameters{
 		Title:           fmt.Sprintf("Automation Benchmark Test Summary %s", k.namespace),
 		Filetype:        "json",
 		Filename:        fmt.Sprintf("automation_benchmark_summary_%s.json", k.namespace),
 		File:            k.keeperSummaryFile,
-		InitialComment:  fmt.Sprintf("Automation Benchmark Test Summary %s.\n<%s|Test Dashboard> ", k.namespace, formattedDashboardUrl),
+		InitialComment:  notificationText,
 		Channels:        []string{testreporters.SlackChannel},
 		ThreadTimestamp: ts,
 	}); err != nil {

integration-tests/testsetups/automation_benchmark.go

@@ -110,6 +110,7 @@ func NewKeeperBenchmarkTest(t *testing.T, inputs KeeperBenchmarkTestInputs) *Kee
 func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config testconfig.TestConfig) {
 	startTime := time.Now()
 	k.TestReporter.Summary.StartTime = startTime.UnixMilli()
+	k.TestReporter.Summary.Config.TestConfig = config
 	k.ensureInputValues()
 	k.env = env
 	k.namespace = k.env.Cfg.Namespace
@@ -180,10 +181,11 @@ func (k *KeeperBenchmarkTest) Setup(env *environment.Environment, config testcon
 	}
 
 	k.log.Info().Str("Setup Time", time.Since(startTime).String()).Msg("Finished Keeper Benchmark Test Setup")
-	err = k.SendSlackNotification(nil, &config)
+	ts, err := k.SendSlackNotification(nil, &config)
 	if err != nil {
 		k.log.Warn().Msg("Sending test start slack notification failed")
 	}
+	k.TestReporter.SlackTs = ts
 }
 
 // Run runs the keeper benchmark test
@@ -618,19 +620,19 @@ func (k *KeeperBenchmarkTest) ensureInputValues() {
 	}
 }
 
-func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, config tt.AutomationBenchmarkTestConfig) error {
+func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, config tt.AutomationBenchmarkTestConfig) (string, error) {
 	if slackClient == nil {
 		slackClient = slack.New(reportModel.SlackAPIKey)
 	}
 
 	grafanaUrl, err := config.GetGrafanaBaseURL()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	dashboardUrl, err := config.GetGrafanaDashboardURL()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	headerText := ":white_check_mark: Automation Benchmark Test STARTED :white_check_mark:"
@@ -646,10 +648,20 @@ func (k *KeeperBenchmarkTest) SendSlackNotification(slackClient *slack.Client, c
 	notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn",
 		fmt.Sprintf("<%s|Test Dashboard> \nNotifying <@%s>",
 			formattedDashboardUrl, reportModel.SlackUserID), false, true), nil, nil))
+	if *config.GetPyroscopeConfig().Enabled {
+		pyroscopeServer := *config.GetPyroscopeConfig().ServerUrl
+		pyroscopeEnvironment := *config.GetPyroscopeConfig().Environment + "-1"
+
+		formattedPyroscopeUrl := fmt.Sprintf("%s/?query=chainlink-node.cpu{Environment=\"%s\"}&from=%d&to=%s", pyroscopeServer, pyroscopeEnvironment, k.TestReporter.Summary.StartTime, "now")
+		log.Info().Str("Pyroscope", formattedPyroscopeUrl).Msg("Dashboard URL")
+		notificationBlocks = append(notificationBlocks, slack.NewSectionBlock(slack.NewTextBlockObject("mrkdwn",
+			fmt.Sprintf("<%s|Pyroscope>",
+				formattedPyroscopeUrl), false, true), nil, nil))
+	}
 
 	ts, err := reportModel.SendSlackMessage(slackClient, slack.MsgOptionBlocks(notificationBlocks...))
 	log.Debug().Str("ts", ts).Msg("Sent Slack Message")
-	return err
+	return ts, err
 }
 
 // SetupBenchmarkKeeperContracts deploys a set amount of keeper Benchmark contracts registered to a single registry