Refactored test interface to enable parallelization

core/scripts/go.mod

@@ -13,7 +13,7 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment
 // creating potential merge conflicts.
 require (
 	github.com/smartcontractkit/chainlink/deployment v0.0.0-20241206210521-125d98cdaf66
-	github.com/smartcontractkit/chainlink/v2 v2.0.0-20241206210521-125d98cdaf66
+	github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a
 )
 
 require (
@@ -33,9 +33,7 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/shopspring/decimal v1.4.0
 	github.com/smartcontractkit/chainlink-automation v0.8.1
-	github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206162350-10f03fd2126b
-	github.com/smartcontractkit/chainlink/deployment v0.0.0-00010101000000-000000000000
-	github.com/smartcontractkit/chainlink/v2 v2.14.0-mercury-20240807.0.20241106193309-5560cd76211a
+	github.com/smartcontractkit/chainlink-common v0.4.1-0.20241217143242-5bd6eb484177
 	github.com/smartcontractkit/libocr v0.0.0-20241007185508-adbe57025f12
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0

core/scripts/go.sum

@@ -1148,10 +1148,10 @@ github.com/smartcontractkit/chain-selectors v1.0.34 h1:MJ17OGu8+jjl426pcKrJkCf3f
 github.com/smartcontractkit/chain-selectors v1.0.34/go.mod h1:xsKM0aN3YGcQKTPRPDDtPx2l4mlTN1Djmg0VVXV40b8=
 github.com/smartcontractkit/chainlink-automation v0.8.1 h1:sTc9LKpBvcKPc1JDYAmgBc2xpDKBco/Q4h4ydl6+UUU=
 github.com/smartcontractkit/chainlink-automation v0.8.1/go.mod h1:Iij36PvWZ6blrdC5A/nrQUBuf3MH3JvsBB9sSyc9W08=
-github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e h1:GnM6ZWV6vlk2+n6c6o+v/R1LtXzBGVVx7r37nt/h6Uc=
-github.com/smartcontractkit/chainlink-ccip v0.0.0-20241204015713-8956bb614e9e/go.mod h1:80vGBbOfertJig0xFKsRfm+i17FkjdKkk1dAaGE45Os=
-github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206162350-10f03fd2126b h1:+QEa/OYBQmzN2woqflQc3bf3SULvindn5xeLiLlT4EM=
-github.com/smartcontractkit/chainlink-common v0.3.1-0.20241206162350-10f03fd2126b/go.mod h1:bQktEJf7sJ0U3SmIcXvbGUox7SmXcnSEZ4kUbT8R5Nk=
+github.com/smartcontractkit/chainlink-ccip v0.0.0-20241213122413-5e8f65dd6b1b h1:iSQJ6ng4FhEswf8SXunGkaJlVP3E3JlgLB8Oo2f3Ud4=
+github.com/smartcontractkit/chainlink-ccip v0.0.0-20241213122413-5e8f65dd6b1b/go.mod h1:F8xQAIW0ymb2BZhqn89sWZLXreJhM5KDVF6Qb4y44N0=
+github.com/smartcontractkit/chainlink-common v0.4.1-0.20241217143242-5bd6eb484177 h1:5fV58ZlNE/d5C28rCVd8gHPTAF5mfdp5lTeVN83haZ4=
+github.com/smartcontractkit/chainlink-common v0.4.1-0.20241217143242-5bd6eb484177/go.mod h1:yti7e1+G9hhkYhj+L5sVUULn9Bn3bBL5/AxaNqdJ5YQ=
 github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e h1:PRoeby6ZlTuTkv2f+7tVU4+zboTfRzI+beECynF4JQ0=
 github.com/smartcontractkit/chainlink-cosmos v0.5.2-0.20241202195413-82468150ac1e/go.mod h1:mUh5/woemsVaHgTorA080hrYmO3syBCmPdnWc/5dOqk=
 github.com/smartcontractkit/chainlink-data-streams v0.1.1-0.20241216163550-fa030d178ba3 h1:aeiBdBHGY8QNftps+VqrIk6OnfeeOD5z4jrAabW4ZSc=

core/services/relay/evm/chain_components_flakes_test.go

@@ -0,0 +1,70 @@
+package evm_test
+
+import (
+	"bytes"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestRunChainComponentsMultipleTimes(t *testing.T) {
+	// Configuration for the test run
+	testName := "TestChainComponents"
+	numRuns := 1000
+	failCount := 0
+	totalTime := time.Duration(0)
+
+	for i := 1; i <= numRuns; i++ {
+		t.Logf("Running %s: iteration %d/%d...\n", testName, i, numRuns)
+
+		// Clear the Go test cache
+		cleanCmd := exec.Command("go", "clean", "-testcache")
+		cleanErr := cleanCmd.Run()
+		if cleanErr != nil {
+			t.Fatalf("Failed to clear test cache: %v", cleanErr)
+		}
+
+		// Start the timer
+		start := time.Now()
+
+		// Run the test via `go test`
+		cmd := exec.Command("go", "test", "-run", testName, "./...")
+		var out bytes.Buffer
+		var stderr bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Stderr = &stderr
+
+		err := cmd.Run()
+
+		// Record the elapsed time
+		elapsed := time.Since(start)
+		totalTime += elapsed
+
+		if err != nil {
+			failCount++
+			t.Errorf("Iteration %d failed with error: %s\n", i, err)
+			t.Logf("Test output:\n%s\n", out.String())
+			t.Logf("Test error output:\n%s\n", stderr.String())
+		} else {
+			t.Logf("Iteration %d passed.\n", i)
+		}
+
+		t.Logf("Test duration: %s\n", elapsed)
+	}
+
+	// Calculate the average test time
+	averageTime := totalTime / time.Duration(numRuns)
+
+	// Print results
+	t.Logf("\nResults for %s:\n", testName)
+	t.Logf("Total runs: %d\n", numRuns)
+	t.Logf("Failures: %d\n", failCount)
+	t.Logf("Pass rate: %.2f%%\n", float64(numRuns-failCount)/float64(numRuns)*100)
+	t.Logf("Total time: %s\n", totalTime)
+	t.Logf("Average test time: %s\n", averageTime)
+
+	// Optional: Fail the entire test if there were any failures
+	if failCount > 0 {
+		t.Fatalf("Test failed %d/%d times.\n", failCount, numRuns)
+	}
+}

core/services/relay/evm/chain_components_test.go

@@ -9,6 +9,7 @@ import (
 	"math/big"
 	"os"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
 
@@ -27,11 +28,11 @@ import (
 	commontestutils "github.com/smartcontractkit/chainlink-common/pkg/loop/testutils"
 	clcommontypes "github.com/smartcontractkit/chainlink-common/pkg/types"
 	"github.com/smartcontractkit/chainlink-common/pkg/types/interfacetests"
-	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
 
 	htMocks "github.com/smartcontractkit/chainlink/v2/common/headtracker/mocks"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
+	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller"
 	lpMocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller/mocks"
 	evmtxmgr "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr"
@@ -50,6 +51,7 @@ import (
 )
 
 const commonGasLimitOnEvms = uint64(4712388)
+const finalityDepth = 4
 
 func TestContractReaderEventsInitValidation(t *testing.T) {
 	tests := []struct {
@@ -228,34 +230,31 @@ func TestChainReader_HealthReport(t *testing.T) {
 }
 
 func TestChainComponents(t *testing.T) {
+	// shared helper so all tests can run efficiently in parallel
+	helper := &helper{}
+	helper.Init(t)
+	deployLock := sync.Mutex{}
 	// add new subtests here so that it can be run on real chains too
 	t.Run("RunChainComponentsEvmTests", func(t *testing.T) {
 		t.Parallel()
-		helper := &helper{}
-		// shared helper for separate parallel testers
-		it := &EVMChainComponentsInterfaceTester[*testing.T]{Helper: helper}
-		it.Init(t)
+		it := &EVMChainComponentsInterfaceTester[*testing.T]{Helper: helper, DeployLock: &deployLock}
 		RunChainComponentsEvmTests(t, it)
 	})
 
 	t.Run("RunChainComponentsInLoopEvmTests", func(t *testing.T) {
 		t.Parallel()
-		helper := &helper{}
-		// shared helper for separate parallel testers
-		it := &EVMChainComponentsInterfaceTester[*testing.T]{Helper: helper}
-		it.Init(t)
-		RunChainComponentsInLoopEvmTests[*testing.T](t, commontestutils.WrapContractReaderTesterForLoop(it))
+		it := &EVMChainComponentsInterfaceTester[*testing.T]{Helper: helper, DeployLock: &deployLock}
+		wrapped := commontestutils.WrapContractReaderTesterForLoop(it)
+		RunChainComponentsInLoopEvmTests[*testing.T](t, wrapped)
 	})
 
 	t.Run("RunChainComponentsInLoopEvmTestsWithBindings", func(t *testing.T) {
 		t.Parallel()
-		helper := &helper{}
-		// shared helper for separate parallel testers
-		helper.Init(t)
-		it := &EVMChainComponentsInterfaceTester[*testing.T]{Helper: helper}
+		it := &EVMChainComponentsInterfaceTester[*testing.T]{Helper: helper, DeployLock: &deployLock}
+		wrapped := WrapContractReaderTesterWithBindings(t, it)
 		// TODO, generated binding tests are broken
-		it.DisableTests([]string{interfacetests.ContractReaderGetLatestValue})
-		RunChainComponentsInLoopEvmTests(t, WrapContractReaderTesterWithBindings(t, it))
+		wrapped.DisableTests([]string{interfacetests.ContractReaderGetLatestValue})
+		RunChainComponentsInLoopEvmTests(t, wrapped)
 	})
 }
 
@@ -267,6 +266,40 @@ type helper struct {
 	txm         evmtxmgr.TxManager
 	client      client.Client
 	db          *sqlx.DB
+	lp          logpoller.LogPoller
+	ht          logpoller.HeadTracker
+}
+
+func getLPOpts() logpoller.Opts {
+	return logpoller.Opts{
+		PollPeriod:               time.Millisecond,
+		FinalityDepth:            finalityDepth,
+		BackfillBatchSize:        1,
+		RpcBatchSize:             1,
+		KeepFinalizedBlocksDepth: 10000,
+	}
+}
+
+func (h *helper) LogPoller(t *testing.T) logpoller.LogPoller {
+	if h.lp != nil {
+		return h.lp
+	}
+	ctx := testutils.Context(t)
+	lggr := logger.NullLogger
+	db := h.Database()
+
+	h.lp = logpoller.NewLogPoller(logpoller.NewORM(h.ChainID(), db, lggr), h.Client(t), lggr, h.HeadTracker(t), getLPOpts())
+	require.NoError(t, h.lp.Start(ctx))
+	return h.lp
+}
+
+func (h *helper) HeadTracker(t *testing.T) logpoller.HeadTracker {
+	if h.ht != nil {
+		return h.ht
+	}
+	lpOpts := getLPOpts()
+	h.ht = headtracker.NewSimulatedHeadTracker(h.Client(t), lpOpts.UseFinalityTag, lpOpts.FinalityDepth)
+	return h.ht
 }
 
 func (h *helper) Init(t *testing.T) {
@@ -278,6 +311,7 @@ func (h *helper) Init(t *testing.T) {
 
 	h.Backend()
 	h.client = h.Client(t)
+	h.LogPoller(t)
 
 	h.txm = h.TXM(t, h.client)
 }
@@ -338,6 +372,10 @@ func (h *helper) ChainID() *big.Int {
 	return testutils.SimulatedChainID
 }
 
+func (h *helper) Database() *sqlx.DB {
+	return h.db
+}
+
 func (h *helper) NewSqlxDB(t *testing.T) *sqlx.DB {
 	return pgtest.NewSqlxDB(t)
 }

core/services/relay/evm/evmtesting/bindings_test_adapter.go

@@ -96,6 +96,7 @@ func newBindingsMapping() bindingsMapping {
 }
 
 func getChainReaderConfig(wrapped *EVMChainComponentsInterfaceTester[*testing.T]) types.ChainReaderConfig {
+	fmt.Println("wrapped config")
 	testStruct := interfacetests.CreateTestStruct[*testing.T](0, wrapped)
 	chainReaderConfig := bindings.NewChainReaderConfig()
 	chainReaderConfig.Contracts["ChainReaderTester"].Configs["ReturnSeen"] = &types.ChainReaderDefinition{