Run e2e and property based tests on nightly builds (#1276)

separation of integration and property tests and introduce nightly build

.github/workflows/e2e-polybft.yml

@@ -6,18 +6,18 @@ on:  # yamllint disable-line rule:truthy
       - main
       - develop
   pull_request:
+  workflow_call:
 
 jobs:
   build:
     runs-on: ubuntu-latest
     env:
       E2E_TESTS: true
       E2E_LOGS: true
+      E2E_TESTS_TYPE: 'integration'
       CI_VERBOSE: true
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
+      - uses: actions/checkout@v3        
       - name: Install Go
         uses: actions/setup-go@v3
         with:

.github/workflows/nightly.yml

@@ -0,0 +1,30 @@
+---
+name: Nightly Build
+on:  # yamllint disable-line rule:truthy
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron: '0 0 * * *'
+
+jobs:
+  build:
+    name: Build
+    uses: ./.github/workflows/build.yml
+
+  test:
+    name: Test
+    uses: ./.github/workflows/test.yml
+    needs: build
+    secrets:
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+
+  e2e:
+    name: Polybft E2E Tests
+    uses: ./.github/workflows/e2e-polybft.yml
+    needs: build
+
+  property:
+    name: Polybft Property Tests
+    uses: ./.github/workflows/property-polybft.yml
+    needs: build
+

.github/workflows/property-polybft.yml

@@ -0,0 +1,32 @@
+---
+name: PolyBFT Property tests
+on:  # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - main
+      - develop  
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      E2E_TESTS: true
+      E2E_LOGS: true
+      E2E_TESTS_TYPE: 'property'
+      CI_VERBOSE: true
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.x
+      - name: Run tests
+        run: make test-property-polybft
+      - name: Archive test logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: property-logs
+          path: property-logs-*/
+          retention-days: 30

Makefile

@@ -54,7 +54,12 @@ test-e2e:
 test-e2e-polybft:
     # We can not build with race because of a bug in boltdb dependency
 	go build -o artifacts/polygon-edge .
-	env EDGE_BINARY=${PWD}/artifacts/polygon-edge E2E_TESTS=true E2E_LOGS=true go test -v -timeout=30m ./e2e-polybft/...
+	env EDGE_BINARY=${PWD}/artifacts/polygon-edge E2E_TESTS=true E2E_LOGS=true E2E_TESTS_TYPE=integration go test -v -timeout=30m ./e2e-polybft/e2e/...
+
+test-property-polybft:
+    # We can not build with race because of a bug in boltdb dependency
+	go build -o artifacts/polygon-edge .
+	env EDGE_BINARY=${PWD}/artifacts/polygon-edge E2E_TESTS=true E2E_LOGS=true E2E_TESTS_TYPE=property go test -v -timeout=30m ./e2e-polybft/property/...
 
 .PHONY: compile-core-contracts
 compile-core-contracts:

e2e-polybft/README.md → e2e-polybft/e2e/README.md

@@ -16,7 +16,7 @@ In this case we expect that $GOPATH is set and $GOPATH/bin is defined in $PATH a
 ## Step 2: Run the tests
 
 ```bash
-export E2E_TESTS=TRUE && go test -v ./e2e-polybft/...
+export E2E_TESTS=TRUE && go test -v ./e2e-polybft/e2e/...
 ```
 
 To enable logs in the e2e test set `E2E_LOGS=true`.

e2e-polybft/bridge_test.go → e2e-polybft/e2e/bridge_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/0xPolygon/polygon-edge/consensus/polybft/contractsapi/artifact"
 	"github.com/0xPolygon/polygon-edge/contracts"
 	"github.com/0xPolygon/polygon-edge/e2e-polybft/framework"
+
 	"github.com/0xPolygon/polygon-edge/txrelayer"
 	"github.com/0xPolygon/polygon-edge/types"
 )

e2e-polybft/framework/test-cluster.go

@@ -36,8 +36,8 @@ const (
 	// envStdoutEnabled signal whether the output of the nodes get piped to stdout
 	envStdoutEnabled = "E2E_STDOUT"
 
-	// property based tests enabled
-	envPropertyBaseTestEnabled = "E2E_PROPERTY_TESTS"
+	// envE2ETestsType used just to display type of test if skipped
+	envE2ETestsType = "E2E_TESTS_TYPE"
 )
 
 const (
@@ -79,7 +79,6 @@ type TestClusterConfig struct {
 	ValidatorSetSize  uint64
 	EpochSize         int
 	EpochReward       int
-	PropertyBaseTests bool
 	SecretsCallback   func([]types.Address, *TestClusterConfig)
 
 	NumBlockConfirmations uint64
@@ -130,7 +129,7 @@ func (c *TestClusterConfig) GetStdout(name string, custom ...io.Writer) io.Write
 }
 
 func (c *TestClusterConfig) initLogsDir() {
-	logsDir := path.Join("..", fmt.Sprintf("e2e-logs-%d", startTime), c.t.Name())
+	logsDir := path.Join("../..", fmt.Sprintf("e2e-logs-%d", startTime), c.t.Name())
 
 	if err := common.CreateDirSafe(logsDir, 0750); err != nil {
 		c.t.Fatal(err)
@@ -215,12 +214,6 @@ func WithBlockGasLimit(blockGasLimit uint64) ClusterOption {
 	}
 }
 
-func WithPropertyBaseTests(propertyBaseTests bool) ClusterOption {
-	return func(h *TestClusterConfig) {
-		h.PropertyBaseTests = propertyBaseTests
-	}
-}
-
 func WithNumBlockConfirmations(numBlockConfirmations uint64) ClusterOption {
 	return func(h *TestClusterConfig) {
 		h.NumBlockConfirmations = numBlockConfirmations
@@ -255,9 +248,13 @@ func NewTestCluster(t *testing.T, validatorsCount int, opts ...ClusterOption) *T
 		opt(config)
 	}
 
-	if !config.PropertyBaseTests && !isTrueEnv(envE2ETestsEnabled) ||
-		config.PropertyBaseTests && !isTrueEnv(envPropertyBaseTestEnabled) {
-		t.Skip("Integration tests are disabled.")
+	if !isTrueEnv(envE2ETestsEnabled) {
+		testType := os.Getenv(envE2ETestsType)
+		if testType == "" {
+			testType = "integration"
+		}
+
+		t.Skip(fmt.Sprintf("%s tests are disabled.", testType))
 	}
 
 	config.TmpDir, err = os.MkdirTemp("/tmp", "e2e-polybft-")

e2e-polybft/property_test.go → e2e-polybft/property/property_test.go

@@ -1,4 +1,4 @@
-package e2e
+package property
 
 import (
 	"fmt"
@@ -35,7 +35,6 @@ func TestProperty_DifferentVotingPower(t *testing.T) {
 
 		cluster := framework.NewTestCluster(t, int(numNodes),
 			framework.WithEpochSize(epochSize),
-			framework.WithPropertyBaseTests(true),
 			framework.WithSecretsCallback(func(adresses []types.Address, config *framework.TestClusterConfig) {
 				for i, a := range adresses {
 					config.Premine = append(config.Premine, fmt.Sprintf("%s:%d", a, premine[i]))