feat: e2e test Pesimisstic L2 to L2 claim (#239)

- E2E to test L2 to L2 bridges
- RPC for aggsender: changes components names
- Added `tools` folder that contains `aggsender_find_imported_bridge` required by e2e test
- Add configuration to aggsender: `DryRun` and `EnableRPC`
---------

Co-authored-by: Stefan Negovanović <93934272+Stefan-Ethernal@users.noreply.github.com>
Co-authored-by: Rachit Sonthalia <54906134+rachit77@users.noreply.github.com>
Co-authored-by: Goran Rojovic <100121253+goran-ethernal@users.noreply.github.com>

.github/workflows/test-e2e-multi_pp.yml

@@ -0,0 +1,85 @@
+# based on: https://github.com/0xPolygon/kurtosis-cdk/blob/jhilliard/multi-pp-testing/multi-pp-test.sh.md
+name: Test e2e multi pp
+on: 
+  push:
+    branches:
+      - '**'
+  workflow_dispatch: {}
+
+
+jobs:
+  test-e2e-multi_pp:
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version: [ 1.22.x ]
+        goarch: [ "amd64" ]
+        e2e-group:
+          - "fork12-pessimistic"
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.go-version }}
+      env:
+        GOARCH: ${{ matrix.goarch }}
+
+
+    - name: Build Docker
+      run: make build-docker
+
+    - name: Build Tools
+      run: make build-tools
+
+    - name: Checkout kurtosis-cdk
+      uses: actions/checkout@v4
+      with:
+        repository: 0xPolygon/kurtosis-cdk
+        path: kurtosis-cdk
+        ref: jhilliard/multi-pp-testing
+
+    - name: Install Kurtosis CDK tools
+      uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk
+
+    - name: Install polycli
+      run: |
+        git clone https://github.com/0xPolygon/polygon-cli -b jhilliard/alonso
+        cd polygon-cli
+        make install
+        cp ~/go/bin/polycli /usr/local/bin/polycli
+        /usr/local/bin/polycli version
+
+    - name: Setup Bats and bats libs
+      uses: bats-core/bats-action@2.0.0
+
+    - name: Test
+      run: make test-e2e-fork12-multi-pessimistic
+
+      working-directory: test
+      env:
+        KURTOSIS_FOLDER: ${{ github.workspace }}/kurtosis-cdk
+        BATS_LIB_PATH: /usr/lib/
+        agglayer_prover_sp1_key: ${{ secrets.SP1_PRIVATE_KEY }}
+
+    - name: Dump enclave logs
+      if: failure()
+      run: kurtosis dump ./dump
+
+    - name: Generate archive name
+      if: failure()
+      run: |
+        archive_name="dump_run_with_args_${{matrix.e2e-group}}_${{ github.run_id }}"
+        echo "ARCHIVE_NAME=${archive_name}" >> "$GITHUB_ENV"
+        echo "Generated archive name: ${archive_name}"
+        kurtosis service exec cdk cdk-node-001 'cat /etc/cdk/cdk-node-config.toml' > ./dump/cdk-node-config.toml
+
+    - name: Upload logs
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.ARCHIVE_NAME }}
+        path: ./dump

Dockerfile

@@ -9,7 +9,7 @@ RUN go mod download
 
 # BUILD BINARY
 COPY . .
-RUN make build-go
+RUN make build-go build-tools
 
 # BUILD RUST BIN
 FROM --platform=${BUILDPLATFORM} rust:slim-bookworm AS chef
@@ -46,4 +46,6 @@ RUN apt-get update && apt-get install -y ca-certificates sqlite3 procps libssl-d
 COPY --from=builder /app/target/release/cdk /usr/local/bin/
 COPY --from=build /go/src/github.com/0xPolygon/cdk/target/cdk-node /usr/local/bin/
 
+EXPOSE 5576/tcp
+
 CMD ["/bin/sh", "-c", "cdk"]

Makefile

@@ -56,7 +56,7 @@ install-linter: check-go check-curl
 generate-code-from-proto: check-protoc
 
 .PHONY: build
-build: build-rust build-go ## Builds the binaries locally into ./target
+build: build-rust build-go  build-tools## Builds the binaries locally into ./target
 
 .PHONY: build-rust
 build-rust:
@@ -66,6 +66,10 @@ build-rust:
 build-go:
 	$(GOENVVARS) go build -ldflags "all=$(LDFLAGS)" -o $(GOBIN)/$(GOBINARY) $(GOCMD)
 
+.PHONY: build-tools
+build-tools: ## Builds the tools
+	$(GOENVVARS) go build -o $(GOBIN)/aggsender_find_imported_bridge ./tools/aggsender_find_imported_bridge
+
 .PHONY: build-docker
 build-docker: ## Builds a docker image with the cdk binary
 	docker build -t cdk -f ./Dockerfile .

agglayer/types.go

@@ -25,7 +25,9 @@ const (
 	InError
 	Settled
 
-	nilStr = "nil"
+	nilStr  = "nil"
+	nullStr = "null"
+	base10  = 10
 )
 
 var (
@@ -77,7 +79,13 @@ func (c *CertificateStatus) UnmarshalJSON(rawStatus []byte) error {
 	case "Settled":
 		*c = Settled
 	default:
-		return fmt.Errorf("invalid status: %s", status)
+		// Maybe the status is numeric:
+		var statusInt int
+		if _, err := fmt.Sscanf(status, "%d", &statusInt); err == nil {
+			*c = CertificateStatus(statusInt)
+		} else {
+			return fmt.Errorf("invalid status: %s", status)
+		}
 	}
 
 	return nil
@@ -93,6 +101,23 @@ func (l LeafType) String() string {
 	return [...]string{"Transfer", "Message"}[l]
 }
 
+func (l *LeafType) UnmarshalJSON(raw []byte) error {
+	rawStr := strings.Trim(string(raw), "\"")
+	switch rawStr {
+	case "Transfer":
+		*l = LeafTypeAsset
+	case "Message":
+		*l = LeafTypeMessage
+	default:
+		var value int
+		if _, err := fmt.Sscanf(rawStr, "%d", &value); err != nil {
+			return fmt.Errorf("invalid LeafType: %s", rawStr)
+		}
+		*l = LeafType(value)
+	}
+	return nil
+}
+
 const (
 	LeafTypeAsset LeafType = iota
 	LeafTypeMessage
@@ -333,6 +358,49 @@ func (b *BridgeExit) MarshalJSON() ([]byte, error) {
 	})
 }
 
+func (b *BridgeExit) UnmarshalJSON(data []byte) error {
+	aux := &struct {
+		LeafType           LeafType       `json:"leaf_type"`
+		TokenInfo          *TokenInfo     `json:"token_info"`
+		DestinationNetwork uint32         `json:"dest_network"`
+		DestinationAddress common.Address `json:"dest_address"`
+		Amount             string         `json:"amount"`
+		Metadata           interface{}    `json:"metadata"`
+	}{}
+
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	b.LeafType = aux.LeafType
+	b.TokenInfo = aux.TokenInfo
+	b.DestinationNetwork = aux.DestinationNetwork
+	b.DestinationAddress = aux.DestinationAddress
+	var ok bool
+	if !strings.Contains(aux.Amount, nilStr) {
+		b.Amount, ok = new(big.Int).SetString(aux.Amount, base10)
+		if !ok {
+			return fmt.Errorf("failed to convert amount to big.Int: %s", aux.Amount)
+		}
+	}
+	if s, ok := aux.Metadata.(string); ok {
+		b.IsMetadataHashed = true
+		b.Metadata = common.Hex2Bytes(s)
+	} else if uints, ok := aux.Metadata.([]interface{}); ok {
+		b.IsMetadataHashed = false
+		b.Metadata = make([]byte, len(uints))
+		for k, v := range uints {
+			value, ok := v.(float64)
+			if !ok {
+				return fmt.Errorf("failed to convert metadata to byte: %v", v)
+			}
+			b.Metadata[k] = byte(value)
+		}
+	} else {
+		b.Metadata = nil
+	}
+	return nil
+}
+
 // bytesToUints converts a byte slice to a slice of uints
 func bytesToUints(data []byte) []uint {
 	uints := make([]uint, len(data))
@@ -361,6 +429,20 @@ func (m *MerkleProof) MarshalJSON() ([]byte, error) {
 	})
 }
 
+func (m *MerkleProof) UnmarshalJSON(data []byte) error {
+	aux := &struct {
+		Root  common.Hash                                 `json:"root"`
+		Proof map[string][types.DefaultHeight]common.Hash `json:"proof"`
+	}{}
+
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	m.Root = aux.Root
+	m.Proof = aux.Proof["siblings"]
+	return nil
+}
+
 // Hash returns the hash of the Merkle proof struct
 func (m *MerkleProof) Hash() common.Hash {
 	proofsAsSingleSlice := make([]byte, 0)
@@ -455,6 +537,28 @@ func (c *ClaimFromMainnnet) MarshalJSON() ([]byte, error) {
 	})
 }
 
+func (c *ClaimFromMainnnet) UnmarshalJSON(data []byte) error {
+	if string(data) == nullStr {
+		return nil
+	}
+
+	claimData := &struct {
+		Child struct {
+			ProofLeafMER     *MerkleProof    `json:"proof_leaf_mer"`
+			ProofGERToL1Root *MerkleProof    `json:"proof_ger_l1root"`
+			L1Leaf           *L1InfoTreeLeaf `json:"l1_leaf"`
+		} `json:"Mainnet"`
+	}{}
+	if err := json.Unmarshal(data, &claimData); err != nil {
+		return fmt.Errorf("failed to unmarshal the subobject: %w", err)
+	}
+	c.ProofLeafMER = claimData.Child.ProofLeafMER
+	c.ProofGERToL1Root = claimData.Child.ProofGERToL1Root
+	c.L1Leaf = claimData.Child.L1Leaf
+
+	return nil
+}
+
 // Hash is the implementation of Claim interface
 func (c *ClaimFromMainnnet) Hash() common.Hash {
 	return crypto.Keccak256Hash(
@@ -496,6 +600,31 @@ func (c *ClaimFromRollup) MarshalJSON() ([]byte, error) {
 	})
 }
 
+func (c *ClaimFromRollup) UnmarshalJSON(data []byte) error {
+	if string(data) == nullStr {
+		return nil
+	}
+
+	claimData := &struct {
+		Child struct {
+			ProofLeafLER     *MerkleProof    `json:"proof_leaf_ler"`
+			ProofLERToRER    *MerkleProof    `json:"proof_ler_rer"`
+			ProofGERToL1Root *MerkleProof    `json:"proof_ger_l1root"`
+			L1Leaf           *L1InfoTreeLeaf `json:"l1_leaf"`
+		} `json:"Rollup"`
+	}{}
+
+	if err := json.Unmarshal(data, &claimData); err != nil {
+		return fmt.Errorf("failed to unmarshal the subobject: %w", err)
+	}
+	c.ProofLeafLER = claimData.Child.ProofLeafLER
+	c.ProofLERToRER = claimData.Child.ProofLERToRER
+	c.ProofGERToL1Root = claimData.Child.ProofGERToL1Root
+	c.L1Leaf = claimData.Child.L1Leaf
+
+	return nil
+}
+
 // Hash is the implementation of Claim interface
 func (c *ClaimFromRollup) Hash() common.Hash {
 	return crypto.Keccak256Hash(
@@ -511,6 +640,35 @@ func (c *ClaimFromRollup) String() string {
 		c.ProofLeafLER.String(), c.ProofLERToRER.String(), c.ProofGERToL1Root.String(), c.L1Leaf.String())
 }
 
+// ClaimSelector is a helper struct that allow to decice which type of claim to unmarshal
+type ClaimSelector struct {
+	obj Claim
+}
+
+func (c *ClaimSelector) GetObject() Claim {
+	return c.obj
+}
+
+func (c *ClaimSelector) UnmarshalJSON(data []byte) error {
+	var obj map[string]interface{}
+	if string(data) == nullStr {
+		return nil
+	}
+	if err := json.Unmarshal(data, &obj); err != nil {
+		return err
+	}
+	var ok bool
+	if _, ok = obj["Mainnet"]; ok {
+		c.obj = &ClaimFromMainnnet{}
+	} else if _, ok = obj["Rollup"]; ok {
+		c.obj = &ClaimFromRollup{}
+	} else {
+		return errors.New("invalid claim type")
+	}
+
+	return json.Unmarshal(data, &c.obj)
+}
+
 // ImportedBridgeExit represents a token bridge exit originating on another network but claimed on the current network.
 type ImportedBridgeExit struct {
 	BridgeExit  *BridgeExit  `json:"bridge_exit"`
@@ -538,6 +696,21 @@ func (c *ImportedBridgeExit) String() string {
 	return res
 }
 
+func (c *ImportedBridgeExit) UnmarshalJSON(data []byte) error {
+	aux := &struct {
+		BridgeExit  *BridgeExit   `json:"bridge_exit"`
+		ClaimData   ClaimSelector `json:"claim_data"`
+		GlobalIndex *GlobalIndex  `json:"global_index"`
+	}{}
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	c.BridgeExit = aux.BridgeExit
+	c.ClaimData = aux.ClaimData.GetObject()
+	c.GlobalIndex = aux.GlobalIndex
+	return nil
+}
+
 // Hash returns a hash that uniquely identifies the imported bridge exit
 func (c *ImportedBridgeExit) Hash() common.Hash {
 	return crypto.Keccak256Hash(
@@ -592,7 +765,7 @@ func (c *CertificateHeader) String() string {
 	if c.PreviousLocalExitRoot != nil {
 		previousLocalExitRoot = c.PreviousLocalExitRoot.String()
 	}
-	return fmt.Sprintf("Height: %d, CertificateID: %s, previousLocalExitRoot:%s, NewLocalExitRoot: %s. Status: %s."+
+	return fmt.Sprintf("Height: %d, CertificateID: %s, PreviousLocalExitRoot: %s, NewLocalExitRoot: %s. Status: %s."+
 		" Errors: [%s]",
 		c.Height, c.CertificateID.String(), previousLocalExitRoot, c.NewLocalExitRoot.String(), c.Status.String(), errors)
 }
@@ -631,8 +804,7 @@ func (c *CertificateHeader) UnmarshalJSON(data []byte) error {
 		var agglayerErr error
 
 		for errKey, errValueRaw := range inErrDataMap {
-			errValueJSON, err := json.Marshal(errValueRaw)
-			if err != nil {
+			if errValueJSON, err := json.Marshal(errValueRaw); err != nil {
 				agglayerErr = &GenericError{
 					Key: errKey,
 					Value: fmt.Sprintf("failed to marshal the agglayer error to the JSON. Raw value: %+v\nReason: %+v",