Add evm sync ci workflow

.github/workflows/tests-evm-sync.yml

@@ -0,0 +1,157 @@
+name: Tests - EVM Sync
+on:
+  workflow_dispatch:
+    inputs:
+      block_ranges:
+        description: 'Optionally restrict to specific block ranges. Should be
+          formatted as multiple start block, such as "350000 50000 750000".
+          Defaults to all block ranges'
+        required: false
+      name:
+        description: 'Workflow run custom name'
+        required: false
+  pull_request:
+    branches:
+      - master
+    types: [labeled, opened, reopened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+run-name: ${{ inputs.name || github.event.pull_request.title || github.ref_name }}
+
+jobs:
+  build:
+    if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch'
+    runs-on: [self-hosted, linux, x64, builder]
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Populate environment
+      run: ./make.sh ci-export-vars
+
+    - name: Setup dependencies
+      run: sudo ./make.sh ci-setup-deps
+
+    - name: Setup dependencies for target
+      run: ./make.sh ci-setup-deps-target
+
+    - uses: Swatinem/rust-cache@v2
+      with:
+        workspaces: lib -> ../build/lib/target
+        save-if: ${{ github.ref == 'refs/heads/master' }}
+
+      # TODO: Switch this to a docker build later and this builds on the native
+      # VM toolchain and loses disparity with the CI release builds
+    - name: Build binaries
+      run: ./make.sh build
+
+    - name: Upload binaries
+      uses: actions/upload-artifact@v3
+      with:
+        name: defi-bins
+        path: |
+         build/src/defid
+         build/src/defi-cli
+
+    - name: Upload shell commands
+      uses: actions/upload-artifact@v3
+      with:
+        name: sync
+        path: ci/sync/evm_main.sh
+
+  generate-matrix:
+      if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch'
+      runs-on: [self-hosted, linux, x64]
+
+      # Add "id-token" with the intended permissions.
+      permissions:
+        contents: 'read'
+        id-token: 'write'
+
+      outputs:
+        matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+      steps:
+        - uses: actions/checkout@v3
+
+        - id: 'auth'
+          name: 'Authenticate to Google Cloud'
+          uses: 'google-github-actions/auth@v0'
+          with:
+            workload_identity_provider: 'projects/965426322273/locations/global/workloadIdentityPools/br-blockchains-pool/providers/br-blockchains-provider'
+            service_account: 'blockchain-dev-service@br-blockchains-dev.iam.gserviceaccount.com'
+
+        - name: 'Set up Cloud SDK'
+          uses: 'google-github-actions/setup-gcloud@v0'
+
+        - id: set-matrix
+          name: 'Set matrix output'
+          run: |
+            if [ -n "${{ inputs.block_ranges }}" ]; then
+              BLOCKS=$(echo ${{ inputs.block_ranges }} | tr ' ' '\n')
+            else
+              SNAPSHOTS=$(gsutil ls gs://team-drop/master-evm-datadir)
+              BLOCKS=$(echo "$SNAPSHOTS" | sed -e 's/.*\-\(.*\)\.tar.*/\1/' | grep -v gs | sort -n | head -n -1)
+            fi
+            MATRIX_JSON=$(jq -n -c -M --arg blocks "$BLOCKS" '{blocks: ($blocks | split("\n") | .[] |= tonumber | to_entries | map({start: .value , stop: (.value + 50000)}))}')
+            echo "MATRIX=$MATRIX_JSON" >> $GITHUB_OUTPUT
+
+  sync:
+    runs-on: [self-hosted, linux, x64]
+    needs: [build, generate-matrix]
+
+    strategy:
+      matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}
+
+    continue-on-error: true
+    env:
+      BASE_PATH: "https://storage.googleapis.com"
+      BUCKET: "team-drop"
+      DATADIR : datadir-${{matrix.blocks.start}}
+      STOP_BLOCK: ${{matrix.blocks.stop}}
+      START_BLOCK: ${{matrix.blocks.start}}
+      DEFID_BIN: ./defid
+      DEFI_CLI_BIN: ./defi-cli
+      BASE_REF: master
+    timeout-minutes: 4320
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Download Snapshot
+      run: aria2c -x16 -s16 ${{ env.BASE_PATH }}/${{ env.BUCKET }}/${{ BASE_REF }}-evm-datadir/datadir-${{matrix.blocks.start}}.tar.gz
+
+    - name: Create datadir
+      run: mkdir $DATADIR && tar -C $DATADIR -xvf datadir-${{matrix.blocks.start}}.tar.gz
+
+    - name: Download Binaries
+      uses: actions/download-artifact@v3
+      with:
+        name: defi-bins
+
+    - name: Download Shell Commands
+      uses: actions/download-artifact@v3
+      with:
+        name: sync
+
+    - name: Rename Artifact
+      run: mv evm_main.sh sync.sh
+
+    - name: Set Permissions
+      run: |
+        chmod 777 defid
+        chmod 777 defi-cli
+        chmod 777 sync.sh
+
+    - name: Sync
+      run: ./sync.sh
+
+    - name: Diff rollback log
+      run: diff pre-rollback.log post-rollback.log
+      if: ${{ failure() ||  success() }}
+
+    - name: Show post-rollback.log
+      run: cat post-rollback.log
+      if: ${{ failure() ||  success() }}

ci/sync/evm_main.sh

@@ -12,8 +12,8 @@ setup_vars() {
     DATADIR=${DATADIR:-".defi"}
     DEBUG_FILE="${DATADIR}/debug.log"
     CONF_FILE="${DATADIR}/defi.conf"
-    PRE_ROLLBACK_STATE_LOG="debug-pre-rollback.log"
-    POST_ROLLBACK_STATE_LOG="debug-post-rollback.log"
+    PRE_ROLLBACK_STATE_LOG="pre-rollback.log"
+    POST_ROLLBACK_STATE_LOG="post-rollback.log"
     BASE_REF=${BASE_REF:-"master"}
     BASE_PATH=${BASE_PATH:-"https://storage.googleapis.com"}
     BUCKET=${BUCKET:-"team-drop"}
@@ -50,7 +50,7 @@ print_info() {
     ${REF_LOG_PATH}
 
   - snapshot:
-   ${BASE_PATH}/${BUCKET}/${BASE_REF}-evmdatadir/datadir-${START_BLOCK}.tar.gz
+   ${BASE_PATH}/${BUCKET}/${BASE_REF}-evm-datadir/datadir-${START_BLOCK}.tar.gz
 
   - defid:
     ${DEFID_CMD}