feat(ci): run wan churn tests on the entire network

.github/workflows/nightly_wan_churn.yml

@@ -0,0 +1,164 @@
+name: Nightly -- Perform long running network churn 
+
+# on:
+#   schedule:
+#     - cron: "0 */12 * * *"
+#   workflow_dispatch:
+
+on:
+  # tests must run for a PR to be valid and pass merge queue muster
+  # on main, we want to know that all commits are passing at a glance, any deviation should help bisecting errors
+  # the merge run checks should show on master and enable this clear test/passing history
+  merge_group:
+    branches: [main, alpha*, beta*, rc*]
+  pull_request:
+    branches: ["*"]
+
+env:
+  CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI.
+  WORKFLOW_URL: https://github.com/maidsafe/stableset_net/actions/runs
+
+jobs:
+  e2e:
+    name: E2E tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+        continue-on-error: true
+
+      - name: Build safe
+        run: cargo build --release --bin safe
+        timeout-minutes: 30
+
+      - name: Start a WAN network
+        uses: maidsafe/sn-testnet-action@main
+        with:
+          action: create
+          re-attempts: 3
+          rust-log: debug
+          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
+          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
+          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
+          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
+          security-group-id: sg-0d47df5b3f0d01e2a
+          subnet-id: subnet-018f2ab26755df7f9
+          node-count: 20
+          vm-count: 5
+          provider: digital-ocean
+          testnet-name: NightlyE2E
+          # if we were to run on a PR, use the below
+          # custom-node-bin-org-name: ${{ github.actor }}"
+          # custom-node-bin-branch-name: ${{ github.event.pull_request.head.ref }}
+          # Specify custom branch to prevent the deployer from fetching the latest release.
+          # The latest release contains the `network-contacts` feature turned on.
+          custom-node-bin-org-name: maidsafe
+          custom-node-bin-branch-name: main
+
+      - name: Download material, 1.6G
+        shell: bash
+        run: |
+          wget https://releases.ubuntu.com/16.04/ubuntu-16.04.7-desktop-amd64.iso
+          ls -l
+
+      - name: Check env variables
+        shell: bash
+        run: |
+          echo "Peer is $SAFE_PEERS"
+          echo "Deployment inventory is $SN_INVENTORY"
+
+      - name: Obtain the funds from the faucet
+        run: |
+          # read the inventory file
+          inventory_path=/home/runner/.local/share/safe/testnet-deploy/NightlyE2E-inventory.json
+          echo "Inventory Path: $inventory_path"
+          faucet_address=$(jq -r '.faucet_address' $inventory_path)
+          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
+          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
+          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
+          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
+          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
+        env:
+          SN_LOG: "all"
+        timeout-minutes: 2
+
+      - name: Start a client to upload
+        run: cargo run --bin safe -- --log-output-dest=data-dir files upload "ubuntu-16.04.7-desktop-amd64.iso" --retry-strategy quick
+        env:
+          SN_LOG: "all"
+        timeout-minutes: 45
+
+      - name: Cause random churn
+        uses: RolandSherwin/sn-testnet-action/network_commands@network_commands
+        with:
+          action: churn-random
+          testnet-name: NightlyE2E
+          churn-cycles: 3
+          retain-peer-id: false
+          random-churn-time-frame: 60
+          random-churn-count: 5
+
+      - name: Start a client to download files
+        run: cargo run --bin safe --release -- --log-output-dest=data-dir files download --retry-strategy quick
+        env:
+          SN_LOG: "all"
+        timeout-minutes: 30
+
+      - name: Fetch network logs
+        if: always()
+        uses: maidsafe/sn-testnet-action@main
+        with:
+          action: logs
+          re-attempts: 3
+          rust-log: debug
+          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
+          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
+          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
+          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
+          node-count: 20
+          vm-count: 1
+          provider: digital-ocean
+          testnet-name: NightlyE2E
+          custom-node-bin-org-name: maidsafe
+          custom-node-bin-branch-name: main
+
+      - name: Upload local logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: local_logs_NightlyE2E
+          path: |
+            ~/.local/share/safe/node/*/logs/*.log*
+            ~/.local/share/safe/*/*/*.log*
+            ~/.local/share/safe/client/logs/*/*.log*
+
+      - name: Stop the WAN network
+        if: always()
+        uses: maidsafe/sn-testnet-action@main
+        with:
+          action: destroy
+          re-attempts: 3
+          rust-log: debug
+          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
+          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
+          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
+          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
+          node-count: 20
+          vm-count: 1
+          provider: digital-ocean
+          testnet-name: NightlyE2E
+          custom-node-bin-org-name: maidsafe
+          custom-node-bin-branch-name: main