Add big dev network for testing scale. (#4008)

* add big (795965) dev network for scale testing * bump pg resouces for big db * fix shard size and helm timeouts * tune aws s3 cli
mobilecoinfoundation · Sep 26, 2024 · d2477f7 · d2477f7
1 parent 52254df
commit d2477f7
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 11 deletions.
diff --git a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml
@@ -29,10 +29,9 @@ on:
         default: none
         options:
         - none
-        - v5.0.3-dev
-        - v5.1.1-dev
         - v5.1.1-dev.alpha.5946
         - v5.2.3-dev.alpha.6224
+        - v6.0.2-dev.alpha.795965
       minimum_block:
         description: "Minimum block to wait for"
         type: choice
@@ -42,6 +41,17 @@ on:
         - '500'
         - '5946'
         - '6224'
+        - '795965'
+      shard_size:
+        description: "Shard Size"
+        type: string
+        required: true
+        default: '20000'
+      shard_exceed_block_height_by:
+        description: "Exceed Block Height By"
+        type: string
+        required: true
+        default: '5000'
       ingest_color:
         description: "Fog Ingest blue/green"
         type: choice
@@ -102,4 +112,6 @@ jobs:
       namespace: ${{ inputs.namespace }}
       version: ${{ inputs.version }}
       minimum_block: ${{ inputs.minimum_block }}
+      shard_size: ${{ inputs.shard_size }}
+      shard_exceed_block_height_by: ${{ inputs.shard_exceed_block_height_by }}
     secrets: inherit
diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml
@@ -44,7 +44,7 @@ jobs:
     secrets: inherit
 
   restore-s3-archive:
-    runs-on: mcf-dev-small-x64
+    runs-on: mcf-dev-large-x64
     needs:
     - reset
     container:
@@ -60,6 +60,15 @@ jobs:
         VERSION: ${{ inputs.bootstrap_version }}
       shell: bash
       run: |
+        echo "[profile s3]" > /tmp/aws_cli_profile
+        # shellcheck disable=SC2129 # style issue on multiple echo lines. The alternative is harder to read.
+        echo "s3 =" >> /tmp/aws_cli_profile
+        echo "  max_concurrent_requests = 50" >> /tmp/aws_cli_profile
+        echo "  max_queue_size = 10000" >> /tmp/aws_cli_profile
+
+        export AWS_CONFIG_FILE=/tmp/aws_cli_profile
+        export AWS_PROFILE=s3
+
         for i in 1 2 3
         do
             aws s3 cp --only-show-errors --recursive --acl public-read \

diff --git a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml
@@ -38,6 +38,16 @@ on:
         type: string
         required: false
         default: "500"
+      shard_size:
+        description: "View/Ledger Shard size in blocks"
+        type: string
+        required: false
+        default: "20000"
+      shard_exceed_block_height_by:
+        description: "View/Ledger New shards should start this many blocks before the end of the current shard"
+        type: string
+        required: false
+        default: "5000"
     secrets:
       DEV_RANCHER_CLUSTER:
         description: "Rancher cluster name"
@@ -109,7 +119,7 @@ jobs:
         chart_name: consensus-node
         chart_version: ${{ inputs.version }}
         chart_values: ${{ env.VALUES_BASE_PATH }}/consensus-node-values.yaml
-        chart_wait_timeout: 10m
+        chart_wait_timeout: 60m
         release_name: ${{ matrix.release.name }}
         namespace: ${{ inputs.namespace }}
         rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }}
@@ -147,7 +157,7 @@ jobs:
         chart_name: mobilecoind
         chart_version: ${{ inputs.version }}
         chart_values: ${{ env.VALUES_BASE_PATH }}/mobilecoind-values.yaml
-        chart_wait_timeout: 5m
+        chart_wait_timeout: 60m
         release_name: mobilecoind
         namespace: ${{ inputs.namespace }}
         rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }}
@@ -263,6 +273,12 @@ jobs:
           partner: dev
 
         fogView:
+          stackConfig:
+            network:
+              default:
+                shardSize: ${{ inputs.shard_size }}
+                exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }}
+
           router:
             hosts:
             - partner: a
@@ -310,6 +326,11 @@ jobs:
           partner: dev
 
         fogLedger:
+          stackConfig:
+            network:
+              default:
+                shardSize: ${{ inputs.shard_size }}
+                exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }}
           router:
             hosts:
             - partner: a

diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml
@@ -53,8 +53,6 @@ jobs:
         - mc-core-dev-env-setup
         - fog-report-a
         - fog-report-b
-        - fog-view
-        - fog-ledger
     steps:
     - name: Delete release
       uses: mobilecoinofficial/gha-k8s-toolbox@v1
@@ -91,7 +89,7 @@ jobs:
         rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }}
 
   reset-s3:
-    runs-on: mcf-dev-small-x64
+    runs-on: mcf-dev-large-x64
     container:
       image: mobilecoin/gha-s3-pg-helper:v0
     steps:
@@ -102,8 +100,25 @@ jobs:
         AWS_DEFAULT_REGION: eu-central-1
         BUCKET: mobilecoin.eu.development.chain
         NAMESPACE: ${{ inputs.namespace }}
+      shell: bash
       run: |
+        echo "[profile s3]" > /tmp/aws_cli_profile
+        # shellcheck disable=SC2129 # style issue on multiple echo lines. The alternative is harder to read.
+        echo "s3 =" >> /tmp/aws_cli_profile
+        echo "  max_concurrent_requests = 50" >> /tmp/aws_cli_profile
+        echo "  max_queue_size = 10000" >> /tmp/aws_cli_profile
+
+        export AWS_CONFIG_FILE=/tmp/aws_cli_profile
+        export AWS_PROFILE=s3
+
         for i in 1 2 3
         do
-          aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com"
+            aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" &
+            pids[i]=$!
+        done
+
+        # wait for all pids to finish
+        for pid in "${pids[@]}"
+        do
+            wait "${pid}"
         done
diff --git a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml
@@ -305,6 +305,20 @@ jobs:
               database: fog_recovery
               existingSecret: fog-recovery-postgresql
         postgresqlSharedPreloadLibraries: pgaudit,pg_stat_statements
+        primary:
+          resources:
+            requests:
+              cpu: 1
+              memory: 1Gi
+            limits:
+              memory: 8Gi
+        readReplicas:
+          resources:
+            requests:
+              cpu: 1
+              memory: 1Gi
+            limits:
+              memory: 8Gi
         EOF
 
     - name: Deploy PostgreSQL instance

diff --git a/.internal-ci/test/check-env-status.sh b/.internal-ci/test/check-env-status.sh
@@ -43,9 +43,9 @@ check()
 
 check_timeout()
 {
-    if [[ ${1} -gt 300 ]]
+    if [[ ${1} -gt 3600 ]]
     then
-        echo "Failed to come up in 10m"
+        echo "Failed to come up in 2h"
         exit 1
     fi
     sleep 2