chore: Analyze all captures at once (#9888)

Building from #9873 we now perform analysis in one step over all soaks. This has the added benefit of allowing us to compute a markdown blob for output, which should set us up nicely for #9622. Signed-off-by: Brian L. Troutwine <brian@troutwine.us>
vectordotdev · Nov 4, 2021 · ac2b1f0 · ac2b1f0
1 parent d230d8a
commit ac2b1f0
Show file tree

Hide file tree

Showing 4 changed files with 110 additions and 76 deletions.
diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml
@@ -29,33 +29,83 @@ jobs:
           access_token: ${{ secrets.GITHUB_TOKEN }}
           all_but_latest: true # can cancel workflows scheduled later
 
-  build-baseline-image:
-    name: Build baseline 'soak-vector' container
-    runs-on: [self-hosted, linux, x64, general]
+  compute-test-plan:
+    name: Compute soak test plan
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      #
-      - uses: actions/checkout@v2.4.0
+      - name: Check out the repo
+        uses: actions/checkout@v2
 
-      - uses: actions/checkout@v2.4.0
+      - uses: actions/github-script@v5
+        id: set-matrix
+        with:
+          script: |
+            const fs = require('fs')
+            target = fs.readdirSync('soaks/tests')
+
+            const matrix = {
+              target
+            }
+
+            core.setOutput('matrix', matrix)
+      - name: Dump matrix context
+        env:
+          MATRIX_CONTEXT: ${{ toJson(steps.set-matrix.outputs.matrix) }}
+        run: echo "$MATRIX_CONTEXT"
+
+  compute-soak-meta:
+    name: Compute metadata for soaks
+    runs-on: ubuntu-20.04
+    outputs:
+      comparison-sha: ${{ steps.comparison.outputs.COMPARISON_SHA }}
+      comparison-tag: ${{ steps.comparison.outputs.COMPARISON_TAG }}
+      baseline-sha: ${{ steps.baseline.outputs.BASELINE_SHA }}
+      baseline-tag: ${{ steps.baseline.outputs.BASELINE_TAG }}
+    steps:
+      - uses: actions/checkout@v2.3.5
         with:
           ref: master
           path: baseline-vector
 
-      - name: Set baseline TAG, store in file
+      - name: Setup comparison variables
+        id: comparison
+        run: |
+          export COMPARISON_SHA=${{ github.sha }}
+          export COMPARISON_TAG="${{ github.sha }}-${{ github.sha }}"
+
+          echo "comparison sha is: ${COMPARISON_SHA}"
+          echo "comparison tag is: ${COMPARISON_TAG}"
+
+          echo "::set-output name=COMPARISON_TAG::${COMPARISON_TAG}"
+          echo "::set-output name=COMPARISON_SHA::${COMPARISON_SHA}"
+
+      - name: Setup baseline variables
         id: baseline
         run: |
           pushd baseline-vector
-          export SHA=$(git rev-parse HEAD)
+          export BASELINE_SHA=$(git rev-parse HEAD)
           popd
 
-          echo "::set-output name=TAG::${{ github.sha }}-${SHA}"
-          echo -n "${{ github.sha }}-${SHA}" > tag
+          export BASELINE_TAG="${{ github.sha }}-${BASELINE_SHA}"
+          echo "baseline sha is: ${BASELINE_SHA}"
+          echo "baseline tag is: ${BASELINE_TAG}"
 
-      - name: Upload TAG for baseline
-        uses: actions/upload-artifact@v1
+          echo "::set-output name=BASELINE_TAG::${BASELINE_TAG}"
+          echo "::set-output name=BASELINE_SHA::${BASELINE_SHA}"
+
+  build-baseline-image:
+    name: Build baseline 'soak-vector' container
+    runs-on: [self-hosted, linux, x64, general]
+    needs: [compute-soak-meta]
+    steps:
+      - uses: actions/checkout@v2.3.5
+
+      - uses: actions/checkout@v2.3.5
         with:
-          name: baseline-soak
-          path: tag
+          ref: master
+          path: baseline-vector
 
       - name: Set up Docker Buildx
         id: buildx
@@ -77,7 +127,7 @@ jobs:
             prefix=
             suffix=
           images: ghcr.io/${{ github.repository }}/soak-vector
-          tags: type=raw, value=${{ steps.baseline.outputs.TAG }}
+          tags: type=raw, value=${{ needs.compute-soak-meta.outputs.baseline-tag }}
 
       - name: Build and push 'soak-vector' image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
@@ -94,6 +144,7 @@ jobs:
   build-comparison-image:
     name: Build comparison 'soak-vector' container
     runs-on: [self-hosted, linux, x64, general]
+    needs: [compute-soak-meta]
     steps:
       - uses: actions/checkout@v2.4.0
 
@@ -116,7 +167,7 @@ jobs:
             prefix=
             suffix=
           images: ghcr.io/${{ github.repository }}/soak-vector
-          tags: type=raw, value=${{ github.sha }}-${{ github.sha }}
+          tags: type=raw, value=${{ needs.compute-soak-meta.outputs.comparison-tag }}
 
       - name: Build and push 'soak-vector' image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
@@ -129,53 +180,22 @@ jobs:
           cache-from: type=gha, scope=${{ github.workflow }}
           cache-to: type=gha, scope=${{ github.workflow }}
 
-  compute-test-plan:
-    name: Compute soak test plan
-    runs-on: ubuntu-20.04
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2.4.0
-
-      - uses: actions/github-script@v5
-        id: set-matrix
-        with:
-          script: |
-            const fs = require('fs')
-            target = fs.readdirSync('soaks/tests')
-
-            const matrix = {
-              target
-            }
-
-            core.setOutput('matrix', matrix)
-      - name: Dump matrix context
-        env:
-          MATRIX_CONTEXT: ${{ toJson(steps.set-matrix.outputs.matrix) }}
-        run: echo "$MATRIX_CONTEXT"
-
   soak-baseline:
     name: Soak (${{ matrix.target }}) - baseline
     if: ${{ github.actor != 'dependabot[bot]' }}
     runs-on: [self-hosted, linux, x64, soak]
-    needs: [compute-test-plan, build-baseline-image]
+    needs: [compute-soak-meta, compute-test-plan, build-baseline-image]
     strategy:
       matrix: ${{ fromJson(needs.compute-test-plan.outputs.matrix) }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2.4.0
 
-      - name: Download baseline-soak artifact
-        uses: actions/download-artifact@v1
-        with:
-          name: baseline-soak
-
       - name: Run baseline experiment
         run: |
           rm -rf /tmp/${{ github.event.number }}/${{ matrix.target }}/
           mkdir -p /tmp/${{ github.event.number }}/${{ matrix.target }}/
-          ./soaks/bin/soak_one.sh "false" ${{ matrix.target }} "baseline" `cat baseline-soak/tag` /tmp/${{ github.event.number }}/${{ matrix.target }}
+          ./soaks/bin/soak_one.sh "false" ${{ matrix.target }} "baseline" ${{ needs.compute-soak-meta.outputs.baseline-tag }} /tmp/${{ github.event.number }}
 
       - name: Upload timing captures
         uses: actions/upload-artifact@v1
@@ -187,23 +207,18 @@ jobs:
     name: Soak (${{ matrix.target }}) - comparison
     if: ${{ github.actor != 'dependabot[bot]' }}
     runs-on: [self-hosted, linux, x64, soak]
-    needs: [compute-test-plan, build-comparison-image]
+    needs: [compute-soak-meta, compute-test-plan, build-comparison-image]
     strategy:
       matrix: ${{ fromJson(needs.compute-test-plan.outputs.matrix) }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2.4.0
 
-      - name: Download baseline-soak artifact
-        uses: actions/download-artifact@v1
-        with:
-          name: baseline-soak
-
       - name: Run comparison experiment
         run: |
           rm -rf /tmp/${{ github.event.number }}/${{ matrix.target }}/
           mkdir -p /tmp/${{ github.event.number }}/${{ matrix.target }}/
-          ./soaks/bin/soak_one.sh "false" ${{ matrix.target }} "comparison" "${{ github.sha }}-${{ github.sha }}" /tmp/${{ github.event.number }}/${{ matrix.target }}
+          ./soaks/bin/soak_one.sh "false" ${{ matrix.target }} "comparison" ${{ needs.compute-soak-meta.outputs.comparison-tag }} /tmp/${{ github.event.number }}
 
       - name: Upload timing captures
         uses: actions/upload-artifact@v1
@@ -212,13 +227,14 @@ jobs:
           path: /tmp/${{ github.event.number }}
 
   analyze-results:
-    name: Soak analysis (${{ matrix.target }})
+    name: Soak analysis
     if: ${{ github.actor != 'dependabot[bot]' }}
-    runs-on: [self-hosted, linux, x64, soak] # could be general if we move away from miller
-    needs: [compute-test-plan, soak-baseline, soak-comparison]
-    strategy:
-      matrix: ${{ fromJson(needs.compute-test-plan.outputs.matrix) }}
+    runs-on: ubuntu-20.04
+    needs: [compute-soak-meta, soak-baseline, soak-comparison]
     steps:
+      - name: Install 'miller'
+        run: sudo apt-get install -y miller
+
       - name: Check out the repo
         uses: actions/checkout@v2.4.0
 
@@ -229,11 +245,11 @@ jobs:
 
       - name: Analyze captures
         run: |
-          ./soaks/bin/analyze_experiment.sh ${{ github.event.number }}-captures/${{ matrix.target }}
+          ./soaks/bin/analyze_experiment.sh ${{ github.event.number }}-captures/ ${{ needs.compute-soak-meta.outputs.baseline-sha }} ${{ needs.compute-soak-meta.outputs.comparison-sha }}
 
   observer:
     name: Build and push 'observer' to Github CR
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2.4.0

diff --git a/soaks/bin/analyze_experiment.sh b/soaks/bin/analyze_experiment.sh
@@ -7,20 +7,36 @@ set -o nounset
 
 display_usage() {
     echo ""
-    echo "Usage: $0 CAPTURE_DIR"
+    echo "Usage: $0  CAPTURE_DIR BASELINE_SHA COMPARISON_SHA"
 }
 
 CAPTURE_DIR="${1}"
+BASELINE_SHA="${2}"
+COMPARISON_SHA="${3}"
 
-echo "Captures recorded into ${CAPTURE_DIR}"
+echo "Captures recorded between ${BASELINE_SHA} and ${COMPARISON_SHA}"
 echo ""
-echo "Here is a statistical summary of the soak captures. Units are bytes,"
-echo "except for 'skewness' and 'kurtosis'. Higher numbers in 'comparison'"
+echo "Here is a statistical summary of the soak captures between the SHAs given"
+echo "above. Units are bytes, except for 'skewness' and 'kurtosis'. Higher numbers in 'comparison'"
 echo "is generally better. Higher skewness or kurtosis numbers indicate a"
 echo "lack of consistency in behavior, making predictions of fitness in the"
 echo "field challenging."
 echo ""
-mlr --tsv \
-    --from "${CAPTURE_DIR}/baseline.captures" \
-    --from "${CAPTURE_DIR}/comparison.captures" \
-    stats1 -a 'min,p90,p99,max,skewness,kurtosis' -g EXPERIMENT -f VALUE
+for soak_dir in "${CAPTURE_DIR}"/*; do
+    SOAK_NAME=$(basename "${soak_dir}")
+    echo " --- "
+    echo "# \`${SOAK_NAME}\`"
+    # NOTE if you change the statistics being pulled here please update the
+    # header/body divisor below. Consider that you need one column for the group
+    # and one for each statistic.
+    OUTPUT=$(mlr --tsv \
+        --from "${soak_dir}/baseline.captures" \
+        --from "${soak_dir}/comparison.captures" \
+        stats1 -a 'min,p90,p99,max,skewness,kurtosis' -g EXPERIMENT -f VALUE)
+    HEADER=$(echo "${OUTPUT}" | head -n1)
+    BODY=$(echo "${OUTPUT}" | tail -n+2)
+
+    echo "${HEADER}" | sed 's/^/|\ /g' | sed 's/\t/\ |\ /g' | sed 's/$/\ |/g'
+    echo "| --- | --- | --- | --- | --- | --- | --- |"
+    echo "${BODY}" | sed 's/^/|\ /g' | sed 's/\t/\ |\ /g' | sed 's/$/\ |/g'
+done
diff --git a/soaks/bin/run_experiment.sh b/soaks/bin/run_experiment.sh
@@ -10,7 +10,7 @@ SOAK_ROOT="${__dir}/.."
 
 display_usage() {
     echo ""
-    echo "Usage: $0 CAPTURE_DIR VARIANT IMAGE"
+    echo "Usage: $0 CAPTURE_DIR VARIANT IMAGE SOAK_NAME"
 }
 
 CAPTURE_DIR="${1}"
@@ -26,20 +26,22 @@ then
     exit 1
 fi
 
+SOAK_CAPTURE_DIR="${CAPTURE_DIR}/${SOAK_NAME}"
+
 pushd "${__dir}"
 ./boot_minikube.sh
-minikube mount "${CAPTURE_DIR}:/captures" &
+minikube mount "${SOAK_CAPTURE_DIR}:/captures" &
 minikube cache add "${IMAGE}"
 MOUNT_PID=$!
 popd
 
 pushd "${SOAK_ROOT}/tests/${SOAK_NAME}/terraform"
 terraform init
 terraform apply -var "type=${VARIANT}" -var "vector_image=${IMAGE}" -auto-approve -compact-warnings -input=false -no-color -parallelism=20
-echo "[${VARIANT}] Captures will be recorded into ${CAPTURE_DIR}"
+echo "[${VARIANT}] Captures will be recorded into ${SOAK_CAPTURE_DIR}"
 echo "[${VARIANT}] Sleeping for ${WARMUP_GRACE} seconds to allow warm-up"
 sleep "${WARMUP_GRACE}"
-echo "[${VARIANT}] Recording captures to ${CAPTURE_DIR}"
+echo "[${VARIANT}] Recording captures to ${SOAK_CAPTURE_DIR}"
 sleep "${TOTAL_SAMPLES}"
 kill "${MOUNT_PID}"
 popd

diff --git a/soaks/soak.sh b/soaks/soak.sh
@@ -71,4 +71,4 @@ echo "Captures will be recorded into ${capture_dir}"
 
 popd
 
-./bin/analyze_experiment.sh "${capture_dir}"
+./bin/analyze_experiment.sh "${capture_dir}" "${BASELINE}" "${COMPARISON}"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -71,4 +71,4 @@ echo "Captures will be recorded into ${capture_dir}"

		popd

		./bin/analyze_experiment.sh "${capture_dir}"
		./bin/analyze_experiment.sh "${capture_dir}" "${BASELINE}" "${COMPARISON}"