From fdb01c0434d9a70c8f3f02e2544367c10e41b346 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Mon, 1 Nov 2021 11:30:56 -0700 Subject: [PATCH] chore: Run soaks in CI (#9818) * Run soaks in CI This commit introduces a new workflow 'soak' that runs the soaks introduced in #9699 in Github Action CI. There is, at present, no PR reporting and every push to the project generates a soak run, except those done by dependabot. The soak.sh script has been changed in its interface as well. Please see README for details. Closes #9619 Closes #9618 Closes #9620 Signed-off-by: Brian L. Troutwine * fix check-script dings Signed-off-by: Brian L. Troutwine * remove build-args, clean runner Signed-off-by: Brian L. Troutwine * pushd into baseline-vector to calculate sha Signed-off-by: Brian L. Troutwine * ignore master <> master soak Signed-off-by: Brian L. Troutwine * adjust tag structure, documentation Signed-off-by: Brian L. Troutwine * correctly compute tag for baseline Signed-off-by: Brian L. Troutwine --- .github/workflows/soak.yml | 263 ++++++++++++++++++ .github/workflows/soak_lib.yml | 101 ------- soaks/Dockerfile | 12 +- soaks/README.md | 25 +- soaks/bin/analyze_experiment.sh | 26 ++ soaks/bin/boot_minikube.sh | 16 +- soaks/bin/build_container.sh | 19 +- soaks/bin/container_name.sh | 30 -- soaks/bin/run_experiment.sh | 49 ++++ soaks/bin/soak_one.sh | 36 +++ .../datadog_agent_remap_datadog_logs/FEATURES | 1 - soaks/soak.sh | 116 ++++---- soaks/syslog_humio_logs/FEATURES | 1 - .../terraform/.terraform.lock.hcl | 2 + .../syslog_log2metric_humio_metrics/FEATURES | 1 - soaks/syslog_loki/FEATURES | 1 - soaks/syslog_loki/terraform/variables.tf | 5 - .../FEATURES | 1 - .../terraform/variables.tf | 5 - soaks/syslog_splunk_hec_logs/FEATURES | 1 - 20 files changed, 448 insertions(+), 263 deletions(-) create mode 100644 .github/workflows/soak.yml delete mode 100644 .github/workflows/soak_lib.yml create mode 100755 soaks/bin/analyze_experiment.sh delete mode 100755 soaks/bin/container_name.sh create mode 100755 soaks/bin/run_experiment.sh create mode 100755 soaks/bin/soak_one.sh delete mode 100644 soaks/datadog_agent_remap_datadog_logs/FEATURES delete mode 100644 soaks/syslog_humio_logs/FEATURES delete mode 100644 soaks/syslog_log2metric_humio_metrics/FEATURES delete mode 100644 soaks/syslog_loki/FEATURES delete mode 100644 soaks/syslog_regex_logs2metric_ddmetrics/FEATURES delete mode 100644 soaks/syslog_splunk_hec_logs/FEATURES diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml new file mode 100644 index 0000000000000..65232e4d55743 --- /dev/null +++ b/.github/workflows/soak.yml @@ -0,0 +1,263 @@ +# Soak test vector +# +# This workflow runs our 'soak' tests, which are relative evaluations of +# vector's master branch HEAD to whatever SHA was just pushed into the project +# (unless that SHA happens to be master branch HEAD). The goal is to give +# quick-ish feedback on all-up vector for a variety of configs as to whether +# throughput performance has gone down, gotten more variable in the pushed SHA. +# +# Soaks are always done relative to the pushed SHA, meaning any changes you +# introduce to the soak tests will be picked up both for the master HEAD soak +# and your current SHA. Tags are SHA-SHA. The first SHA is the one that +# triggered this workflow, the second is the one of the vector being tested. For +# comparison images the two SHAs are identical. +name: Soak + +on: + push: + branches-ignore: + - master + +jobs: + cancel-previous: + runs-on: ubuntu-20.04 + timeout-minutes: 3 + if: github.ref != 'refs/heads/master' + steps: + - uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + all_but_latest: true # can cancel workflows scheduled later + + build-baseline-image: + name: Build baseline 'soak-vector' container + runs-on: [self-hosted, linux, x64, general] + steps: + # + - uses: actions/checkout@v2.3.5 + + - uses: actions/checkout@v2.3.5 + with: + ref: master + path: baseline-vector + + - name: Set baseline TAG, store in file + id: baseline + run: | + pushd baseline-vector + export SHA=$(git rev-parse HEAD) + popd + + echo "::set-output name=TAG::${{ github.sha }}-${SHA}" + echo -n "${{ github.sha }}-${SHA}" > tag + + - name: Upload TAG for baseline + uses: actions/upload-artifact@v1 + with: + name: baseline-soak + path: tag + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + flavor: | + latest=false + prefix= + suffix= + images: ghcr.io/${{ github.repository }}/soak-vector + tags: type=raw, value=${{ steps.baseline.outputs.TAG }} + + - name: Build and push 'soak-vector' image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: baseline-vector/ + file: soaks/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha, scope=${{ github.workflow }} + cache-to: type=gha, scope=${{ github.workflow }} + + build-comparison-image: + name: Build comparison 'soak-vector' container + runs-on: [self-hosted, linux, x64, general] + steps: + - uses: actions/checkout@v2.3.5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + flavor: | + latest=false + prefix= + suffix= + images: ghcr.io/${{ github.repository }}/soak-vector + tags: type=raw, value=${{ github.sha }}-${{ github.sha }} + + - name: Build and push 'soak-vector' image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + file: soaks/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha, scope=${{ github.workflow }} + cache-to: type=gha, scope=${{ github.workflow }} + + soak-baseline: + name: Soak (${{ matrix.target }}) - baseline + if: ${{ github.actor != 'dependabot[bot]' }} + runs-on: [self-hosted, linux, x64, soak] + needs: [build-baseline-image] + strategy: + matrix: + target: + - datadog_agent_remap_datadog_logs + - syslog_humio_logs + - syslog_log2metric_humio_metrics + - syslog_loki + - syslog_regex_logs2metric_ddmetrics + - syslog_splunk_hec_logs + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Download baseline-soak artifact + uses: actions/download-artifact@v1 + with: + name: baseline-soak + + - name: Run baseline experiment + run: | + rm -rf /tmp/${{ matrix.target }}/ + mkdir -p /tmp/${{ matrix.target }}/ + ./soaks/bin/soak_one.sh "false" ${{ matrix.target }} "baseline" `cat baseline-soak/tag` /tmp/${{ matrix.target }} + + - name: Upload timing captures + uses: actions/upload-artifact@v1 + with: + name: ${{ matrix.target}}-captures + path: /tmp/${{ matrix.target }} + + soak-comparison: + name: Soak (${{ matrix.target }}) - comparison + if: ${{ github.actor != 'dependabot[bot]' }} + runs-on: [self-hosted, linux, x64, soak] + needs: [build-comparison-image] + strategy: + matrix: + target: + - datadog_agent_remap_datadog_logs + - syslog_humio_logs + - syslog_log2metric_humio_metrics + - syslog_loki + - syslog_regex_logs2metric_ddmetrics + - syslog_splunk_hec_logs + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Download baseline-soak artifact + uses: actions/download-artifact@v1 + with: + name: baseline-soak + + - name: Run comparison experiment + run: | + rm -rf /tmp/${{ matrix.target }}/ + mkdir -p /tmp/${{ matrix.target }}/ + ./soaks/bin/soak_one.sh "false" ${{ matrix.target }} "comparison" "${{ github.sha }}-${{ github.sha }}" /tmp/${{ matrix.target }} + + - name: Upload timing captures + uses: actions/upload-artifact@v1 + with: + name: ${{ matrix.target}}-captures + path: /tmp/${{ matrix.target }} + + analyze-results: + name: Soak analysis (${{ matrix.target }}) + if: ${{ github.actor != 'dependabot[bot]' }} + runs-on: [self-hosted, linux, x64, soak] # could be general if we move away from miller + needs: [soak-baseline, soak-comparison] + strategy: + matrix: + target: + - datadog_agent_remap_datadog_logs + - syslog_humio_logs + - syslog_log2metric_humio_metrics + - syslog_loki + - syslog_regex_logs2metric_ddmetrics + - syslog_splunk_hec_logs + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Download captures artifact + uses: actions/download-artifact@v1 + with: + name: ${{ matrix.target }}-captures + + - name: Analyze captures + run: | + ./soaks/bin/analyze_experiment.sh ${{ matrix.target }}-captures + + observer: + name: Build and push 'observer' to Github CR + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ghcr.io/${{ github.repository }}/soak-observer + tags: type=sha, format=long + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + file: lib/soak/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha, scope=${{ github.workflow }} + cache-to: type=gha, scope=${{ github.workflow }} diff --git a/.github/workflows/soak_lib.yml b/.github/workflows/soak_lib.yml deleted file mode 100644 index 519830f034439..0000000000000 --- a/.github/workflows/soak_lib.yml +++ /dev/null @@ -1,101 +0,0 @@ -# Workflow to build soak related containers. These trigger per-push. - -name: Soak Infra - -on: push - -jobs: - vector-soak: - name: Build 'soak-vector' container (${{ matrix.target }}) - if: ${{ github.actor != 'dependabot[bot]' }} - runs-on: [self-hosted, linux, x64, general] - strategy: - matrix: - target: - - datadog_agent_remap_datadog_logs - - syslog_humio_logs - - syslog_log2metric_humio_metrics - - syslog_loki - - syslog_regex_logs2metric_ddmetrics - - syslog_splunk_hec_logs - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Compute feature flag SHA - run: | - . "soaks/${{ matrix.target }}/FEATURES" - echo "::set-output name=FEATURE_SHA::$(echo -n "${FEATURES}" | sha256sum - | head -c40)" - id: flag_sha - - - name: Compute feature flags - run: | - . "soaks/${{ matrix.target }}/FEATURES" - echo "::set-output name=FEATURES::$(echo "${FEATURES}")" - id: features - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@548e2346a9987b56d8a4104fe776321ff8e23440 - with: - flavor: | - latest=false - prefix= - suffix= - labels: | - soak_test_name=${{ matrix.target }} - images: ghcr.io/${{ github.repository }}/soak-vector - tags: type=raw, value=${{ steps.flag_sha.outputs.FEATURE_SHA }}-${{ github.sha }} - - - name: Build and push 'soak-vector' image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - file: soaks/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - VECTOR_FEATURES=${{ steps.features.outputs.FEATURES }} - - observer: - name: Build and push 'observer' to Github CR - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@548e2346a9987b56d8a4104fe776321ff8e23440 - with: - images: ghcr.io/${{ github.repository }}/soak-observer - tags: type=sha, format=long - - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - file: lib/soak/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/soaks/Dockerfile b/soaks/Dockerfile index de4b50ea51bea..632127fbb34c2 100644 --- a/soaks/Dockerfile +++ b/soaks/Dockerfile @@ -2,19 +2,21 @@ ARG RUST_VERSION=1.56 ARG DEBIAN_VERSION=bullseye # -# BUILDER +# VECTOR BUILDER # FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_VERSION} as builder -RUN apt-get -y update && apt-get -y install build-essential cmake libclang-dev libsasl2-dev +RUN apt-get update && apt-get -y install build-essential git clang cmake libclang-dev libsasl2-dev libstdc++-10-dev libssl-dev libxxhash-dev zlib1g-dev zlib1g +RUN git clone https://github.com/rui314/mold.git +RUN cd mold && git checkout v0.9.6 && make -j$(nproc) && make install WORKDIR vector -ARG VECTOR_FEATURES COPY . . -RUN cargo build --release --bin vector --no-default-features --features "${VECTOR_FEATURES}" +RUN /usr/bin/mold -run cargo build --release --bin vector # # TARGET # -FROM gcr.io/distroless/cc-debian11 +FROM docker.io/debian:bullseye-slim +RUN apt-get update && apt-get -y install zlib1g COPY --from=builder /vector/target/release/vector /usr/bin/vector VOLUME /var/lib/vector/ diff --git a/soaks/README.md b/soaks/README.md index 41778cb23ab98..4af7a2d4397ac 100644 --- a/soaks/README.md +++ b/soaks/README.md @@ -34,17 +34,14 @@ support programs in a minikube and some glue code to observe vector in operation. Consider this command: ```shell -> ./soaks/soak.sh datadog_agent_remap_datadog_logs a32c7fd09978f76a3f1bd360c3a8d07a49538b70 be8ceafbf994d06f505bdd9fb392b00e0ba661f2 +> ./soaks/soak.sh --local-image --soak datadog_agent_remap_datadog_logs --baseline a32c7fd09978f76a3f1bd360c3a8d07a49538b70 --comparison be8ceafbf994d06f505bdd9fb392b00e0ba661f2 ``` Here we run the soak test `datadog_agent_remap_datadog_logs` comparing vector at `a32c7fd09978f76a3f1bd360c3a8d07a49538b70` with vector at `be8ceafbf994d06f505bdd9fb392b00e0ba661f2`. Two vector containers will be built -for each SHA. Time is saved by building vector only against the features needed -to complete the experiment. Users define these flags in files named `FEATURES` -under the soak directory, see -[`soaks/datadog_agent_remap_datadog_logs/FEATURES`]. The soak itself is defined -in terraform, see [`soaks/datadog_agent_remap_datadog_logs/terraform`]. +for each SHA. The soak itself is defined in terraform, see +[`soaks/datadog_agent_remap_datadog_logs/terraform`]. After running this command you will, in about ten minutes depending on whether you need to build containers or not, see a summary: @@ -89,7 +86,6 @@ the "Datadog Agent -> Remap -> Datadog Logs" soak in ```shell > tree datadog_agent_remap_datadog_logs datadog_agent_remap_datadog_logs -├── FEATURES ├── README.md └── terraform ├── http_blackhole.toml @@ -103,18 +99,9 @@ datadog_agent_remap_datadog_logs 1 directory, 9 files ``` -The `FEATURES` file defines which feature flags will be lit when vector is -built. As of this writing that file contains: - -```shell -FEATURES="sources-internal_metrics,sinks-prometheus,sources-datadog_agent,sinks-datadog_logs,transforms-remap" -``` - -This is a shell include file. You must set the features you need for vector to -run for your test and the fewer flags you include the faster your build time -will be. The `terraform/` sub-directory contains a small project -definition. It's clear we can thin this out further -- the prometheus setup is -common to all soaks -- but the primary things you need to concern yourself with are: +The `terraform/` sub-directory contains a small project definition. It's clear +we can thin this out further -- the prometheus setup is common to all soaks -- +but the primary things you need to concern yourself with are: * `main.tf` * `vector.toml` diff --git a/soaks/bin/analyze_experiment.sh b/soaks/bin/analyze_experiment.sh new file mode 100755 index 0000000000000..49eb0a14734f1 --- /dev/null +++ b/soaks/bin/analyze_experiment.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +#set -o xtrace + +display_usage() { + echo "" + echo "Usage: $0 CAPTURE_DIR" +} + +CAPTURE_DIR="${1}" + +echo "Captures recorded into ${CAPTURE_DIR}" +echo "" +echo "Here is a statistical summary of the soak captures. Units are bytes," +echo "except for 'skewness' and 'kurtosis'. Higher numbers in 'comparison'" +echo "is generally better. Higher skewness or kurtosis numbers indicate a" +echo "lack of consistency in behavior, making predictions of fitness in the" +echo "field challenging." +echo "" +mlr --tsv \ + --from "${CAPTURE_DIR}/baseline.captures" \ + --from "${CAPTURE_DIR}/comparison.captures" \ + stats1 -a 'min,p90,p99,max,skewness,kurtosis' -g EXPERIMENT -f VALUE diff --git a/soaks/bin/boot_minikube.sh b/soaks/bin/boot_minikube.sh index 00e78fca61947..f890465137c64 100755 --- a/soaks/bin/boot_minikube.sh +++ b/soaks/bin/boot_minikube.sh @@ -5,20 +5,6 @@ set -o pipefail set -o nounset #set -o xtrace -display_usage() { - echo -e "\nUsage: \$0 IMAGE\n" -} - -if [ $# -le 0 ] -then - display_usage - exit 1 -fi - -IMG="${1:-}" - minikube stop || true minikube delete || true -minikube start --cpus=7 --memory=8g - -minikube image load "${IMG}" +minikube start --cpus=7 --memory=8g --driver=docker diff --git a/soaks/bin/build_container.sh b/soaks/bin/build_container.sh index 3f0f5d37d5340..b5e066ea4ed60 100755 --- a/soaks/bin/build_container.sh +++ b/soaks/bin/build_container.sh @@ -5,16 +5,12 @@ set -o pipefail set -o nounset #set -o xtrace -# We need to build two copies of vector with the same flags, one for the -# baseline SHA and the other for current. Baseline is either 'master' or -# whatever the user sets. - __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SOAK_ROOT="${__dir}/.." -PATCH_DIR="${SOAK_ROOT}/patches" display_usage() { - echo -e "\nUsage: \$0 SOAK_NAME COMMIT_SHA\n" + echo "" + echo "Usage: $0 COMMIT_SHA IMAGE_NAME" } build_vector() { @@ -29,11 +25,10 @@ build_vector() { git remote add origin https://github.com/vectordotdev/vector.git git fetch --depth 1 origin "${SHA}" git checkout FETCH_HEAD - git apply "${PATCH_DIR}/blank_global_dockerfileignore.patch" popd fi - docker build --file "${SOAK_ROOT}/Dockerfile" --build-arg=VECTOR_FEATURES="${FEATURES}" --tag "${IMAGE}" "${BUILD_DIR}" + docker build --file "${SOAK_ROOT}/Dockerfile" --tag "${IMAGE}" "${BUILD_DIR}" rm -rf "${BUILD_DIR}" } @@ -43,11 +38,7 @@ then exit 1 fi -SOAK_NAME="${1:-}" -COMMIT_SHA="${2:-}" - -# shellcheck source=/dev/null -. "${SOAK_ROOT}/${SOAK_NAME}/FEATURES" +COMMIT_SHA="${1:-}" +IMAGE="${2:-}" -IMAGE=$(./bin/container_name.sh "${SOAK_NAME}" "${COMMIT_SHA}") docker image inspect "${IMAGE}" > /dev/null || build_vector "${IMAGE}" "${COMMIT_SHA}" diff --git a/soaks/bin/container_name.sh b/soaks/bin/container_name.sh deleted file mode 100755 index 2b804a11c56c9..0000000000000 --- a/soaks/bin/container_name.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail -set -o nounset -#set -o xtrace - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SOAK_ROOT="${__dir}/.." - -display_usage() { - echo -e "\nUsage: \$0 SOAK_NAME COMMIT_SHA\n" -} - -if [ $# -le 1 ] -then - display_usage - exit 1 -fi - -SOAK_NAME="${1:-}" -COMMIT_SHA="${2:-}" - -# shellcheck source=/dev/null -. "${SOAK_ROOT}/${SOAK_NAME}/FEATURES" - -FEATURE_SHA=$(echo -n "${FEATURES}" | sha256sum - | head -c40) -IMAGE="vector:${COMMIT_SHA}-${FEATURE_SHA}" - -echo "${IMAGE}" diff --git a/soaks/bin/run_experiment.sh b/soaks/bin/run_experiment.sh new file mode 100755 index 0000000000000..16634eff3cee6 --- /dev/null +++ b/soaks/bin/run_experiment.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +#set -o xtrace + +__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOAK_ROOT="${__dir}/.." + +display_usage() { + echo "" + echo "Usage: $0 CAPTURE_DIR VARIANT IMAGE" +} + +CAPTURE_DIR="${1}" +VARIANT="${2}" +IMAGE="${3}" +SOAK_NAME="${4}" +WARMUP_GRACE=90 +TOTAL_SAMPLES=120 + +if [ $# -le 1 ] +then + display_usage + exit 1 +fi + +pushd "${__dir}" +./boot_minikube.sh +minikube mount "${CAPTURE_DIR}:/captures" & +minikube cache add "${IMAGE}" +MOUNT_PID=$! +popd + +pushd "${SOAK_ROOT}/${SOAK_NAME}/terraform" +terraform init +terraform apply -var "type=${VARIANT}" -var "vector_image=${IMAGE}" -auto-approve -compact-warnings -input=false -no-color -parallelism=20 +echo "[${VARIANT}] Captures will be recorded into ${CAPTURE_DIR}" +echo "[${VARIANT}] Sleeping for ${WARMUP_GRACE} seconds to allow warm-up" +sleep "${WARMUP_GRACE}" +echo "[${VARIANT}] Recording captures to ${CAPTURE_DIR}" +sleep "${TOTAL_SAMPLES}" +kill "${MOUNT_PID}" +popd + +pushd "${__dir}" +./shutdown_minikube.sh +popd diff --git a/soaks/bin/soak_one.sh b/soaks/bin/soak_one.sh new file mode 100755 index 0000000000000..b4145b82799db --- /dev/null +++ b/soaks/bin/soak_one.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +#set -o xtrace + +__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +display_usage() { + echo "" + echo "Usage: $0 USE_LOCAL_IMAGE SOAK_NAME VARIANT TAG CAPTURE_DIR" +} + +USE_LOCAL_IMAGE="${1}" +SOAK_NAME="${2}" +VARIANT="${3}" +TAG="${4}" +CAPTURE_DIR="${5}" + +pushd "${__dir}" + +IMAGE="vector:${TAG}" +if [ "${USE_LOCAL_IMAGE}" = "true" ]; then + echo "Building images locally..." + + ./build_container.sh "${TAG}" "${IMAGE}" +else + REMOTE_IMAGE="ghcr.io/vectordotdev/vector/soak-vector:${TAG}" + docker pull "${REMOTE_IMAGE}" + docker image tag "${REMOTE_IMAGE}" "${IMAGE}" +fi + +./run_experiment.sh "${CAPTURE_DIR}" "${VARIANT}" "${IMAGE}" "${SOAK_NAME}" + +popd diff --git a/soaks/datadog_agent_remap_datadog_logs/FEATURES b/soaks/datadog_agent_remap_datadog_logs/FEATURES deleted file mode 100644 index 1035f0dad0d56..0000000000000 --- a/soaks/datadog_agent_remap_datadog_logs/FEATURES +++ /dev/null @@ -1 +0,0 @@ -FEATURES="sources-internal_metrics,sinks-prometheus,sources-datadog_agent,sinks-datadog_logs,transforms-remap" diff --git a/soaks/soak.sh b/soaks/soak.sh index 7193bdf03d5c3..3911c9ab1c803 100755 --- a/soaks/soak.sh +++ b/soaks/soak.sh @@ -3,82 +3,72 @@ set -o errexit set -o pipefail set -o nounset -set -o xtrace +#set -o xtrace __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" display_usage() { - echo -e "\nUsage: \$0 SOAK_NAME BASELINE_SHA COMPARISON_SHA\n" + echo "" + echo "Usage: soak [OPTIONS]" + echo "" + echo "Options:" + echo " --help: display this information" + echo " --soak: the experiment to run" + echo " --remote-image|--local-image: whether to use a local vector image or remote" + echo " --baseline: the baseline SHA to compare against" + echo " --comparison: the SHA to compare against 'baseline'" + echo "" } -if [ $# -le 1 ] -then - display_usage - exit 1 -fi +USE_LOCAL_IMAGE="true" -SOAK_NAME="${1:-}" -BASELINE="${2:-}" -COMPARISON="${3:-}" -WARMUP_GRACE=90 -TOTAL_SAMPLES=120 +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --soak) + SOAK_NAME=$2 + shift # past argument + shift # past value + ;; + --baseline) + BASELINE=$2 + shift # past argument + shift # past value + ;; + --comparison) + COMPARISON=$2 + shift # past argument + shift # past value + ;; + --remote-image) + USE_LOCAL_IMAGE="false" + shift # past argument + ;; + --local-image) + USE_LOCAL_IMAGE="true" + shift # past argument + ;; + --help) + display_usage + exit 0 + ;; + *) + echo "unknown option" + display_usage + exit 1 + ;; + esac +done pushd "${__dir}" -./bin/build_container.sh "${SOAK_NAME}" "${BASELINE}" -./bin/build_container.sh "${SOAK_NAME}" "${COMPARISON}" -BASELINE_IMAGE=$(./bin/container_name.sh "${SOAK_NAME}" "${BASELINE}") -COMPARISON_IMAGE=$(./bin/container_name.sh "${SOAK_NAME}" "${COMPARISON}") capture_dir=$(mktemp -d /tmp/"${SOAK_NAME}".XXXXXX) echo "Captures will be recorded into ${capture_dir}" -# -# BASELINE -# -./bin/boot_minikube.sh "${BASELINE_IMAGE}" -minikube mount "${capture_dir}:/captures" & -MOUNT_PID=$! - -pushd "${__dir}/${SOAK_NAME}/terraform" -terraform init -terraform apply -var 'type=baseline' -var 'type=baseline' -var "vector_image=${BASELINE_IMAGE}" --auto-approve -echo "Captures will be recorded into ${capture_dir}" -echo "Sleeping for ${WARMUP_GRACE} seconds to allow warm-up" -sleep "${WARMUP_GRACE}" -echo "Recording 'baseline' captures to ${capture_dir}" -sleep "${TOTAL_SAMPLES}" -kill "${MOUNT_PID}" -popd -./bin/shutdown_minikube.sh - - -# -# COMPARISON -# -./bin/boot_minikube.sh "${COMPARISON_IMAGE}" -minikube mount "${capture_dir}:/captures" & -MOUNT_PID=$! - -pushd "${__dir}/${SOAK_NAME}/terraform" -terraform init -terraform apply -var 'type=comparison' -var 'type=comparison' -var "vector_image=${COMPARISON_IMAGE}" --auto-approve -echo "Captures will be recorded into ${capture_dir}" -echo "Sleeping for ${WARMUP_GRACE} seconds to allow warm-up" -sleep "${WARMUP_GRACE}" -echo "Recording 'comparison' captures to ${capture_dir}" -sleep "${TOTAL_SAMPLES}" -kill "${MOUNT_PID}" -popd -./bin/shutdown_minikube.sh +./bin/soak_one.sh "${USE_LOCAL_IMAGE}" "${SOAK_NAME}" "baseline" "${BASELINE}" "${capture_dir}" +./bin/soak_one.sh "${USE_LOCAL_IMAGE}" "${SOAK_NAME}" "comparison" "${COMPARISON}" "${capture_dir}" popd -echo "Captures recorded into ${capture_dir}" -echo "" -echo "Here is a statistical summary of that file. Units are bytes." -echo "Higher numbers in the 'comparison' is better." -echo "" -mlr --tsv \ - --from "${capture_dir}/baseline.captures" \ - --from "${capture_dir}/comparison.captures" \ - stats1 -a 'min,p90,p99,max,skewness,kurtosis' -g EXPERIMENT -f VALUE +./bin/analyze_experiment.sh "${capture_dir}" diff --git a/soaks/syslog_humio_logs/FEATURES b/soaks/syslog_humio_logs/FEATURES deleted file mode 100644 index 413e723e754a2..0000000000000 --- a/soaks/syslog_humio_logs/FEATURES +++ /dev/null @@ -1 +0,0 @@ -FEATURES="sources-internal_metrics,sinks-prometheus,sources-syslog,sinks-humio" diff --git a/soaks/syslog_humio_logs/terraform/.terraform.lock.hcl b/soaks/syslog_humio_logs/terraform/.terraform.lock.hcl index e320b848c092d..1ef381c052713 100644 --- a/soaks/syslog_humio_logs/terraform/.terraform.lock.hcl +++ b/soaks/syslog_humio_logs/terraform/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/hashicorp/kubernetes" { constraints = "~> 2.5.0" hashes = [ "h1:N++QrDDOE8L/FSYxFQSEicuwsUkh5V+OHFOJ4zRzLMc=", + "h1:wg+6NFJUKrwccLZQPkJTgdRMgh709ZX+sBuJ9IlJoNU=", "zh:05f42363a1bdf1858bfdb56dc55509a066a640d05ef62e0fddcf7cc4d97a0dfa", "zh:2b974bebbc7823f759159fcf1efe0717930b7756f643ff1551e2d633a6651d10", "zh:329817c534ae807ae88030f78e5c1ad1f124e9d5d7ae2830229a6d18ba7dd22e", @@ -24,6 +25,7 @@ provider "registry.terraform.io/hashicorp/template" { version = "2.2.0" hashes = [ "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", diff --git a/soaks/syslog_log2metric_humio_metrics/FEATURES b/soaks/syslog_log2metric_humio_metrics/FEATURES deleted file mode 100644 index 54bb591344362..0000000000000 --- a/soaks/syslog_log2metric_humio_metrics/FEATURES +++ /dev/null @@ -1 +0,0 @@ -FEATURES="sources-internal_metrics,sinks-prometheus,sources-syslog,transforms-remap,transforms-log_to_metric,sinks-humio" diff --git a/soaks/syslog_loki/FEATURES b/soaks/syslog_loki/FEATURES deleted file mode 100644 index 01e854109290a..0000000000000 --- a/soaks/syslog_loki/FEATURES +++ /dev/null @@ -1 +0,0 @@ -FEATURES="sources-internal_metrics,sources-syslog,sinks-prometheus,sinks-loki" diff --git a/soaks/syslog_loki/terraform/variables.tf b/soaks/syslog_loki/terraform/variables.tf index 45ca583a2ffe1..519512362c4fb 100644 --- a/soaks/syslog_loki/terraform/variables.tf +++ b/soaks/syslog_loki/terraform/variables.tf @@ -7,8 +7,3 @@ variable "vector_image" { description = "The image of vector to use in this investigation" type = string } - -variable "sha" { - description = "The commit SHA from the Vector project under investigation" - type = string -} diff --git a/soaks/syslog_regex_logs2metric_ddmetrics/FEATURES b/soaks/syslog_regex_logs2metric_ddmetrics/FEATURES deleted file mode 100644 index bdaeb27f317be..0000000000000 --- a/soaks/syslog_regex_logs2metric_ddmetrics/FEATURES +++ /dev/null @@ -1 +0,0 @@ -FEATURES="sources-internal_metrics,sources-syslog,transforms-remap,transforms-log_to_metric,sinks-prometheus,sinks-datadog" diff --git a/soaks/syslog_regex_logs2metric_ddmetrics/terraform/variables.tf b/soaks/syslog_regex_logs2metric_ddmetrics/terraform/variables.tf index ff8ef169277b7..2fe029d08c139 100644 --- a/soaks/syslog_regex_logs2metric_ddmetrics/terraform/variables.tf +++ b/soaks/syslog_regex_logs2metric_ddmetrics/terraform/variables.tf @@ -7,8 +7,3 @@ variable "vector_image" { description = "The image of vector to use in this investigation" type = string } - -variable "sha" { - description = "The commit SHA from the Vector project under investigation" - type = string -} diff --git a/soaks/syslog_splunk_hec_logs/FEATURES b/soaks/syslog_splunk_hec_logs/FEATURES deleted file mode 100644 index 4a5caddf9be80..0000000000000 --- a/soaks/syslog_splunk_hec_logs/FEATURES +++ /dev/null @@ -1 +0,0 @@ -FEATURES="sources-internal_metrics,sinks-prometheus,sources-syslog,sinks-splunk_hec"