Add performance tests

.github/workflows/perfromance-testing.yaml

@@ -0,0 +1,70 @@
+---
+name: performance-testing
+on:
+  push:
+    branches:
+    - 'release/**'
+jobs:
+  ### INTERDOMAIN CLUSTER
+  interdomain-kind:
+    runs-on: ubuntu-latest
+    env:
+      KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }}
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.9.0
+      with:
+        access_token: ${{ github.token }}
+    - uses: actions/setup-go@v1
+      with:
+        go-version: 1.16
+        github-token: ${{ github.token }}
+    - name: Set go env
+      run: |
+        echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV
+        echo GO111MODULE=on >> $GITHUB_ENV
+        echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH
+    - uses: actions/checkout@v2
+      with:
+        path: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+    - name: Get kind
+      run: go get sigs.k8s.io/kind@v0.11.1
+    - name: Create kind clusters
+      run: |
+        if [[ $KUBERNETES_VERSION=="" ]]; then
+            KUBERNETES_VERSION="v1.22.1"
+        fi
+        for (( i = 1; i <= 2; i++ )); do
+            kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION"
+            configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i}
+            kind get kubeconfig --name "kind-${i}" > ${configPath}
+            echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV
+            echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV
+        done
+      working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+    - name: Performance tests
+      run: |
+        performance_testing/scripts/full_ci_run.sh "$NSM_VERSION" "$ARTIFACTS_DIR" "$QPS_LIST" "$DURATION" "$CONNECTIONS" "$ITERATIONS"
+      env:
+        NSM_VERSION: ${{ github.ref_name }}
+        ARTIFACTS_DIR: perf-test-results
+        QPS_LIST: 100 1000 1000000
+        DURATION: 60s
+        CONNECTIONS: 1
+        ITERATIONS: 3
+      working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+    - name: Print results
+      run: |
+        performance_testing/scripts/print_all_summaries.sh "$ARTIFACTS_DIR"
+      env:
+        ARTIFACTS_DIR: perf-test-results
+      working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+    - name: Cleanup resources
+      if: ${{ success() || failure() || cancelled() }}
+      run: kind delete clusters $(kind get clusters)
+    - name: Upload artifacts
+      if: ${{ success() || failure() || cancelled() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: Performance tests results and logs
+        path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/perf-test-results

performance_testing/README.md

@@ -0,0 +1,65 @@
+
+# Performance testing
+
+This folder contains deployment yaml files and scripts
+that deploy, run and clear applications for performance testing.
+
+# Parameters
+
+Parameters to be considered are:
+
+1. `qps_list`: requested load of the system
+2. `duration`: duration of a single test
+3. `connections`: the amount of simultaneous connections from test client to test server
+4. `iterations`: how many times to run each test
+
+To inspect results you can install Fortio and run `fortio server`.
+In the web ui you will be able to see graphs for different runs and compare them.
+
+Alternatively you can simply open .json files and inspect them for QPS and different latency percentiles.
+
+# Running the tests manually locally
+
+Make sure that you have load ballancer in you cluster.
+For Kind and bare metal clusters you can use metallb installation script:
+```bash
+./performance_testing/scripts/setup_metallb.sh
+```
+
+Prepare DNS and Spire:
+```bash
+./performance_testing/scripts/nsm_setup_dns.sh &&
+./performance_testing/scripts/nsm_setup_spire.sh
+```
+
+Test interdomain vl3:
+```bash
+./performance_testing/scripts/run_test_suite.sh \
+    vl3 \
+    ./performance_testing/results/raw/ \
+    3 \
+    "http://nginx.my-vl3-network:80" \
+    "./performance_testing/use-cases/vl3/deploy.sh" \
+    "./performance_testing/use-cases/vl3/clear.sh" \
+    "v1.8.0" \
+    "./performance_testing/nsm"
+```
+
+Test interdomain wireguard:
+```bash
+./performance_testing/scripts/run_test_suite.sh \
+    k2wireguard2k \
+    ./performance_testing/results/raw/ \
+    3 \
+    "http://172.16.1.2:80" \
+    "./performance_testing/use-cases/k2wireguard2k/deploy.sh" \
+    "./performance_testing/use-cases/k2wireguard2k/clear.sh" \
+    "v1.8.0" \
+    "./performance_testing/nsm"
+```
+
+Clear cluster if needed:
+```bash
+./performance_testing/scripts/nsm_clear_spire.sh
+./performance_testing/scripts/nsm_clear_dns.sh
+```

performance_testing/known-results.md

@@ -0,0 +1,57 @@
+
+# Known results
+
+This file contains info about results we already have.
+
+There are several different QPS targets. Each target has its own result expectations.
+
+# NSM v1.8.0, vl3
+
+vl3 tests in v1.8.0 seems to be CPU throttled by github, which affects max latency.
+
+1. Target QPS == 100
+    Actual QPS: 100
+    Min latency: 0.3-0.35 ms
+    Max latency: 100-250 ms
+    Avg latency: 4-5 ms
+    p50 latency: 1.5-2.5 ms
+    p99 atency: 90-150 ms
+2. Target QPS == 1000
+    Actual QPS: 350-450
+    Min latency: 0.25-0.3 ms
+    Max latency: 100-300 ms
+    Avg latency: 2-2.5 ms
+    p50 latency: 0.7-1.3 ms
+    p99 atency: 40-80 ms
+3. Target QPS == 1000000
+    Actual QPS: 350-450
+    Min latency: 0.25-0.3 ms
+    Max latency: 100-300
+    Avg latency: 2-2.5 ms
+    p50 latency: 0.7-1.3 ms
+    p99 atency: 40-80 ms
+
+# NSM v1.8.0, wireguard
+
+1. Target QPS == 100
+    Actual QPS: 100
+    Min latency: 0.3 ms
+    Max latency: 20-50 ms
+    Avg latency: 1-2 ms
+    p50 latency: 0.6 ms
+    p99 atency: 15-35 ms
+2. Target QPS == 1000
+    Actual QPS: 1000
+    Min latency: 0.2 ms
+    Max latency: 30-50 ms
+    Avg latency: 0.8 ms
+    p50 latency: 0.4-0.5 ms
+    p99 atency: 12-15 ms
+3. Target QPS == 1000000
+    Actual QPS: 1200-1400
+    Min latency: 0.2 ms
+    Max latency: 40-50 ms
+    Avg latency: 0.7 ms
+    p50 latency: 0.4 ms
+    p99 atency: 12-15 ms
+

performance_testing/nsm/c1/.gitignore

@@ -0,0 +1 @@
+/kustomization.yaml

performance_testing/nsm/c1/forwarder-patch.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: forwarder-vpp
+  labels:
+    app: forwarder-vpp
+spec:
+  template:
+    spec:
+      containers:
+        - name: forwarder-vpp
+          resources:
+            limits:
+              cpu: null

performance_testing/nsm/c2/.gitignore

@@ -0,0 +1 @@
+/kustomization.yaml

performance_testing/nsm/c2/forwarder-patch.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: forwarder-vpp
+  labels:
+    app: forwarder-vpp
+spec:
+  template:
+    spec:
+      containers:
+        - name: forwarder-vpp
+          resources:
+            limits:
+              cpu: null

performance_testing/nsm/nsm_clear_nsm.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+WH=$(kubectl "--kubeconfig=$KUBECONFIG1" get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+kubectl "--kubeconfig=$KUBECONFIG1" delete mutatingwebhookconfiguration "${WH}"
+kubectl "--kubeconfig=$KUBECONFIG1" delete ns nsm-system
+
+WH=$(kubectl "--kubeconfig=$KUBECONFIG2" get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+kubectl "--kubeconfig=$KUBECONFIG2" delete mutatingwebhookconfiguration "${WH}"
+kubectl "--kubeconfig=$KUBECONFIG2" delete ns nsm-system
+
+true

performance_testing/nsm/nsm_setup_nsm.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit
+
+if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi
+
+nsm_version=$1
+
+echo nsm_version is "$nsm_version"
+
+#########################
+
+cat <<EOF > "$parent_path/c1/kustomization.yaml"
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- https://github.com/networkservicemesh/deployments-k8s/examples/interdomain/nsm/cluster1?ref=$nsm_version
+
+patchesStrategicMerge:
+- forwarder-patch.yaml
+EOF
+
+cat <<EOF > "$parent_path/c2/kustomization.yaml"
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- https://github.com/networkservicemesh/deployments-k8s/examples/interdomain/nsm/cluster2?ref=$nsm_version
+
+patchesStrategicMerge:
+- forwarder-patch.yaml
+EOF
+
+kubectl "--kubeconfig=$KUBECONFIG1" apply -k "$parent_path/c1" || (sleep 10 && kubectl "--kubeconfig=$KUBECONFIG1" apply -k "$parent_path/c1") || exit
+kubectl "--kubeconfig=$KUBECONFIG2" apply -k "$parent_path/c2" || (sleep 10 && kubectl "--kubeconfig=$KUBECONFIG2" apply -k "$parent_path/c2") || exit
+
+sleep 5
+
+kubectl "--kubeconfig=$KUBECONFIG1" wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit
+kubectl "--kubeconfig=$KUBECONFIG2" wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit

performance_testing/scripts/fortio-config-template.json

@@ -0,0 +1,9 @@
+{
+    "url": "<url>",
+    "qps": "<qps>",
+    "r": "<resolution>",
+    "c": "<connections>",
+    "t": "<duration>",
+    "headers": [],
+    "save": "on"
+}

performance_testing/scripts/full_ci_run.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+echo running "$0"
+
+parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit
+
+if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi
+if [ -z "$2" ]; then echo 2nd arg 'result_folder' is missing; exit 1; fi
+
+nsm_version=$1
+result_folder=$2
+qps_list=${3:-1000000}
+duration=${4:-60s}
+connections=${5:-1}
+iterations=${6:-3}
+
+echo nsm_version: "$nsm_version"
+echo result_folder: "$result_folder"
+echo qps_list: "$qps_list"
+echo duration: "$duration"
+echo connections: "$connections"
+echo iterations: "$iterations"
+
+"$parent_path/setup_metallb.sh" || exit
+
+"$parent_path/nsm_setup_dns.sh" || exit
+"$parent_path/nsm_setup_spire.sh" || exit
+
+"$parent_path/run_test_suite.sh" \
+    vl3 \
+    "$result_folder" \
+    "$iterations" \
+    "http://nginx.my-vl3-network:80" \
+    "$parent_path/../use-cases/vl3/deploy.sh" \
+    "$parent_path/../use-cases/vl3/clear.sh" \
+    "$nsm_version" \
+    "$parent_path/../nsm" \
+    "$qps_list" \
+    "$duration" \
+    "$connections" \
+    || exit
+
+"$parent_path/run_test_suite.sh" \
+    k2wireguard2k \
+    "$result_folder" \
+    "$iterations" \
+    "http://172.16.1.2:80" \
+    "$parent_path/../use-cases/k2wireguard2k/deploy.sh" \
+    "$parent_path/../use-cases/k2wireguard2k/clear.sh" \
+    "$nsm_version" \
+    "$parent_path/../nsm" \
+    "$qps_list" \
+    "$duration" \
+    "$connections" \
+    || exit
+
+"$parent_path/nsm_clear_spire.sh"
+"$parent_path/nsm_clear_dns.sh"
+
+true

performance_testing/scripts/nsm_clear_dns.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+kubectl "--kubeconfig=$KUBECONFIG1" delete service -n kube-system exposed-kube-dns
+kubectl "--kubeconfig=$KUBECONFIG2" delete service -n kube-system exposed-kube-dns
+
+true

performance_testing/scripts/nsm_clear_spire.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+kubectl "--kubeconfig=$KUBECONFIG1" delete crd spiffeids.spiffeid.spiffe.io
+kubectl "--kubeconfig=$KUBECONFIG1" delete ns spire
+
+kubectl "--kubeconfig=$KUBECONFIG2" delete crd spiffeids.spiffeid.spiffe.io
+kubectl "--kubeconfig=$KUBECONFIG2" delete ns spire
+
+true