From 27fea69e82f370b6a8bcf50916032cd74ae3f977 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Wed, 5 Apr 2023 15:03:33 +0700 Subject: [PATCH 01/22] add draft for performance tests Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 60 +++++++ configmap.yaml | 31 ++++ custom-configmap.yaml | 14 ++ performance_testing/README.md | 40 +++++ performance_testing/nsm/c1/.gitignore | 1 + .../nsm/c1/forwarder-patch.yaml | 15 ++ performance_testing/nsm/c2/.gitignore | 1 + .../nsm/c2/forwarder-patch.yaml | 15 ++ performance_testing/nsm/nsm_clear_nsm.sh | 11 ++ performance_testing/nsm/nsm_setup_nsm.sh | 43 ++++++ .../scripts/fortio-config-template.json | 9 ++ performance_testing/scripts/nsm_clear_dns.sh | 6 + .../scripts/nsm_clear_spire.sh | 9 ++ performance_testing/scripts/nsm_setup_dns.sh | 146 ++++++++++++++++++ .../scripts/nsm_setup_spire.sh | 18 +++ .../scripts/run_test_single.sh | 116 ++++++++++++++ performance_testing/scripts/run_test_suite.sh | 59 +++++++ .../use-cases/k2wireguard2k/README.md | 105 +++++++++++++ .../use-cases/k2wireguard2k/clear.sh | 21 +++ .../k2wireguard2k/cluster1/.gitignore | 1 + .../k2wireguard2k/cluster1/patch-nse.yaml | 22 +++ .../k2wireguard2k/cluster2/fortio.yaml | 49 ++++++ .../use-cases/k2wireguard2k/deploy.sh | 52 +++++++ .../use-cases/vl3/apps/fortio.yaml | 38 +++++ .../use-cases/vl3/apps/nginx.yaml | 36 +++++ performance_testing/use-cases/vl3/clear.sh | 23 +++ performance_testing/use-cases/vl3/deploy.sh | 61 ++++++++ .../use-cases/vl3/vl3-dns/.gitignore | 1 + .../use-cases/vl3/vl3-dns/namespace.yaml | 5 + .../use-cases/vl3/vl3-dns/nse-patch.yaml | 23 +++ .../use-cases/vl3/vl3-dns/vl3-netsvc.yaml | 8 + 31 files changed, 1039 insertions(+) create mode 100644 .github/workflows/perfromance-testing.yaml create mode 100644 configmap.yaml create mode 100644 custom-configmap.yaml create mode 100644 performance_testing/README.md create mode 100644 performance_testing/nsm/c1/.gitignore create mode 100644 performance_testing/nsm/c1/forwarder-patch.yaml create mode 100644 performance_testing/nsm/c2/.gitignore create mode 100644 performance_testing/nsm/c2/forwarder-patch.yaml create mode 100644 performance_testing/nsm/nsm_clear_nsm.sh create mode 100644 performance_testing/nsm/nsm_setup_nsm.sh create mode 100644 performance_testing/scripts/fortio-config-template.json create mode 100644 performance_testing/scripts/nsm_clear_dns.sh create mode 100644 performance_testing/scripts/nsm_clear_spire.sh create mode 100644 performance_testing/scripts/nsm_setup_dns.sh create mode 100644 performance_testing/scripts/nsm_setup_spire.sh create mode 100644 performance_testing/scripts/run_test_single.sh create mode 100644 performance_testing/scripts/run_test_suite.sh create mode 100644 performance_testing/use-cases/k2wireguard2k/README.md create mode 100644 performance_testing/use-cases/k2wireguard2k/clear.sh create mode 100644 performance_testing/use-cases/k2wireguard2k/cluster1/.gitignore create mode 100644 performance_testing/use-cases/k2wireguard2k/cluster1/patch-nse.yaml create mode 100644 performance_testing/use-cases/k2wireguard2k/cluster2/fortio.yaml create mode 100644 performance_testing/use-cases/k2wireguard2k/deploy.sh create mode 100644 performance_testing/use-cases/vl3/apps/fortio.yaml create mode 100644 performance_testing/use-cases/vl3/apps/nginx.yaml create mode 100644 performance_testing/use-cases/vl3/clear.sh create mode 100644 performance_testing/use-cases/vl3/deploy.sh create mode 100644 performance_testing/use-cases/vl3/vl3-dns/.gitignore create mode 100644 performance_testing/use-cases/vl3/vl3-dns/namespace.yaml create mode 100644 performance_testing/use-cases/vl3/vl3-dns/nse-patch.yaml create mode 100644 performance_testing/use-cases/vl3/vl3-dns/vl3-netsvc.yaml diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml new file mode 100644 index 000000000..6239ac2ee --- /dev/null +++ b/.github/workflows/perfromance-testing.yaml @@ -0,0 +1,60 @@ +--- +name: performance-testing +on: + push: + branches: + - 'release/**' +jobs: + ### INTERDOMAIN CLUSTER + interdomain-kind: + runs-on: ubuntu-latest + env: + KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - uses: actions/setup-go@v1 + with: + go-version: 1.16 + github-token: ${{ github.token }} + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Get kind + run: go get sigs.k8s.io/kind@v0.11.1 + - name: Create kind clusters + run: | + if [[ $KUBERNETES_VERSION=="" ]]; then + KUBERNETES_VERSION="v1.22.1" + fi + for (( i = 1; i <= 2; i++ )); do + kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" + configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} + kind get kubeconfig --name "kind-${i}" > ${configPath} + echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV + echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV + done + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Performance tests + run: | + # TODO + exit 1 + env: + ARTIFACTS_DIR: perf-test-results + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Cleanup resources + if: ${{ success() || failure() || cancelled() }} + run: kind delete clusters $(kind get clusters) + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: Performance tests results and logs + path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/perf-test-results diff --git a/configmap.yaml b/configmap.yaml new file mode 100644 index 000000000..0eb7b9931 --- /dev/null +++ b/configmap.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + k8s_external my.cluster2 + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + loop + reload 5s + } + my.cluster1:53 { + forward . 172.18.201.0:53 { + force_tcp + } + } diff --git a/custom-configmap.yaml b/custom-configmap.yaml new file mode 100644 index 000000000..d8d770762 --- /dev/null +++ b/custom-configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + server.override: | + k8s_external my.cluster1 + proxy1.server: | + my.cluster1:53 { + forward . 172.18.201.0:53 { + force_tcp + } + } diff --git a/performance_testing/README.md b/performance_testing/README.md new file mode 100644 index 000000000..6aff3ceb5 --- /dev/null +++ b/performance_testing/README.md @@ -0,0 +1,40 @@ + +# Test use-cases + +Prepare DNS and Spire: +```bash +./performance_testing/scripts/nsm_setup_dns.sh && +./performance_testing/scripts/nsm_setup_spire.sh +``` + +Test with vl3 DNS: +```bash +./performance_testing/scripts/run_test_suite.sh \ + vl3 \ + ./performance_testing/results/raw/ \ + 3 \ + "http://nginx.my-vl3-network:80" \ + "./performance_testing/use-cases/vl3/deploy.sh" \ + "./performance_testing/use-cases/vl3/clear.sh" \ + "v1.8.0" \ + "./performance_testing/nsm" +``` + +Test interdomain wireguard: +```bash +./performance_testing/scripts/run_test_suite.sh \ + k2wireguard2k \ + ./performance_testing/results/raw/ \ + 3 \ + "http://172.16.1.2:80" \ + "./performance_testing/use-cases/k2wireguard2k/deploy.sh" \ + "./performance_testing/use-cases/k2wireguard2k/clear.sh" \ + "v1.8.0" \ + "./performance_testing/nsm" +``` + +Clear cluster if needed: +```bash +./performance_testing/scripts/nsm_clear_spire.sh +./performance_testing/scripts/nsm_clear_dns.sh +``` diff --git a/performance_testing/nsm/c1/.gitignore b/performance_testing/nsm/c1/.gitignore new file mode 100644 index 000000000..1cbe99618 --- /dev/null +++ b/performance_testing/nsm/c1/.gitignore @@ -0,0 +1 @@ +/kustomization.yaml diff --git a/performance_testing/nsm/c1/forwarder-patch.yaml b/performance_testing/nsm/c1/forwarder-patch.yaml new file mode 100644 index 000000000..c7566d76c --- /dev/null +++ b/performance_testing/nsm/c1/forwarder-patch.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: forwarder-vpp + labels: + app: forwarder-vpp +spec: + template: + spec: + containers: + - name: forwarder-vpp + resources: + limits: + cpu: null diff --git a/performance_testing/nsm/c2/.gitignore b/performance_testing/nsm/c2/.gitignore new file mode 100644 index 000000000..1cbe99618 --- /dev/null +++ b/performance_testing/nsm/c2/.gitignore @@ -0,0 +1 @@ +/kustomization.yaml diff --git a/performance_testing/nsm/c2/forwarder-patch.yaml b/performance_testing/nsm/c2/forwarder-patch.yaml new file mode 100644 index 000000000..c7566d76c --- /dev/null +++ b/performance_testing/nsm/c2/forwarder-patch.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: forwarder-vpp + labels: + app: forwarder-vpp +spec: + template: + spec: + containers: + - name: forwarder-vpp + resources: + limits: + cpu: null diff --git a/performance_testing/nsm/nsm_clear_nsm.sh b/performance_testing/nsm/nsm_clear_nsm.sh new file mode 100644 index 000000000..dedf26e23 --- /dev/null +++ b/performance_testing/nsm/nsm_clear_nsm.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +WH=$(kubectl --kubeconfig=$KUBECONFIG1 get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl --kubeconfig=$KUBECONFIG1 delete mutatingwebhookconfiguration ${WH} +kubectl --kubeconfig=$KUBECONFIG1 delete ns nsm-system + +WH=$(kubectl --kubeconfig=$KUBECONFIG2 get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl --kubeconfig=$KUBECONFIG2 delete mutatingwebhookconfiguration ${WH} +kubectl --kubeconfig=$KUBECONFIG2 delete ns nsm-system + +true diff --git a/performance_testing/nsm/nsm_setup_nsm.sh b/performance_testing/nsm/nsm_setup_nsm.sh new file mode 100644 index 000000000..7a35fab79 --- /dev/null +++ b/performance_testing/nsm/nsm_setup_nsm.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi + +nsm_version=$1 + +echo nsm_version is $nsm_version + +######################### + +cat <$parent_path/c1/kustomization.yaml +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: +- https://github.com/networkservicemesh/deployments-k8s/examples/interdomain/nsm/cluster1?ref=$nsm_version + +patchesStrategicMerge: +- forwarder-patch.yaml +EOF + +cat <$parent_path/c2/kustomization.yaml +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: +- https://github.com/networkservicemesh/deployments-k8s/examples/interdomain/nsm/cluster2?ref=$nsm_version + +patchesStrategicMerge: +- forwarder-patch.yaml +EOF + +kubectl --kubeconfig=$KUBECONFIG1 apply -k $parent_path/c1 || (sleep 10 && kubectl --kubeconfig=$KUBECONFIG1 apply -k $parent_path/c1) || exit +kubectl --kubeconfig=$KUBECONFIG2 apply -k $parent_path/c2 || (sleep 10 && kubectl --kubeconfig=$KUBECONFIG2 apply -k $parent_path/c2) || exit + +sleep 5 + +kubectl --kubeconfig=$KUBECONFIG1 wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit +kubectl --kubeconfig=$KUBECONFIG2 wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit diff --git a/performance_testing/scripts/fortio-config-template.json b/performance_testing/scripts/fortio-config-template.json new file mode 100644 index 000000000..352bce601 --- /dev/null +++ b/performance_testing/scripts/fortio-config-template.json @@ -0,0 +1,9 @@ +{ + "url": "", + "qps": "", + "r": "", + "c": "", + "t": "", + "headers": [], + "save": "on" +} diff --git a/performance_testing/scripts/nsm_clear_dns.sh b/performance_testing/scripts/nsm_clear_dns.sh new file mode 100644 index 000000000..5cd6c06fe --- /dev/null +++ b/performance_testing/scripts/nsm_clear_dns.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +kubectl --kubeconfig=$KUBECONFIG1 delete service -n kube-system exposed-kube-dns +kubectl --kubeconfig=$KUBECONFIG2 delete service -n kube-system exposed-kube-dns + +true diff --git a/performance_testing/scripts/nsm_clear_spire.sh b/performance_testing/scripts/nsm_clear_spire.sh new file mode 100644 index 000000000..d4660b437 --- /dev/null +++ b/performance_testing/scripts/nsm_clear_spire.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +kubectl --kubeconfig=$KUBECONFIG1 delete crd spiffeids.spiffeid.spiffe.io +kubectl --kubeconfig=$KUBECONFIG1 delete ns spire + +kubectl --kubeconfig=$KUBECONFIG2 delete crd spiffeids.spiffeid.spiffe.io +kubectl --kubeconfig=$KUBECONFIG2 delete ns spire + +true diff --git a/performance_testing/scripts/nsm_setup_dns.sh b/performance_testing/scripts/nsm_setup_dns.sh new file mode 100644 index 000000000..0c4dc8f32 --- /dev/null +++ b/performance_testing/scripts/nsm_setup_dns.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +kubectl --kubeconfig=$KUBECONFIG1 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer +kubectl --kubeconfig=$KUBECONFIG2 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer + +kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || sleep 10 +kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || exit +echo + +kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || sleep 10 +kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || exit +echo + +ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit +if [[ $ip1 == *"no value"* ]]; then + hostname1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit + echo hostname1 is $hostname1 + ip1=$(dig +short $hostname1 | head -1) || exit +fi +# if IPv6 +if [[ $ip1 =~ ":" ]]; then ip1=[$ip1]; fi + +echo Selected externalIP: $ip1 for cluster1 + +if [[ -z "$ip1" ]]; then echo ip1 is empty; exit 1; fi + +ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit +if [[ $ip2 == *"no value"* ]]; then + hostname2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit + echo hostname2 is $hostname2 + ip2=$(dig +short $hostname2 | head -1) || exit +fi +# if IPv6 +if [[ $ip2 =~ ":" ]]; then ip2=[$ip2]; fi + +echo Selected externalIP: $ip2 for cluster2 + +if [[ -z "$ip2" ]]; then echo ip2 is empty; exit 1; fi + +cat > configmap.yaml < custom-configmap.yaml < configmap.yaml < custom-configmap.yaml <^$url^g" \ + -e "s//$qps/g" \ + -e "s//$resolution/g" \ + -e "s//$connections/g" \ + -e "s//$duration/g" \ + $parent_path/fortio-config-template.json +} + +function captureState() { + result_folder=$1 + k1 get pod -A -o wide > $result_folder/pods-k1.log + k1 get svc -A -o wide > $result_folder/svc-k1.log + k2 get pod -A -o wide > $result_folder/pods-k2.log + k2 get svc -A -o wide > $result_folder/svc-k2.log +} + +function runTest() { + iterations=${1:-3} + url=$2 + qps=$3 + connections=$4 + duration=$5 + deploy_script=$6 + clear_script=$7 + nsm_version=$8 + + config=$(makeConfig $url $qps 0.00005 $connections $duration) || exit + config_name="q$qps-c$connections-d$duration" + + warmup_results=$result_folder/warmup + mkdir -p $warmup_results + + deploy_logs=$result_folder/deploy + mkdir -p $deploy_logs + + echo config name: $config_name + + echo measure for $iterations iterations + for i in $(seq -w 1 1 $iterations) + do + echo round $i + test_full_name=$test_name-$config_name-$i + echo deploying nsm... + $nsm_deploy_folder/nsm_setup_nsm.sh > $deploy_logs/$test_full_name-deploy-nsm.log $nsm_version 2>&1 || exit + echo deploying apps... + $deploy_script > $deploy_logs/$test_full_name-deploy-apps.log $nsm_version 2>&1 || exit + echo doing warmup run... + curl -s -d "$config" "localhost:8080/fortio/rest/run" > $warmup_results/$test_full_name-warmup.json + echo doing main run... + curl -s -d "$config" "localhost:8080/fortio/rest/run" > $result_folder/$test_full_name.json + result_code=$? + echo saving pod layout + k1 get pod -A -o wide > $deploy_logs/$test_full_name-k1-pods.log + k2 get pod -A -o wide > $deploy_logs/$test_full_name-k2-pods.log + echo clearing apps... + $clear_script > $deploy_logs/$test_full_name-clear-apps.log 2>&1 + echo clearing nsm... + $nsm_deploy_folder/nsm_clear_nsm.sh > $deploy_logs/$test_full_name-clear-nsm.log 2>&1 + $(exit $result_code) || exit + done +} + +runTest $test_iterations $test_url $test_qps $test_connections $test_duration $deploy_script $clear_script $nsm_version + diff --git a/performance_testing/scripts/run_test_suite.sh b/performance_testing/scripts/run_test_suite.sh new file mode 100644 index 000000000..75fef5bf7 --- /dev/null +++ b/performance_testing/scripts/run_test_suite.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } +function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +if [ -z "$1" ]; then echo 1st arg 'name' is missing; exit 1; fi +if [ -z "$2" ]; then echo 2nd arg 'result_folder' is missing; exit 1; fi +if [ -z "$3" ]; then echo 3rd arg 'test_iterations' is missing; exit 1; fi +if [ -z "$4" ]; then echo 4th arg 'test_url' is missing; exit 1; fi +if [ -z "$5" ]; then echo 5th arg 'deploy_script' is missing; exit 1; fi +if [ -z "$6" ]; then echo 6th arg 'clear_script' is missing; exit 1; fi +if [ -z "$7" ]; then echo 7th arg 'nsm_version' is missing; exit 1; fi +if [ -z "$8" ]; then echo 8th arg 'nsm_deploy_folder' is missing; exit 1; fi + +test_name=test-$(TZ=UTC date +%F-T%H-%M-%S)-$1 +result_folder=$2/$test_name +test_iterations=$3 +test_url=$4 +deploy_script=$5 +clear_script=$6 +nsm_version=$7 +nsm_deploy_folder=$8 + +echo test_name: $test_name +echo result_folder: $result_folder +echo test_iterations: $test_iterations +echo test_url: $test_url +echo deploy_script: $deploy_script +echo clear_script: $clear_script + +mkdir -p $result_folder || exit + +qps1=100 +qps2=1000 +qps3=1000000 +connections1=1 +duration1=60s + +echo running tests for $url +# for current_qps in $qps3 +for current_qps in $qps1 $qps2 $qps3 +do + echo testing qps $current_qps + $parent_path/run_test_single.sh \ + $test_name \ + "$result_folder" \ + "$test_iterations" \ + "$test_url" \ + "$current_qps" \ + "$connections1" \ + "$duration1" \ + "$deploy_script" \ + "$clear_script" \ + "$nsm_version" \ + "$nsm_deploy_folder" \ + || exit +done diff --git a/performance_testing/use-cases/k2wireguard2k/README.md b/performance_testing/use-cases/k2wireguard2k/README.md new file mode 100644 index 000000000..1f5363022 --- /dev/null +++ b/performance_testing/use-cases/k2wireguard2k/README.md @@ -0,0 +1,105 @@ +# Test kernel to wireguard to kernel connection + +This example shows that NSC and NSE on the different clusters could find and work with each other. + +NSC and NSE are using the `kernel` mechanism to connect to its local forwarder. +Forwarders are using the `wireguard` mechanism to connect with each other. + +## Requires + +Make sure that you have completed steps from [interdomain](../../) + +## Run + +**1. Deploy endpoint on cluster2** + +```bash +export KUBECONFIG=$KUBECONFIG2 +``` + +```bash +kubectl create ns ns-interdomain-kernel2wireguard2kernel +``` + +Deploy NSE: +```bash +kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/multicluster/usecases/interdomain_Kernel2Wireguard2Kernel/cluster2?ref=v1.7.0-rc.2 +``` + +Wait for applications ready: +```bash +kubectl wait --for=condition=ready --timeout=1m pod -l app=nse-kernel -n ns-interdomain-kernel2wireguard2kernel +``` + +Find NSE pod by labels: +```bash +NSE=$(kubectl get pods -l app=nse-kernel -n ns-interdomain-kernel2wireguard2kernel --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +[[ ! -z $NSE ]] +``` + +**2. Deploy client on cluster1** + +```bash +export KUBECONFIG=$KUBECONFIG1 +``` + +```bash +kubectl create ns ns-interdomain-kernel2wireguard2kernel +``` + +Deploy NSC: +```bash +kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/multicluster/usecases/interdomain_Kernel2Wireguard2Kernel/cluster1?ref=v1.7.0-rc.2 +``` + +Wait for applications ready: +```bash +kubectl wait --for=condition=ready --timeout=5m pod -l app=alpine -n ns-interdomain-kernel2wireguard2kernel +``` + +Find NSC pod by labels: +```bash +NSC=$(kubectl get pods -l app=alpine -n ns-interdomain-kernel2wireguard2kernel --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +[[ ! -z $NSC ]] +``` + +**3. Check connectivity** + +Switch to *cluster1*: + +```bash +export KUBECONFIG=$KUBECONFIG1 +``` + +```bash +kubectl exec ${NSC} -n ns-interdomain-kernel2wireguard2kernel -- ping -c 4 172.16.1.2 +``` + +Switch to *cluster2*: + +```bash +export KUBECONFIG=$KUBECONFIG2 +``` + +Ping from NSE to NSC: +```bash +kubectl exec ${NSE} -n ns-interdomain-kernel2wireguard2kernel -- ping -c 4 172.16.1.3 +``` + +## Cleanup + +1. Cleanup resources for *cluster1*: +```bash +export KUBECONFIG=$KUBECONFIG1 +``` +```bash +kubectl delete ns ns-interdomain-kernel2wireguard2kernel +``` + +2. Cleanup resources for *cluster2*: +```bash +export KUBECONFIG=$KUBECONFIG2 +``` +```bash +kubectl delete ns ns-interdomain-kernel2wireguard2kernel +``` diff --git a/performance_testing/use-cases/k2wireguard2k/clear.sh b/performance_testing/use-cases/k2wireguard2k/clear.sh new file mode 100644 index 000000000..be0f3db3d --- /dev/null +++ b/performance_testing/use-cases/k2wireguard2k/clear.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } +function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } + +echo running $0 + +pkill -f "port-forward svc/fortio-service 8080:8080" + +# delete without waiting, to delete in parallel +k1 delete ns perf-test-wg --wait=false +k2 delete ns perf-test-wg --wait=false + +# wait for everything to be deleted +k1 delete ns perf-test-wg +k2 delete ns perf-test-wg + +# previous command may have failed if the setup have failed and not all resources have been deployed +true diff --git a/performance_testing/use-cases/k2wireguard2k/cluster1/.gitignore b/performance_testing/use-cases/k2wireguard2k/cluster1/.gitignore new file mode 100644 index 000000000..1cbe99618 --- /dev/null +++ b/performance_testing/use-cases/k2wireguard2k/cluster1/.gitignore @@ -0,0 +1 @@ +/kustomization.yaml diff --git a/performance_testing/use-cases/k2wireguard2k/cluster1/patch-nse.yaml b/performance_testing/use-cases/k2wireguard2k/cluster1/patch-nse.yaml new file mode 100644 index 000000000..171283432 --- /dev/null +++ b/performance_testing/use-cases/k2wireguard2k/cluster1/patch-nse.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nse-kernel +spec: + template: + spec: + containers: + - name: nse + env: + - name: NSM_CIDR_PREFIX + value: 172.16.1.2/31 + - name: NSM_PAYLOAD + value: IP + - name: NSM_SERVICE_NAMES + value: my-networkservice-ip + - name: nginx + image: nginx + ports: + - containerPort: 80 + imagePullPolicy: IfNotPresent diff --git a/performance_testing/use-cases/k2wireguard2k/cluster2/fortio.yaml b/performance_testing/use-cases/k2wireguard2k/cluster2/fortio.yaml new file mode 100644 index 000000000..a0ad45ebe --- /dev/null +++ b/performance_testing/use-cases/k2wireguard2k/cluster2/fortio.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: fortio-service + labels: + app: fortio +spec: + ports: + - port: 8080 + name: http + selector: + app: fortio +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fortio-deployment +spec: + selector: + matchLabels: + app: fortio + replicas: 1 + template: + metadata: + annotations: + networkservicemesh.io: kernel://my-networkservice-ip@my.cluster1/nsm-1 + labels: + app: fortio + spec: + containers: + - name: fortio + image: fortio/fortio:1.40.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http-fortio + - containerPort: 8079 + name: grpc-ping + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - nse-kernel diff --git a/performance_testing/use-cases/k2wireguard2k/deploy.sh b/performance_testing/use-cases/k2wireguard2k/deploy.sh new file mode 100644 index 000000000..bc1deb6c9 --- /dev/null +++ b/performance_testing/use-cases/k2wireguard2k/deploy.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } +function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } + +echo running $0 + +if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi + +nsm_version=$1 + +echo nsm_version is $nsm_version + +######################### + +# Specify NSM version for NSE +cat < $parent_path/cluster1/kustomization.yaml +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: perf-test-wg + +bases: +- https://github.com/networkservicemesh/deployments-k8s/apps/nse-kernel?ref=$nsm_version + +patchesStrategicMerge: +- patch-nse.yaml +EOF + +# Deploy nginx +k1 create ns perf-test-wg +k1 apply -k $parent_path/cluster1 || exit + +# we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately +sleep 1 + +# Deploy fortio +k2 create ns perf-test-wg +k2 apply -n perf-test-wg -f $parent_path/cluster2/fortio.yaml || exit + +# we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately +sleep 5 +k1 -n perf-test-wg wait --for=condition=ready --timeout=1m pod -l app=nse-kernel || exit +k2 -n perf-test-wg wait --for=condition=ready --timeout=5m pod -l app=fortio || exit + +# open access to the test-load service on local machine +k2 -n perf-test-wg port-forward svc/fortio-service 8080:8080 & +# it can take some time for the background job to start listening to local port +sleep 5 diff --git a/performance_testing/use-cases/vl3/apps/fortio.yaml b/performance_testing/use-cases/vl3/apps/fortio.yaml new file mode 100644 index 000000000..cdeb85b3d --- /dev/null +++ b/performance_testing/use-cases/vl3/apps/fortio.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: fortio-service + labels: + app: fortio +spec: + ports: + - port: 8080 + name: http + selector: + app: fortio +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fortio-deployment +spec: + selector: + matchLabels: + app: fortio + replicas: 1 + template: + metadata: + annotations: + networkservicemesh.io: kernel://my-vl3-network@my.cluster1/nsm-1?dnsName=fortio + labels: + app: fortio + spec: + containers: + - name: fortio + image: fortio/fortio:1.40.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http-fortio + - containerPort: 8079 + name: grpc-ping diff --git a/performance_testing/use-cases/vl3/apps/nginx.yaml b/performance_testing/use-cases/vl3/apps/nginx.yaml new file mode 100644 index 000000000..fa472ea76 --- /dev/null +++ b/performance_testing/use-cases/vl3/apps/nginx.yaml @@ -0,0 +1,36 @@ + +apiVersion: v1 +kind: Service +metadata: + name: nginx-service + labels: + app: nginx +spec: + ports: + - port: 80 + protocol: TCP + selector: + app: nginx +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + selector: + matchLabels: + app: nginx + replicas: 1 + template: + metadata: + annotations: + networkservicemesh.io: kernel://my-vl3-network/nsm-1?dnsName=nginx + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.23.3 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 diff --git a/performance_testing/use-cases/vl3/clear.sh b/performance_testing/use-cases/vl3/clear.sh new file mode 100644 index 000000000..468f443e5 --- /dev/null +++ b/performance_testing/use-cases/vl3/clear.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } +function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } + +echo running $0 + +pkill -f "port-forward svc/fortio-service 8080:8080" + +# delete without waiting, to delete in parallel +k1 delete -k $parent_path/vl3-dns --wait=false +k1 delete ns perf-test-vl3 --wait=false +k2 delete ns perf-test-vl3 --wait=false + +# wait for everything to be deleted +k1 delete -k $parent_path/vl3-dns +k1 delete ns perf-test-vl3 +k2 delete ns perf-test-vl3 + +# previous command may have failed if the setup have failed and not all resources have been deployed +true diff --git a/performance_testing/use-cases/vl3/deploy.sh b/performance_testing/use-cases/vl3/deploy.sh new file mode 100644 index 000000000..0f3e72a4f --- /dev/null +++ b/performance_testing/use-cases/vl3/deploy.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } +function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } + +echo running $0 + +if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi + +nsm_version=$1 + +echo nsm_version is $nsm_version + +######################### + +# Specify vl3 NSE version +cat < $parent_path/vl3-dns/kustomization.yaml +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ns-dns-vl3 + +bases: +- https://github.com/networkservicemesh/deployments-k8s/apps/nse-vl3-vpp?ref=$nsm_version +- https://github.com/networkservicemesh/deployments-k8s/apps/vl3-ipam?ref=$nsm_version + +resources: +- namespace.yaml +- vl3-netsvc.yaml + +patchesStrategicMerge: +- nse-patch.yaml +EOF + +# Start vl3 NSE +k1 apply -k $parent_path/vl3-dns || exit + +# we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately +sleep 1 +k1 -n ns-dns-vl3 wait --for=condition=ready --timeout=5m pod -l app=vl3-ipam || exit +k1 -n ns-dns-vl3 wait --for=condition=ready --timeout=5m pod -l app=nse-vl3-vpp || exit + +# Deploy test apps: +k1 create ns perf-test-vl3 +k1 apply -n perf-test-vl3 -f $parent_path/apps/nginx.yaml || exit + +k2 create ns perf-test-vl3 +k2 apply -n perf-test-vl3 -f $parent_path/apps/fortio.yaml || exit + +# we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately +sleep 5 +k1 -n perf-test-vl3 wait --for=condition=ready --timeout=5m pod -l app=nginx || exit +k2 -n perf-test-vl3 wait --for=condition=ready --timeout=5m pod -l app=fortio || exit + +# open access to the test-load service on local machine +k2 -n perf-test-vl3 port-forward svc/fortio-service 8080:8080 & +# it can take some time for the background job to start listening to local port +sleep 5 diff --git a/performance_testing/use-cases/vl3/vl3-dns/.gitignore b/performance_testing/use-cases/vl3/vl3-dns/.gitignore new file mode 100644 index 000000000..1cbe99618 --- /dev/null +++ b/performance_testing/use-cases/vl3/vl3-dns/.gitignore @@ -0,0 +1 @@ +/kustomization.yaml diff --git a/performance_testing/use-cases/vl3/vl3-dns/namespace.yaml b/performance_testing/use-cases/vl3/vl3-dns/namespace.yaml new file mode 100644 index 000000000..a2188fc1a --- /dev/null +++ b/performance_testing/use-cases/vl3/vl3-dns/namespace.yaml @@ -0,0 +1,5 @@ +--- +kind: Namespace +apiVersion: v1 +metadata: + name: ns-dns-vl3 diff --git a/performance_testing/use-cases/vl3/vl3-dns/nse-patch.yaml b/performance_testing/use-cases/vl3/vl3-dns/nse-patch.yaml new file mode 100644 index 000000000..2659075c6 --- /dev/null +++ b/performance_testing/use-cases/vl3/vl3-dns/nse-patch.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nse-vl3-vpp + labels: + app: nse-vl3-vpp +spec: + replicas: 1 + template: + spec: + containers: + - name: nse + env: + - name: NSM_SERVICE_NAMES + value: "my-vl3-network" + - name: NSM_REGISTER_SERVICE + value: "false" + - name: NSM_DNS_TEMPLATES + value: "{{ index .Labels \"dnsName\" }}.{{ .NetworkService }}." + resources: + limits: + cpu: null diff --git a/performance_testing/use-cases/vl3/vl3-dns/vl3-netsvc.yaml b/performance_testing/use-cases/vl3/vl3-dns/vl3-netsvc.yaml new file mode 100644 index 000000000..881890cc8 --- /dev/null +++ b/performance_testing/use-cases/vl3/vl3-dns/vl3-netsvc.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: networkservicemesh.io/v1 +kind: NetworkService +metadata: + name: my-vl3-network + namespace: nsm-system +spec: + payload: IP From 534ecd126faa47bd58bef099d50f91843c67f107 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Wed, 5 Apr 2023 15:41:11 +0700 Subject: [PATCH 02/22] finish perf test ci Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 4 +- performance_testing/scripts/full_ci_run.sh | 46 ++++++++++++++++++++ performance_testing/scripts/setup_metallb.sh | 46 ++++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100755 performance_testing/scripts/full_ci_run.sh create mode 100644 performance_testing/scripts/setup_metallb.sh diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml index 6239ac2ee..3967370fe 100644 --- a/.github/workflows/perfromance-testing.yaml +++ b/.github/workflows/perfromance-testing.yaml @@ -44,9 +44,9 @@ jobs: working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Performance tests run: | - # TODO - exit 1 + performance_testing/scripts/full_ci_run.sh $NSM_VERSION $ARTIFACTS_DIR env: + NSM_VERSION: v1.8.0 ARTIFACTS_DIR: perf-test-results working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Cleanup resources diff --git a/performance_testing/scripts/full_ci_run.sh b/performance_testing/scripts/full_ci_run.sh new file mode 100755 index 000000000..13c06d116 --- /dev/null +++ b/performance_testing/scripts/full_ci_run.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo running $0 + +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + +if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi +if [ -z "$2" ]; then echo 2nd arg 'result_folder' is missing; exit 1; fi + +nsm_version=$1 +result_folder=$2 + +echo nsm_version: $nsm_version +echo result_folder: $result_folder + +$parent_path/setup_metallb.sh || exit + +$parent_path/nsm_setup_dns.sh || exit +$parent_path/nsm_setup_spire.sh || exit + +$parent_path/run_test_suite.sh \ + vl3 \ + "$result_folder" \ + 3 \ + "http://nginx.my-vl3-network:80" \ + "$parent_path/../use-cases/vl3/deploy.sh" \ + "$parent_path/../use-cases/vl3/clear.sh" \ + "$nsm_version" \ + "$parent_path/../nsm" \ + || exit + +$parent_path/run_test_suite.sh \ + k2wireguard2k \ + "$result_folder" \ + 3 \ + "http://172.16.1.2:80" \ + "$parent_path/../use-cases/k2wireguard2k/deploy.sh" \ + "$parent_path/../use-cases/k2wireguard2k/clear.sh" \ + "$nsm_version" \ + "$parent_path/../nsm" \ + || exit + +$parent_path/nsm_clear_spire.sh +$parent_path/nsm_clear_dns.sh + +true diff --git a/performance_testing/scripts/setup_metallb.sh b/performance_testing/scripts/setup_metallb.sh new file mode 100644 index 000000000..f21eddfa5 --- /dev/null +++ b/performance_testing/scripts/setup_metallb.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo CLUSTER1_CIDR is \'$CLUSTER1_CIDR\' +echo CLUSTER2_CIDR is \'$CLUSTER2_CIDR\' + +if [[ ! -z $CLUSTER1_CIDR ]]; then + kubectl "--kubeconfig=$KUBECONFIG1" apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml + kubectl "--kubeconfig=$KUBECONFIG1" apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/metallb.yaml + cat > metallb-config.yaml < metallb-config.yaml < Date: Wed, 5 Apr 2023 16:25:34 +0700 Subject: [PATCH 03/22] fix file permissions Signed-off-by: Danil Uzlov --- performance_testing/scripts/nsm_clear_dns.sh | 0 performance_testing/scripts/nsm_clear_spire.sh | 0 performance_testing/scripts/nsm_setup_dns.sh | 0 performance_testing/scripts/nsm_setup_spire.sh | 0 performance_testing/scripts/run_test_single.sh | 0 performance_testing/scripts/run_test_suite.sh | 0 performance_testing/scripts/setup_metallb.sh | 0 7 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 performance_testing/scripts/nsm_clear_dns.sh mode change 100644 => 100755 performance_testing/scripts/nsm_clear_spire.sh mode change 100644 => 100755 performance_testing/scripts/nsm_setup_dns.sh mode change 100644 => 100755 performance_testing/scripts/nsm_setup_spire.sh mode change 100644 => 100755 performance_testing/scripts/run_test_single.sh mode change 100644 => 100755 performance_testing/scripts/run_test_suite.sh mode change 100644 => 100755 performance_testing/scripts/setup_metallb.sh diff --git a/performance_testing/scripts/nsm_clear_dns.sh b/performance_testing/scripts/nsm_clear_dns.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/nsm_clear_spire.sh b/performance_testing/scripts/nsm_clear_spire.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/nsm_setup_dns.sh b/performance_testing/scripts/nsm_setup_dns.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/nsm_setup_spire.sh b/performance_testing/scripts/nsm_setup_spire.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/run_test_single.sh b/performance_testing/scripts/run_test_single.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/run_test_suite.sh b/performance_testing/scripts/run_test_suite.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/setup_metallb.sh b/performance_testing/scripts/setup_metallb.sh old mode 100644 new mode 100755 From 0ea3489d7e1f3d7730dacb4e005411a10e5fcd74 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Wed, 5 Apr 2023 16:38:27 +0700 Subject: [PATCH 04/22] fix file permissions Signed-off-by: Danil Uzlov --- performance_testing/nsm/nsm_clear_nsm.sh | 0 performance_testing/nsm/nsm_setup_nsm.sh | 0 performance_testing/use-cases/k2wireguard2k/clear.sh | 0 performance_testing/use-cases/k2wireguard2k/deploy.sh | 0 performance_testing/use-cases/vl3/clear.sh | 0 performance_testing/use-cases/vl3/deploy.sh | 0 6 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 performance_testing/nsm/nsm_clear_nsm.sh mode change 100644 => 100755 performance_testing/nsm/nsm_setup_nsm.sh mode change 100644 => 100755 performance_testing/use-cases/k2wireguard2k/clear.sh mode change 100644 => 100755 performance_testing/use-cases/k2wireguard2k/deploy.sh mode change 100644 => 100755 performance_testing/use-cases/vl3/clear.sh mode change 100644 => 100755 performance_testing/use-cases/vl3/deploy.sh diff --git a/performance_testing/nsm/nsm_clear_nsm.sh b/performance_testing/nsm/nsm_clear_nsm.sh old mode 100644 new mode 100755 diff --git a/performance_testing/nsm/nsm_setup_nsm.sh b/performance_testing/nsm/nsm_setup_nsm.sh old mode 100644 new mode 100755 diff --git a/performance_testing/use-cases/k2wireguard2k/clear.sh b/performance_testing/use-cases/k2wireguard2k/clear.sh old mode 100644 new mode 100755 diff --git a/performance_testing/use-cases/k2wireguard2k/deploy.sh b/performance_testing/use-cases/k2wireguard2k/deploy.sh old mode 100644 new mode 100755 diff --git a/performance_testing/use-cases/vl3/clear.sh b/performance_testing/use-cases/vl3/clear.sh old mode 100644 new mode 100755 diff --git a/performance_testing/use-cases/vl3/deploy.sh b/performance_testing/use-cases/vl3/deploy.sh old mode 100644 new mode 100755 From a3f2a2397233fa84e8b2346ee6881e37b4ddf42a Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Wed, 5 Apr 2023 16:38:57 +0700 Subject: [PATCH 05/22] disable standard ci Signed-off-by: Danil Uzlov --- .github/workflows/ci.yaml | 466 +++++++++++++++++++------------------- 1 file changed, 233 insertions(+), 233 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 566f7c3b9..4a571dabd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,239 +22,239 @@ jobs: uses: networkservicemesh/.github/.github/workflows/checkgomod.yaml@main ### SINGLE CLUSTER - kind: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: - - v1.25.2 - - v1.24.6 - - v1.23.12 - - v1.22.13 - - v1.21.14 - - v1.20.15 - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - uses: actions/setup-go@v1 - with: - go-version: 1.16 - github-token: ${{ github.token }} - - name: Set go env - run: | - echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - echo GO111MODULE=on >> $GITHUB_ENV - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - - uses: actions/checkout@v2 - with: - path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - uses: engineerd/setup-kind@v0.5.0 - with: - config: src/github.com/${{ github.repository }}/cluster-config.yaml - version: v0.13.0 - image: kindest/node:${{ matrix.image }} - - name: Check kind cluster - run: | - kubectl version - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Set loadbalancer CIDR - run: | - echo CLUSTER_CIDR="172.18.1.128/25" >> $GITHUB_ENV - - name: Integration tests - run: | - go test -count 1 -timeout 2h30m -race -v ./tests_single - env: - ARTIFACTS_DIR: ${{ matrix.image }}-logs/${{ matrix.image }} - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Upload artifacts - if: ${{ success() || failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: Single logs - path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/${{ matrix.image }}-logs + # kind: + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: + # image: + # - v1.25.2 + # - v1.24.6 + # - v1.23.12 + # - v1.22.13 + # - v1.21.14 + # - v1.20.15 + # steps: + # - name: Cancel Previous Runs + # uses: styfle/cancel-workflow-action@0.9.0 + # with: + # access_token: ${{ github.token }} + # - uses: actions/setup-go@v1 + # with: + # go-version: 1.16 + # github-token: ${{ github.token }} + # - name: Set go env + # run: | + # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + # echo GO111MODULE=on >> $GITHUB_ENV + # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + # - uses: actions/checkout@v2 + # with: + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - uses: engineerd/setup-kind@v0.5.0 + # with: + # config: src/github.com/${{ github.repository }}/cluster-config.yaml + # version: v0.13.0 + # image: kindest/node:${{ matrix.image }} + # - name: Check kind cluster + # run: | + # kubectl version + # kubectl cluster-info + # kubectl get pods -n kube-system + # echo "current-context:" $(kubectl config current-context) + # echo "environment-kubeconfig:" ${KUBECONFIG} + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Set loadbalancer CIDR + # run: | + # echo CLUSTER_CIDR="172.18.1.128/25" >> $GITHUB_ENV + # - name: Integration tests + # run: | + # go test -count 1 -timeout 2h30m -race -v ./tests_single + # env: + # ARTIFACTS_DIR: ${{ matrix.image }}-logs/${{ matrix.image }} + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Upload artifacts + # if: ${{ success() || failure() || cancelled() }} + # uses: actions/upload-artifact@v2 + # with: + # name: Single logs + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/${{ matrix.image }}-logs - ### SINGLE IPv6 CLUSTER - kind-ipv6: - runs-on: ubuntu-latest - env: - KUBERNETES_VERSION: "v1.25.0" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - uses: actions/setup-go@v1 - with: - go-version: 1.16 - github-token: ${{ github.token }} - - name: Set go env - run: | - echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - echo GO111MODULE=on >> $GITHUB_ENV - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - - uses: actions/checkout@v2 - with: - path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - uses: engineerd/setup-kind@v0.5.0 - with: - config: src/github.com/${{ github.repository }}/cluster-config-ipv6.yaml - version: v0.13.0 - image: kindest/node:${{ env.KUBERNETES_VERSION }} - - name: Check kind cluster - run: | - kubectl version - kubectl get pods -A -o wide - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Set loadbalancer CIDR - run: | - echo CLUSTER_CIDR="fc00:f853:ccd:e793:1::/80" >> $GITHUB_ENV - - name: Integration tests - run: | - go test -count 1 -timeout 2h10m -race -v \ - ./tests_single/basic_test.go \ - ./tests_single/memory_test.go - env: - ARTIFACTS_DIR: ipv6-logs/${{ env.KUBERNETES_VERSION }} - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Upload artifacts - if: ${{ success() || failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: Single IPv6 logs - path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/ipv6-logs + # ### SINGLE IPv6 CLUSTER + # kind-ipv6: + # runs-on: ubuntu-latest + # env: + # KUBERNETES_VERSION: "v1.25.0" + # steps: + # - name: Cancel Previous Runs + # uses: styfle/cancel-workflow-action@0.9.0 + # with: + # access_token: ${{ github.token }} + # - uses: actions/setup-go@v1 + # with: + # go-version: 1.16 + # github-token: ${{ github.token }} + # - name: Set go env + # run: | + # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + # echo GO111MODULE=on >> $GITHUB_ENV + # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + # - uses: actions/checkout@v2 + # with: + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - uses: engineerd/setup-kind@v0.5.0 + # with: + # config: src/github.com/${{ github.repository }}/cluster-config-ipv6.yaml + # version: v0.13.0 + # image: kindest/node:${{ env.KUBERNETES_VERSION }} + # - name: Check kind cluster + # run: | + # kubectl version + # kubectl get pods -A -o wide + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Set loadbalancer CIDR + # run: | + # echo CLUSTER_CIDR="fc00:f853:ccd:e793:1::/80" >> $GITHUB_ENV + # - name: Integration tests + # run: | + # go test -count 1 -timeout 2h10m -race -v \ + # ./tests_single/basic_test.go \ + # ./tests_single/memory_test.go + # env: + # ARTIFACTS_DIR: ipv6-logs/${{ env.KUBERNETES_VERSION }} + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Upload artifacts + # if: ${{ success() || failure() || cancelled() }} + # uses: actions/upload-artifact@v2 + # with: + # name: Single IPv6 logs + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/ipv6-logs - ### SINGLE CALICO CLUSTER - calico-kind: - runs-on: ubuntu-latest - env: - KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - uses: actions/setup-go@v1 - with: - go-version: 1.16 - github-token: ${{ github.token }} - - name: Set go env - run: | - echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - echo GO111MODULE=on >> $GITHUB_ENV - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - - name: Set kubernetes version - run: | - if [[ $KUBERNETES_VERSION=="" ]]; then - echo "KUBERNETES_VERSION=v1.22.1" >> $GITHUB_ENV - fi - - uses: actions/checkout@v2 - with: - path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - uses: engineerd/setup-kind@v0.5.0 - with: - config: src/github.com/${{ github.repository }}/cluster-config-calico.yaml - version: v0.11.1 - image: kindest/node:${{ env.KUBERNETES_VERSION }} - wait: 0s - - name: Setup external CNI plugin - shell: bash {0} - run: | - kubectl create -f https://projectcalico.docs.tigera.io/archive/v3.24/manifests/tigera-operator.yaml - for i in {1..5}; do - kubectl wait --for condition=established --timeout=1s crd/installations.operator.tigera.io - result=$? - if [ $result -eq 0 ]; then - break - fi - sleep 1s - done - kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/calico/installation-default.yaml - kubectl apply -k ${{ github.workspace }}/src/github.com/${{ github.repository }}/calico - kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=5m - - name: Check kind cluster - run: | - kubectl version - kubectl cluster-info - kubectl get pods --all-namespaces - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Integration tests - continue-on-error: true - run: | - go test -count 1 -timeout 1h30m -race -v \ - ./tests_single/basic_test.go \ - ./tests_single/heal_test.go \ - ./tests_single/memory_test.go \ - ./tests_single/observability_test.go \ - ./tests_single/feature_test.go \ - -calico - env: - ARTIFACTS_DIR: calico-logs/${{ env.KUBERNETES_VERSION }} - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Upload artifacts - if: ${{ success() || failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: Calico logs - path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/calico-logs + # ### SINGLE CALICO CLUSTER + # calico-kind: + # runs-on: ubuntu-latest + # env: + # KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} + # steps: + # - name: Cancel Previous Runs + # uses: styfle/cancel-workflow-action@0.9.0 + # with: + # access_token: ${{ github.token }} + # - uses: actions/setup-go@v1 + # with: + # go-version: 1.16 + # github-token: ${{ github.token }} + # - name: Set go env + # run: | + # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + # echo GO111MODULE=on >> $GITHUB_ENV + # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + # - name: Set kubernetes version + # run: | + # if [[ $KUBERNETES_VERSION=="" ]]; then + # echo "KUBERNETES_VERSION=v1.22.1" >> $GITHUB_ENV + # fi + # - uses: actions/checkout@v2 + # with: + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - uses: engineerd/setup-kind@v0.5.0 + # with: + # config: src/github.com/${{ github.repository }}/cluster-config-calico.yaml + # version: v0.11.1 + # image: kindest/node:${{ env.KUBERNETES_VERSION }} + # wait: 0s + # - name: Setup external CNI plugin + # shell: bash {0} + # run: | + # kubectl create -f https://projectcalico.docs.tigera.io/archive/v3.24/manifests/tigera-operator.yaml + # for i in {1..5}; do + # kubectl wait --for condition=established --timeout=1s crd/installations.operator.tigera.io + # result=$? + # if [ $result -eq 0 ]; then + # break + # fi + # sleep 1s + # done + # kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/calico/installation-default.yaml + # kubectl apply -k ${{ github.workspace }}/src/github.com/${{ github.repository }}/calico + # kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=5m + # - name: Check kind cluster + # run: | + # kubectl version + # kubectl cluster-info + # kubectl get pods --all-namespaces + # echo "current-context:" $(kubectl config current-context) + # echo "environment-kubeconfig:" ${KUBECONFIG} + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Integration tests + # continue-on-error: true + # run: | + # go test -count 1 -timeout 1h30m -race -v \ + # ./tests_single/basic_test.go \ + # ./tests_single/heal_test.go \ + # ./tests_single/memory_test.go \ + # ./tests_single/observability_test.go \ + # ./tests_single/feature_test.go \ + # -calico + # env: + # ARTIFACTS_DIR: calico-logs/${{ env.KUBERNETES_VERSION }} + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Upload artifacts + # if: ${{ success() || failure() || cancelled() }} + # uses: actions/upload-artifact@v2 + # with: + # name: Calico logs + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/calico-logs - ### INTERDOMAIN CLUSTER - interdomain-kind: - runs-on: ubuntu-latest - env: - KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - uses: actions/setup-go@v1 - with: - go-version: 1.16 - github-token: ${{ github.token }} - - name: Set go env - run: | - echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - echo GO111MODULE=on >> $GITHUB_ENV - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - - uses: actions/checkout@v2 - with: - path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Get kind - run: go get sigs.k8s.io/kind@v0.11.1 - - name: Create kind clusters - run: | - if [[ $KUBERNETES_VERSION=="" ]]; then - KUBERNETES_VERSION="v1.22.1" - fi - for (( i = 1; i <= 3; i++ )); do - kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" - configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} - kind get kubeconfig --name "kind-${i}" > ${configPath} - echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV - echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV - done - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Interdomain tests - run: | - go test -count 1 -timeout 1h -race -v ./tests_interdomain - env: - ARTIFACTS_DIR: interdomain-logs - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Cleanup resources - if: ${{ success() || failure() || cancelled() }} - run: kind delete clusters $(kind get clusters) - - name: Upload artifacts - if: ${{ success() || failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: Interdomain logs - path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_interdomain/interdomain-logs + # ### INTERDOMAIN CLUSTER + # interdomain-kind: + # runs-on: ubuntu-latest + # env: + # KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} + # steps: + # - name: Cancel Previous Runs + # uses: styfle/cancel-workflow-action@0.9.0 + # with: + # access_token: ${{ github.token }} + # - uses: actions/setup-go@v1 + # with: + # go-version: 1.16 + # github-token: ${{ github.token }} + # - name: Set go env + # run: | + # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + # echo GO111MODULE=on >> $GITHUB_ENV + # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + # - uses: actions/checkout@v2 + # with: + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Get kind + # run: go get sigs.k8s.io/kind@v0.11.1 + # - name: Create kind clusters + # run: | + # if [[ $KUBERNETES_VERSION=="" ]]; then + # KUBERNETES_VERSION="v1.22.1" + # fi + # for (( i = 1; i <= 3; i++ )); do + # kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" + # configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} + # kind get kubeconfig --name "kind-${i}" > ${configPath} + # echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV + # echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV + # done + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Interdomain tests + # run: | + # go test -count 1 -timeout 1h -race -v ./tests_interdomain + # env: + # ARTIFACTS_DIR: interdomain-logs + # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + # - name: Cleanup resources + # if: ${{ success() || failure() || cancelled() }} + # run: kind delete clusters $(kind get clusters) + # - name: Upload artifacts + # if: ${{ success() || failure() || cancelled() }} + # uses: actions/upload-artifact@v2 + # with: + # name: Interdomain logs + # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_interdomain/interdomain-logs From dc81f6d33fe574987390909ccec54f8c1f429ce6 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 10:28:39 +0700 Subject: [PATCH 06/22] fix bash scripts Signed-off-by: Danil Uzlov --- performance_testing/nsm/nsm_clear_nsm.sh | 12 ++--- performance_testing/nsm/nsm_setup_nsm.sh | 14 ++--- performance_testing/scripts/full_ci_run.sh | 18 +++---- performance_testing/scripts/nsm_clear_dns.sh | 4 +- .../scripts/nsm_clear_spire.sh | 8 +-- performance_testing/scripts/nsm_setup_dns.sh | 28 +++++----- .../scripts/nsm_setup_spire.sh | 20 +++---- .../scripts/run_test_single.sh | 54 +++++++++---------- performance_testing/scripts/run_test_suite.sh | 22 ++++---- .../use-cases/k2wireguard2k/clear.sh | 4 +- .../use-cases/k2wireguard2k/deploy.sh | 12 ++--- performance_testing/use-cases/vl3/clear.sh | 8 +-- performance_testing/use-cases/vl3/deploy.sh | 14 ++--- 13 files changed, 110 insertions(+), 108 deletions(-) diff --git a/performance_testing/nsm/nsm_clear_nsm.sh b/performance_testing/nsm/nsm_clear_nsm.sh index dedf26e23..2f5991b90 100755 --- a/performance_testing/nsm/nsm_clear_nsm.sh +++ b/performance_testing/nsm/nsm_clear_nsm.sh @@ -1,11 +1,11 @@ #!/bin/bash -WH=$(kubectl --kubeconfig=$KUBECONFIG1 get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') -kubectl --kubeconfig=$KUBECONFIG1 delete mutatingwebhookconfiguration ${WH} -kubectl --kubeconfig=$KUBECONFIG1 delete ns nsm-system +WH=$(kubectl "--kubeconfig=$KUBECONFIG1" get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl "--kubeconfig=$KUBECONFIG1" delete mutatingwebhookconfiguration "${WH}" +kubectl "--kubeconfig=$KUBECONFIG1" delete ns nsm-system -WH=$(kubectl --kubeconfig=$KUBECONFIG2 get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') -kubectl --kubeconfig=$KUBECONFIG2 delete mutatingwebhookconfiguration ${WH} -kubectl --kubeconfig=$KUBECONFIG2 delete ns nsm-system +WH=$(kubectl "--kubeconfig=$KUBECONFIG2" get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl "--kubeconfig=$KUBECONFIG2" delete mutatingwebhookconfiguration "${WH}" +kubectl "--kubeconfig=$KUBECONFIG2" delete ns nsm-system true diff --git a/performance_testing/nsm/nsm_setup_nsm.sh b/performance_testing/nsm/nsm_setup_nsm.sh index 7a35fab79..39d4d772c 100755 --- a/performance_testing/nsm/nsm_setup_nsm.sh +++ b/performance_testing/nsm/nsm_setup_nsm.sh @@ -6,11 +6,11 @@ if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi nsm_version=$1 -echo nsm_version is $nsm_version +echo nsm_version is "$nsm_version" ######################### -cat <$parent_path/c1/kustomization.yaml +cat < "$parent_path/c1/kustomization.yaml" --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -22,7 +22,7 @@ patchesStrategicMerge: - forwarder-patch.yaml EOF -cat <$parent_path/c2/kustomization.yaml +cat < "$parent_path/c2/kustomization.yaml" --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -34,10 +34,10 @@ patchesStrategicMerge: - forwarder-patch.yaml EOF -kubectl --kubeconfig=$KUBECONFIG1 apply -k $parent_path/c1 || (sleep 10 && kubectl --kubeconfig=$KUBECONFIG1 apply -k $parent_path/c1) || exit -kubectl --kubeconfig=$KUBECONFIG2 apply -k $parent_path/c2 || (sleep 10 && kubectl --kubeconfig=$KUBECONFIG2 apply -k $parent_path/c2) || exit +kubectl "--kubeconfig=$KUBECONFIG1" apply -k "$parent_path/c1" || (sleep 10 && kubectl "--kubeconfig=$KUBECONFIG1" apply -k "$parent_path/c1") || exit +kubectl "--kubeconfig=$KUBECONFIG2" apply -k "$parent_path/c2" || (sleep 10 && kubectl "--kubeconfig=$KUBECONFIG2" apply -k "$parent_path/c2") || exit sleep 5 -kubectl --kubeconfig=$KUBECONFIG1 wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit -kubectl --kubeconfig=$KUBECONFIG2 wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit +kubectl "--kubeconfig=$KUBECONFIG1" wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit +kubectl "--kubeconfig=$KUBECONFIG2" wait --for=condition=ready --timeout=1m pod -n nsm-system --all || exit diff --git a/performance_testing/scripts/full_ci_run.sh b/performance_testing/scripts/full_ci_run.sh index 13c06d116..1fc1e0435 100755 --- a/performance_testing/scripts/full_ci_run.sh +++ b/performance_testing/scripts/full_ci_run.sh @@ -10,15 +10,15 @@ if [ -z "$2" ]; then echo 2nd arg 'result_folder' is missing; exit 1; fi nsm_version=$1 result_folder=$2 -echo nsm_version: $nsm_version -echo result_folder: $result_folder +echo nsm_version: "$nsm_version" +echo result_folder: "$result_folder" -$parent_path/setup_metallb.sh || exit +"$parent_path/setup_metallb.sh" || exit -$parent_path/nsm_setup_dns.sh || exit -$parent_path/nsm_setup_spire.sh || exit +"$parent_path/nsm_setup_dns.sh" || exit +"$parent_path/nsm_setup_spire.sh" || exit -$parent_path/run_test_suite.sh \ +"$parent_path/run_test_suite.sh" \ vl3 \ "$result_folder" \ 3 \ @@ -29,7 +29,7 @@ $parent_path/run_test_suite.sh \ "$parent_path/../nsm" \ || exit -$parent_path/run_test_suite.sh \ +"$parent_path/run_test_suite.sh" \ k2wireguard2k \ "$result_folder" \ 3 \ @@ -40,7 +40,7 @@ $parent_path/run_test_suite.sh \ "$parent_path/../nsm" \ || exit -$parent_path/nsm_clear_spire.sh -$parent_path/nsm_clear_dns.sh +"$parent_path/nsm_clear_spire.sh" +"$parent_path/nsm_clear_dns.sh" true diff --git a/performance_testing/scripts/nsm_clear_dns.sh b/performance_testing/scripts/nsm_clear_dns.sh index 5cd6c06fe..57842b18d 100755 --- a/performance_testing/scripts/nsm_clear_dns.sh +++ b/performance_testing/scripts/nsm_clear_dns.sh @@ -1,6 +1,6 @@ #!/bin/bash -kubectl --kubeconfig=$KUBECONFIG1 delete service -n kube-system exposed-kube-dns -kubectl --kubeconfig=$KUBECONFIG2 delete service -n kube-system exposed-kube-dns +kubectl "--kubeconfig=$KUBECONFIG1" delete service -n kube-system exposed-kube-dns +kubectl "--kubeconfig=$KUBECONFIG2" delete service -n kube-system exposed-kube-dns true diff --git a/performance_testing/scripts/nsm_clear_spire.sh b/performance_testing/scripts/nsm_clear_spire.sh index d4660b437..54d474fd6 100755 --- a/performance_testing/scripts/nsm_clear_spire.sh +++ b/performance_testing/scripts/nsm_clear_spire.sh @@ -1,9 +1,9 @@ #!/bin/bash -kubectl --kubeconfig=$KUBECONFIG1 delete crd spiffeids.spiffeid.spiffe.io -kubectl --kubeconfig=$KUBECONFIG1 delete ns spire +kubectl "--kubeconfig=$KUBECONFIG1" delete crd spiffeids.spiffeid.spiffe.io +kubectl "--kubeconfig=$KUBECONFIG1" delete ns spire -kubectl --kubeconfig=$KUBECONFIG2 delete crd spiffeids.spiffeid.spiffe.io -kubectl --kubeconfig=$KUBECONFIG2 delete ns spire +kubectl "--kubeconfig=$KUBECONFIG2" delete crd spiffeids.spiffeid.spiffe.io +kubectl "--kubeconfig=$KUBECONFIG2" delete ns spire true diff --git a/performance_testing/scripts/nsm_setup_dns.sh b/performance_testing/scripts/nsm_setup_dns.sh index 0c4dc8f32..4e0897c06 100755 --- a/performance_testing/scripts/nsm_setup_dns.sh +++ b/performance_testing/scripts/nsm_setup_dns.sh @@ -1,19 +1,19 @@ #!/bin/bash -kubectl --kubeconfig=$KUBECONFIG1 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer -kubectl --kubeconfig=$KUBECONFIG2 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer +kubectl "--kubeconfig=$KUBECONFIG1" expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer +kubectl "--kubeconfig=$KUBECONFIG2" expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer -kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || sleep 10 -kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || exit +kubectl "--kubeconfig=$KUBECONFIG1" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || sleep 10 +kubectl "--kubeconfig=$KUBECONFIG1" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || exit echo -kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || sleep 10 -kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || exit +kubectl "--kubeconfig=$KUBECONFIG2" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || sleep 10 +kubectl "--kubeconfig=$KUBECONFIG2" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' || exit echo -ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit +ip1=$(kubectl "--kubeconfig=$KUBECONFIG1" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit if [[ $ip1 == *"no value"* ]]; then - hostname1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit + hostname1=$(kubectl "--kubeconfig=$KUBECONFIG1" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit echo hostname1 is $hostname1 ip1=$(dig +short $hostname1 | head -1) || exit fi @@ -24,9 +24,9 @@ echo Selected externalIP: $ip1 for cluster1 if [[ -z "$ip1" ]]; then echo ip1 is empty; exit 1; fi -ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit +ip2=$(kubectl "--kubeconfig=$KUBECONFIG2" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit if [[ $ip2 == *"no value"* ]]; then - hostname2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit + hostname2=$(kubectl "--kubeconfig=$KUBECONFIG2" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit echo hostname2 is $hostname2 ip2=$(dig +short $hostname2 | head -1) || exit fi @@ -70,7 +70,7 @@ data: } } EOF -kubectl --kubeconfig=$KUBECONFIG1 apply -f configmap.yaml +kubectl "--kubeconfig=$KUBECONFIG1" apply -f configmap.yaml cat > custom-configmap.yaml < configmap.yaml < custom-configmap.yaml </$resolution/g" \ -e "s//$connections/g" \ -e "s//$duration/g" \ - $parent_path/fortio-config-template.json + "$parent_path/fortio-config-template.json" } function captureState() { result_folder=$1 - k1 get pod -A -o wide > $result_folder/pods-k1.log - k1 get svc -A -o wide > $result_folder/svc-k1.log - k2 get pod -A -o wide > $result_folder/pods-k2.log - k2 get svc -A -o wide > $result_folder/svc-k2.log + k1 get pod -A -o wide > "$result_folder/pods-k1.log" + k1 get svc -A -o wide > "$result_folder/svc-k1.log" + k2 get pod -A -o wide > "$result_folder/pods-k2.log" + k2 get svc -A -o wide > "$result_folder/svc-k2.log" } function runTest() { @@ -85,32 +85,32 @@ function runTest() { deploy_logs=$result_folder/deploy mkdir -p $deploy_logs - echo config name: $config_name + echo "config name: $config_name" - echo measure for $iterations iterations - for i in $(seq -w 1 1 $iterations) + echo "measure for $iterations iterations" + for i in $(seq -w 1 1 "$iterations") do - echo round $i + echo "round $i" test_full_name=$test_name-$config_name-$i echo deploying nsm... - $nsm_deploy_folder/nsm_setup_nsm.sh > $deploy_logs/$test_full_name-deploy-nsm.log $nsm_version 2>&1 || exit + "$nsm_deploy_folder/nsm_setup_nsm.sh" > "$deploy_logs/$test_full_name-deploy-nsm.log" "$nsm_version" 2>&1 || exit echo deploying apps... - $deploy_script > $deploy_logs/$test_full_name-deploy-apps.log $nsm_version 2>&1 || exit + "$deploy_script" > "$deploy_logs/$test_full_name-deploy-apps.log" "$nsm_version" 2>&1 || exit echo doing warmup run... - curl -s -d "$config" "localhost:8080/fortio/rest/run" > $warmup_results/$test_full_name-warmup.json + curl -s -d "$config" "localhost:8080/fortio/rest/run" > "$warmup_results/$test_full_name-warmup.json" echo doing main run... - curl -s -d "$config" "localhost:8080/fortio/rest/run" > $result_folder/$test_full_name.json + curl -s -d "$config" "localhost:8080/fortio/rest/run" > "$result_folder/$test_full_name.json" result_code=$? echo saving pod layout - k1 get pod -A -o wide > $deploy_logs/$test_full_name-k1-pods.log - k2 get pod -A -o wide > $deploy_logs/$test_full_name-k2-pods.log + k1 get pod -A -o wide > "$deploy_logs/$test_full_name-k1-pods.log" + k2 get pod -A -o wide > "$deploy_logs/$test_full_name-k2-pods.log" echo clearing apps... - $clear_script > $deploy_logs/$test_full_name-clear-apps.log 2>&1 + "$clear_script" > "$deploy_logs/$test_full_name-clear-apps.log" 2>&1 echo clearing nsm... - $nsm_deploy_folder/nsm_clear_nsm.sh > $deploy_logs/$test_full_name-clear-nsm.log 2>&1 + "$nsm_deploy_folder/nsm_clear_nsm.sh" > "$deploy_logs/$test_full_name-clear-nsm.log" 2>&1 $(exit $result_code) || exit done } -runTest $test_iterations $test_url $test_qps $test_connections $test_duration $deploy_script $clear_script $nsm_version +runTest "$test_iterations" "$test_url" "$test_qps" "$test_connections" "$test_duration" "$deploy_script" "$clear_script" "$nsm_version" diff --git a/performance_testing/scripts/run_test_suite.sh b/performance_testing/scripts/run_test_suite.sh index 75fef5bf7..e251cfa82 100755 --- a/performance_testing/scripts/run_test_suite.sh +++ b/performance_testing/scripts/run_test_suite.sh @@ -23,14 +23,16 @@ clear_script=$6 nsm_version=$7 nsm_deploy_folder=$8 -echo test_name: $test_name -echo result_folder: $result_folder -echo test_iterations: $test_iterations -echo test_url: $test_url -echo deploy_script: $deploy_script -echo clear_script: $clear_script +echo "test_name: $test_name" +echo "result_folder: $result_folder" +echo "test_iterations: $test_iterations" +echo "test_url: $test_url" +echo "deploy_script: $deploy_script" +echo "clear_script: $clear_script" +echo "nsm_version: $nsm_version" +echo "nsm_deploy_folder: $nsm_deploy_folder" -mkdir -p $result_folder || exit +mkdir -p "$result_folder" || exit qps1=100 qps2=1000 @@ -42,9 +44,9 @@ echo running tests for $url # for current_qps in $qps3 for current_qps in $qps1 $qps2 $qps3 do - echo testing qps $current_qps - $parent_path/run_test_single.sh \ - $test_name \ + echo "testing qps $current_qps" + "$parent_path/run_test_single.sh" \ + "$test_name" \ "$result_folder" \ "$test_iterations" \ "$test_url" \ diff --git a/performance_testing/use-cases/k2wireguard2k/clear.sh b/performance_testing/use-cases/k2wireguard2k/clear.sh index be0f3db3d..0cfcdc9c2 100755 --- a/performance_testing/use-cases/k2wireguard2k/clear.sh +++ b/performance_testing/use-cases/k2wireguard2k/clear.sh @@ -2,8 +2,8 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit -function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } -function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } +function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } +function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } echo running $0 diff --git a/performance_testing/use-cases/k2wireguard2k/deploy.sh b/performance_testing/use-cases/k2wireguard2k/deploy.sh index bc1deb6c9..bd0e1ea99 100755 --- a/performance_testing/use-cases/k2wireguard2k/deploy.sh +++ b/performance_testing/use-cases/k2wireguard2k/deploy.sh @@ -2,8 +2,8 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit -function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } -function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } +function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } +function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } echo running $0 @@ -11,12 +11,12 @@ if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi nsm_version=$1 -echo nsm_version is $nsm_version +echo nsm_version is "$nsm_version" ######################### # Specify NSM version for NSE -cat < $parent_path/cluster1/kustomization.yaml +cat < "$parent_path/cluster1/kustomization.yaml" --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -32,14 +32,14 @@ EOF # Deploy nginx k1 create ns perf-test-wg -k1 apply -k $parent_path/cluster1 || exit +k1 apply -k "$parent_path/cluster1" || exit # we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately sleep 1 # Deploy fortio k2 create ns perf-test-wg -k2 apply -n perf-test-wg -f $parent_path/cluster2/fortio.yaml || exit +k2 apply -n perf-test-wg -f "$parent_path/cluster2/fortio.yaml" || exit # we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately sleep 5 diff --git a/performance_testing/use-cases/vl3/clear.sh b/performance_testing/use-cases/vl3/clear.sh index 468f443e5..35d1c413b 100755 --- a/performance_testing/use-cases/vl3/clear.sh +++ b/performance_testing/use-cases/vl3/clear.sh @@ -2,20 +2,20 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit -function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } -function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } +function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } +function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } echo running $0 pkill -f "port-forward svc/fortio-service 8080:8080" # delete without waiting, to delete in parallel -k1 delete -k $parent_path/vl3-dns --wait=false +k1 delete -k "$parent_path/vl3-dns" --wait=false k1 delete ns perf-test-vl3 --wait=false k2 delete ns perf-test-vl3 --wait=false # wait for everything to be deleted -k1 delete -k $parent_path/vl3-dns +k1 delete -k "$parent_path/vl3-dns" k1 delete ns perf-test-vl3 k2 delete ns perf-test-vl3 diff --git a/performance_testing/use-cases/vl3/deploy.sh b/performance_testing/use-cases/vl3/deploy.sh index 0f3e72a4f..cba6ae9f2 100755 --- a/performance_testing/use-cases/vl3/deploy.sh +++ b/performance_testing/use-cases/vl3/deploy.sh @@ -2,8 +2,8 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit -function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } -function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } +function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } +function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } echo running $0 @@ -11,12 +11,12 @@ if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi nsm_version=$1 -echo nsm_version is $nsm_version +echo nsm_version is "$nsm_version" ######################### # Specify vl3 NSE version -cat < $parent_path/vl3-dns/kustomization.yaml +cat < "$parent_path/vl3-dns/kustomization.yaml" --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization @@ -36,7 +36,7 @@ patchesStrategicMerge: EOF # Start vl3 NSE -k1 apply -k $parent_path/vl3-dns || exit +k1 apply -k "$parent_path/vl3-dns" || exit # we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately sleep 1 @@ -45,10 +45,10 @@ k1 -n ns-dns-vl3 wait --for=condition=ready --timeout=5m pod -l app=nse-vl3-vpp # Deploy test apps: k1 create ns perf-test-vl3 -k1 apply -n perf-test-vl3 -f $parent_path/apps/nginx.yaml || exit +k1 apply -n perf-test-vl3 -f "$parent_path/apps/nginx.yaml" || exit k2 create ns perf-test-vl3 -k2 apply -n perf-test-vl3 -f $parent_path/apps/fortio.yaml || exit +k2 apply -n perf-test-vl3 -f "$parent_path/apps/fortio.yaml" || exit # we need to wait a bit to make sure that pods are created, so that wait commands don't fail immediately sleep 5 From ce704c5ac88ef76da32bbabf79d9c3cf57d6b667 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 10:41:20 +0700 Subject: [PATCH 07/22] fix yaml files Signed-off-by: Danil Uzlov --- configmap.yaml | 31 ------------------- custom-configmap.yaml | 14 --------- .../use-cases/vl3/apps/fortio.yaml | 1 + .../use-cases/vl3/apps/nginx.yaml | 2 +- 4 files changed, 2 insertions(+), 46 deletions(-) delete mode 100644 configmap.yaml delete mode 100644 custom-configmap.yaml diff --git a/configmap.yaml b/configmap.yaml deleted file mode 100644 index 0eb7b9931..000000000 --- a/configmap.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system -data: - Corefile: | - .:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - k8s_external my.cluster2 - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - loop - reload 5s - } - my.cluster1:53 { - forward . 172.18.201.0:53 { - force_tcp - } - } diff --git a/custom-configmap.yaml b/custom-configmap.yaml deleted file mode 100644 index d8d770762..000000000 --- a/custom-configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns-custom - namespace: kube-system -data: - server.override: | - k8s_external my.cluster1 - proxy1.server: | - my.cluster1:53 { - forward . 172.18.201.0:53 { - force_tcp - } - } diff --git a/performance_testing/use-cases/vl3/apps/fortio.yaml b/performance_testing/use-cases/vl3/apps/fortio.yaml index cdeb85b3d..cfa0a6483 100644 --- a/performance_testing/use-cases/vl3/apps/fortio.yaml +++ b/performance_testing/use-cases/vl3/apps/fortio.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: diff --git a/performance_testing/use-cases/vl3/apps/nginx.yaml b/performance_testing/use-cases/vl3/apps/nginx.yaml index fa472ea76..62889c300 100644 --- a/performance_testing/use-cases/vl3/apps/nginx.yaml +++ b/performance_testing/use-cases/vl3/apps/nginx.yaml @@ -1,4 +1,4 @@ - +--- apiVersion: v1 kind: Service metadata: From ba18faef77a957e27709d5e83e0b12c65fb3068b Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 10:53:14 +0700 Subject: [PATCH 08/22] fix bash scripts Signed-off-by: Danil Uzlov --- performance_testing/scripts/full_ci_run.sh | 2 +- performance_testing/scripts/nsm_setup_dns.sh | 16 ++++++++-------- performance_testing/scripts/nsm_setup_spire.sh | 4 ++-- performance_testing/scripts/run_test_single.sh | 16 ++++++++-------- performance_testing/scripts/run_test_suite.sh | 6 +++--- performance_testing/scripts/setup_metallb.sh | 4 ++-- .../use-cases/k2wireguard2k/clear.sh | 2 +- .../use-cases/k2wireguard2k/deploy.sh | 2 +- performance_testing/use-cases/vl3/clear.sh | 2 +- performance_testing/use-cases/vl3/deploy.sh | 2 +- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/performance_testing/scripts/full_ci_run.sh b/performance_testing/scripts/full_ci_run.sh index 1fc1e0435..4235c8874 100755 --- a/performance_testing/scripts/full_ci_run.sh +++ b/performance_testing/scripts/full_ci_run.sh @@ -1,6 +1,6 @@ #!/bin/bash -echo running $0 +echo running "$0" parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit diff --git a/performance_testing/scripts/nsm_setup_dns.sh b/performance_testing/scripts/nsm_setup_dns.sh index 4e0897c06..31350bdd2 100755 --- a/performance_testing/scripts/nsm_setup_dns.sh +++ b/performance_testing/scripts/nsm_setup_dns.sh @@ -14,26 +14,26 @@ echo ip1=$(kubectl "--kubeconfig=$KUBECONFIG1" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit if [[ $ip1 == *"no value"* ]]; then hostname1=$(kubectl "--kubeconfig=$KUBECONFIG1" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit - echo hostname1 is $hostname1 - ip1=$(dig +short $hostname1 | head -1) || exit + echo hostname1 is "$hostname1" + ip1=$(dig +short "$hostname1" | head -1) || exit fi # if IPv6 -if [[ $ip1 =~ ":" ]]; then ip1=[$ip1]; fi +if [[ $ip1 =~ ":" ]]; then ip1="[$ip1]"; fi -echo Selected externalIP: $ip1 for cluster1 +echo Selected externalIP: "$ip1" for cluster1 if [[ -z "$ip1" ]]; then echo ip1 is empty; exit 1; fi ip2=$(kubectl "--kubeconfig=$KUBECONFIG2" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') || exit if [[ $ip2 == *"no value"* ]]; then hostname2=$(kubectl "--kubeconfig=$KUBECONFIG2" get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') || exit - echo hostname2 is $hostname2 - ip2=$(dig +short $hostname2 | head -1) || exit + echo hostname2 is "$hostname2" + ip2=$(dig +short "$hostname2" | head -1) || exit fi # if IPv6 -if [[ $ip2 =~ ":" ]]; then ip2=[$ip2]; fi +if [[ $ip2 =~ ":" ]]; then ip2="[$ip2]"; fi -echo Selected externalIP: $ip2 for cluster2 +echo Selected externalIP: "$ip2" for cluster2 if [[ -z "$ip2" ]]; then echo ip2 is empty; exit 1; fi diff --git a/performance_testing/scripts/nsm_setup_spire.sh b/performance_testing/scripts/nsm_setup_spire.sh index 011d197f9..1451f3d6a 100755 --- a/performance_testing/scripts/nsm_setup_spire.sh +++ b/performance_testing/scripts/nsm_setup_spire.sh @@ -14,5 +14,5 @@ kubectl "--kubeconfig=$KUBECONFIG2" wait -n spire --timeout=1m --for=condition=r bundle1=$(kubectl "--kubeconfig=$KUBECONFIG1" exec spire-server-0 -n spire -- bin/spire-server bundle show -format spiffe) || exit bundle2=$(kubectl "--kubeconfig=$KUBECONFIG2" exec spire-server-0 -n spire -- bin/spire-server bundle show -format spiffe) || exit -echo $bundle2 | kubectl "--kubeconfig=$KUBECONFIG1" exec -i spire-server-0 -n spire -- bin/spire-server bundle set -format spiffe -id "spiffe://nsm.cluster2" || exit -echo $bundle1 | kubectl "--kubeconfig=$KUBECONFIG2" exec -i spire-server-0 -n spire -- bin/spire-server bundle set -format spiffe -id "spiffe://nsm.cluster1" || exit +echo "$bundle2" | kubectl "--kubeconfig=$KUBECONFIG1" exec -i spire-server-0 -n spire -- bin/spire-server bundle set -format spiffe -id "spiffe://nsm.cluster2" || exit +echo "$bundle1" | kubectl "--kubeconfig=$KUBECONFIG2" exec -i spire-server-0 -n spire -- bin/spire-server bundle set -format spiffe -id "spiffe://nsm.cluster1" || exit diff --git a/performance_testing/scripts/run_test_single.sh b/performance_testing/scripts/run_test_single.sh index 463c214e3..dbd73fc23 100755 --- a/performance_testing/scripts/run_test_single.sh +++ b/performance_testing/scripts/run_test_single.sh @@ -2,8 +2,8 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit -function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } -function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } +function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } +function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } if [ -z "$1" ]; then echo 1st arg 'test_name' is missing; exit 1; fi if [ -z "$2" ]; then echo 2nd arg 'result_folder' is missing; exit 1; fi @@ -38,8 +38,8 @@ echo "test_connections: $test_connections" echo "test_duration: $test_duration" echo "deploy_script: $deploy_script" echo "clear_script: $clear_script" -echo nsm_version: $nsm_version -echo nsm_deploy_folder: $nsm_deploy_folder +echo "nsm_version: $nsm_version" +echo "nsm_deploy_folder: $nsm_deploy_folder" echo ------ @@ -76,14 +76,14 @@ function runTest() { clear_script=$7 nsm_version=$8 - config=$(makeConfig $url $qps 0.00005 $connections $duration) || exit + config=$(makeConfig "$url" "$qps" 0.00005 "$connections" "$duration") || exit config_name="q$qps-c$connections-d$duration" warmup_results=$result_folder/warmup - mkdir -p $warmup_results + mkdir -p "$warmup_results" deploy_logs=$result_folder/deploy - mkdir -p $deploy_logs + mkdir -p "$deploy_logs" echo "config name: $config_name" @@ -108,7 +108,7 @@ function runTest() { "$clear_script" > "$deploy_logs/$test_full_name-clear-apps.log" 2>&1 echo clearing nsm... "$nsm_deploy_folder/nsm_clear_nsm.sh" > "$deploy_logs/$test_full_name-clear-nsm.log" 2>&1 - $(exit $result_code) || exit + $(exit "$result_code") || exit done } diff --git a/performance_testing/scripts/run_test_suite.sh b/performance_testing/scripts/run_test_suite.sh index e251cfa82..7f8e75bc5 100755 --- a/performance_testing/scripts/run_test_suite.sh +++ b/performance_testing/scripts/run_test_suite.sh @@ -1,7 +1,7 @@ #!/bin/bash -function k1() { kubectl --kubeconfig $KUBECONFIG1 "$@" ; } -function k2() { kubectl --kubeconfig $KUBECONFIG2 "$@" ; } +function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } +function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit @@ -40,7 +40,7 @@ qps3=1000000 connections1=1 duration1=60s -echo running tests for $url +echo running tests for "$url" # for current_qps in $qps3 for current_qps in $qps1 $qps2 $qps3 do diff --git a/performance_testing/scripts/setup_metallb.sh b/performance_testing/scripts/setup_metallb.sh index f21eddfa5..55b3b7f4e 100755 --- a/performance_testing/scripts/setup_metallb.sh +++ b/performance_testing/scripts/setup_metallb.sh @@ -1,7 +1,7 @@ #!/bin/bash -echo CLUSTER1_CIDR is \'$CLUSTER1_CIDR\' -echo CLUSTER2_CIDR is \'$CLUSTER2_CIDR\' +echo CLUSTER1_CIDR is "'$CLUSTER1_CIDR'" +echo CLUSTER2_CIDR is "'$CLUSTER2_CIDR'" if [[ ! -z $CLUSTER1_CIDR ]]; then kubectl "--kubeconfig=$KUBECONFIG1" apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml diff --git a/performance_testing/use-cases/k2wireguard2k/clear.sh b/performance_testing/use-cases/k2wireguard2k/clear.sh index 0cfcdc9c2..5384c2066 100755 --- a/performance_testing/use-cases/k2wireguard2k/clear.sh +++ b/performance_testing/use-cases/k2wireguard2k/clear.sh @@ -5,7 +5,7 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } -echo running $0 +echo running "$0" pkill -f "port-forward svc/fortio-service 8080:8080" diff --git a/performance_testing/use-cases/k2wireguard2k/deploy.sh b/performance_testing/use-cases/k2wireguard2k/deploy.sh index bd0e1ea99..525b0c913 100755 --- a/performance_testing/use-cases/k2wireguard2k/deploy.sh +++ b/performance_testing/use-cases/k2wireguard2k/deploy.sh @@ -5,7 +5,7 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } -echo running $0 +echo running "$0" if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi diff --git a/performance_testing/use-cases/vl3/clear.sh b/performance_testing/use-cases/vl3/clear.sh index 35d1c413b..2389e77ac 100755 --- a/performance_testing/use-cases/vl3/clear.sh +++ b/performance_testing/use-cases/vl3/clear.sh @@ -5,7 +5,7 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } -echo running $0 +echo running "$0" pkill -f "port-forward svc/fortio-service 8080:8080" diff --git a/performance_testing/use-cases/vl3/deploy.sh b/performance_testing/use-cases/vl3/deploy.sh index cba6ae9f2..333111ef5 100755 --- a/performance_testing/use-cases/vl3/deploy.sh +++ b/performance_testing/use-cases/vl3/deploy.sh @@ -5,7 +5,7 @@ parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit function k1() { kubectl --kubeconfig "$KUBECONFIG1" "$@" ; } function k2() { kubectl --kubeconfig "$KUBECONFIG2" "$@" ; } -echo running $0 +echo running "$0" if [ -z "$1" ]; then echo 1st arg 'nsm_version' is missing; exit 1; fi From c2c7d0e93e8ec14d2d11e91c43795a4fe45a777a Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 10:56:49 +0700 Subject: [PATCH 09/22] fix bash scripts Signed-off-by: Danil Uzlov --- performance_testing/scripts/run_test_single.sh | 3 +-- performance_testing/scripts/run_test_suite.sh | 2 +- performance_testing/scripts/setup_metallb.sh | 4 ++-- performance_testing/use-cases/k2wireguard2k/clear.sh | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/performance_testing/scripts/run_test_single.sh b/performance_testing/scripts/run_test_single.sh index dbd73fc23..7e857933f 100755 --- a/performance_testing/scripts/run_test_single.sh +++ b/performance_testing/scripts/run_test_single.sh @@ -108,9 +108,8 @@ function runTest() { "$clear_script" > "$deploy_logs/$test_full_name-clear-apps.log" 2>&1 echo clearing nsm... "$nsm_deploy_folder/nsm_clear_nsm.sh" > "$deploy_logs/$test_full_name-clear-nsm.log" 2>&1 - $(exit "$result_code") || exit + (exit "$result_code") || exit done } runTest "$test_iterations" "$test_url" "$test_qps" "$test_connections" "$test_duration" "$deploy_script" "$clear_script" "$nsm_version" - diff --git a/performance_testing/scripts/run_test_suite.sh b/performance_testing/scripts/run_test_suite.sh index 7f8e75bc5..8818530d6 100755 --- a/performance_testing/scripts/run_test_suite.sh +++ b/performance_testing/scripts/run_test_suite.sh @@ -40,7 +40,7 @@ qps3=1000000 connections1=1 duration1=60s -echo running tests for "$url" +echo running tests for "$test_url" # for current_qps in $qps3 for current_qps in $qps1 $qps2 $qps3 do diff --git a/performance_testing/scripts/setup_metallb.sh b/performance_testing/scripts/setup_metallb.sh index 55b3b7f4e..b1134f348 100755 --- a/performance_testing/scripts/setup_metallb.sh +++ b/performance_testing/scripts/setup_metallb.sh @@ -3,7 +3,7 @@ echo CLUSTER1_CIDR is "'$CLUSTER1_CIDR'" echo CLUSTER2_CIDR is "'$CLUSTER2_CIDR'" -if [[ ! -z $CLUSTER1_CIDR ]]; then +if [[ -n $CLUSTER1_CIDR ]]; then kubectl "--kubeconfig=$KUBECONFIG1" apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml kubectl "--kubeconfig=$KUBECONFIG1" apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/metallb.yaml cat > metallb-config.yaml < metallb-config.yaml < Date: Thu, 6 Apr 2023 11:02:45 +0700 Subject: [PATCH 10/22] Revert "disable standard ci" This reverts commit fe2831a67168c5a204ea1aefb29520ad642e5040. Signed-off-by: Danil Uzlov --- .github/workflows/ci.yaml | 466 +++++++++++++++++++------------------- 1 file changed, 233 insertions(+), 233 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a571dabd..566f7c3b9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,239 +22,239 @@ jobs: uses: networkservicemesh/.github/.github/workflows/checkgomod.yaml@main ### SINGLE CLUSTER - # kind: - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # image: - # - v1.25.2 - # - v1.24.6 - # - v1.23.12 - # - v1.22.13 - # - v1.21.14 - # - v1.20.15 - # steps: - # - name: Cancel Previous Runs - # uses: styfle/cancel-workflow-action@0.9.0 - # with: - # access_token: ${{ github.token }} - # - uses: actions/setup-go@v1 - # with: - # go-version: 1.16 - # github-token: ${{ github.token }} - # - name: Set go env - # run: | - # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - # echo GO111MODULE=on >> $GITHUB_ENV - # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - # - uses: actions/checkout@v2 - # with: - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - uses: engineerd/setup-kind@v0.5.0 - # with: - # config: src/github.com/${{ github.repository }}/cluster-config.yaml - # version: v0.13.0 - # image: kindest/node:${{ matrix.image }} - # - name: Check kind cluster - # run: | - # kubectl version - # kubectl cluster-info - # kubectl get pods -n kube-system - # echo "current-context:" $(kubectl config current-context) - # echo "environment-kubeconfig:" ${KUBECONFIG} - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Set loadbalancer CIDR - # run: | - # echo CLUSTER_CIDR="172.18.1.128/25" >> $GITHUB_ENV - # - name: Integration tests - # run: | - # go test -count 1 -timeout 2h30m -race -v ./tests_single - # env: - # ARTIFACTS_DIR: ${{ matrix.image }}-logs/${{ matrix.image }} - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Upload artifacts - # if: ${{ success() || failure() || cancelled() }} - # uses: actions/upload-artifact@v2 - # with: - # name: Single logs - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/${{ matrix.image }}-logs + kind: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - v1.25.2 + - v1.24.6 + - v1.23.12 + - v1.22.13 + - v1.21.14 + - v1.20.15 + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - uses: actions/setup-go@v1 + with: + go-version: 1.16 + github-token: ${{ github.token }} + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - uses: engineerd/setup-kind@v0.5.0 + with: + config: src/github.com/${{ github.repository }}/cluster-config.yaml + version: v0.13.0 + image: kindest/node:${{ matrix.image }} + - name: Check kind cluster + run: | + kubectl version + kubectl cluster-info + kubectl get pods -n kube-system + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Set loadbalancer CIDR + run: | + echo CLUSTER_CIDR="172.18.1.128/25" >> $GITHUB_ENV + - name: Integration tests + run: | + go test -count 1 -timeout 2h30m -race -v ./tests_single + env: + ARTIFACTS_DIR: ${{ matrix.image }}-logs/${{ matrix.image }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: Single logs + path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/${{ matrix.image }}-logs - # ### SINGLE IPv6 CLUSTER - # kind-ipv6: - # runs-on: ubuntu-latest - # env: - # KUBERNETES_VERSION: "v1.25.0" - # steps: - # - name: Cancel Previous Runs - # uses: styfle/cancel-workflow-action@0.9.0 - # with: - # access_token: ${{ github.token }} - # - uses: actions/setup-go@v1 - # with: - # go-version: 1.16 - # github-token: ${{ github.token }} - # - name: Set go env - # run: | - # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - # echo GO111MODULE=on >> $GITHUB_ENV - # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - # - uses: actions/checkout@v2 - # with: - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - uses: engineerd/setup-kind@v0.5.0 - # with: - # config: src/github.com/${{ github.repository }}/cluster-config-ipv6.yaml - # version: v0.13.0 - # image: kindest/node:${{ env.KUBERNETES_VERSION }} - # - name: Check kind cluster - # run: | - # kubectl version - # kubectl get pods -A -o wide - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Set loadbalancer CIDR - # run: | - # echo CLUSTER_CIDR="fc00:f853:ccd:e793:1::/80" >> $GITHUB_ENV - # - name: Integration tests - # run: | - # go test -count 1 -timeout 2h10m -race -v \ - # ./tests_single/basic_test.go \ - # ./tests_single/memory_test.go - # env: - # ARTIFACTS_DIR: ipv6-logs/${{ env.KUBERNETES_VERSION }} - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Upload artifacts - # if: ${{ success() || failure() || cancelled() }} - # uses: actions/upload-artifact@v2 - # with: - # name: Single IPv6 logs - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/ipv6-logs + ### SINGLE IPv6 CLUSTER + kind-ipv6: + runs-on: ubuntu-latest + env: + KUBERNETES_VERSION: "v1.25.0" + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - uses: actions/setup-go@v1 + with: + go-version: 1.16 + github-token: ${{ github.token }} + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - uses: engineerd/setup-kind@v0.5.0 + with: + config: src/github.com/${{ github.repository }}/cluster-config-ipv6.yaml + version: v0.13.0 + image: kindest/node:${{ env.KUBERNETES_VERSION }} + - name: Check kind cluster + run: | + kubectl version + kubectl get pods -A -o wide + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Set loadbalancer CIDR + run: | + echo CLUSTER_CIDR="fc00:f853:ccd:e793:1::/80" >> $GITHUB_ENV + - name: Integration tests + run: | + go test -count 1 -timeout 2h10m -race -v \ + ./tests_single/basic_test.go \ + ./tests_single/memory_test.go + env: + ARTIFACTS_DIR: ipv6-logs/${{ env.KUBERNETES_VERSION }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: Single IPv6 logs + path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/ipv6-logs - # ### SINGLE CALICO CLUSTER - # calico-kind: - # runs-on: ubuntu-latest - # env: - # KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} - # steps: - # - name: Cancel Previous Runs - # uses: styfle/cancel-workflow-action@0.9.0 - # with: - # access_token: ${{ github.token }} - # - uses: actions/setup-go@v1 - # with: - # go-version: 1.16 - # github-token: ${{ github.token }} - # - name: Set go env - # run: | - # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - # echo GO111MODULE=on >> $GITHUB_ENV - # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - # - name: Set kubernetes version - # run: | - # if [[ $KUBERNETES_VERSION=="" ]]; then - # echo "KUBERNETES_VERSION=v1.22.1" >> $GITHUB_ENV - # fi - # - uses: actions/checkout@v2 - # with: - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - uses: engineerd/setup-kind@v0.5.0 - # with: - # config: src/github.com/${{ github.repository }}/cluster-config-calico.yaml - # version: v0.11.1 - # image: kindest/node:${{ env.KUBERNETES_VERSION }} - # wait: 0s - # - name: Setup external CNI plugin - # shell: bash {0} - # run: | - # kubectl create -f https://projectcalico.docs.tigera.io/archive/v3.24/manifests/tigera-operator.yaml - # for i in {1..5}; do - # kubectl wait --for condition=established --timeout=1s crd/installations.operator.tigera.io - # result=$? - # if [ $result -eq 0 ]; then - # break - # fi - # sleep 1s - # done - # kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/calico/installation-default.yaml - # kubectl apply -k ${{ github.workspace }}/src/github.com/${{ github.repository }}/calico - # kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=5m - # - name: Check kind cluster - # run: | - # kubectl version - # kubectl cluster-info - # kubectl get pods --all-namespaces - # echo "current-context:" $(kubectl config current-context) - # echo "environment-kubeconfig:" ${KUBECONFIG} - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Integration tests - # continue-on-error: true - # run: | - # go test -count 1 -timeout 1h30m -race -v \ - # ./tests_single/basic_test.go \ - # ./tests_single/heal_test.go \ - # ./tests_single/memory_test.go \ - # ./tests_single/observability_test.go \ - # ./tests_single/feature_test.go \ - # -calico - # env: - # ARTIFACTS_DIR: calico-logs/${{ env.KUBERNETES_VERSION }} - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Upload artifacts - # if: ${{ success() || failure() || cancelled() }} - # uses: actions/upload-artifact@v2 - # with: - # name: Calico logs - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/calico-logs + ### SINGLE CALICO CLUSTER + calico-kind: + runs-on: ubuntu-latest + env: + KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - uses: actions/setup-go@v1 + with: + go-version: 1.16 + github-token: ${{ github.token }} + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - name: Set kubernetes version + run: | + if [[ $KUBERNETES_VERSION=="" ]]; then + echo "KUBERNETES_VERSION=v1.22.1" >> $GITHUB_ENV + fi + - uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - uses: engineerd/setup-kind@v0.5.0 + with: + config: src/github.com/${{ github.repository }}/cluster-config-calico.yaml + version: v0.11.1 + image: kindest/node:${{ env.KUBERNETES_VERSION }} + wait: 0s + - name: Setup external CNI plugin + shell: bash {0} + run: | + kubectl create -f https://projectcalico.docs.tigera.io/archive/v3.24/manifests/tigera-operator.yaml + for i in {1..5}; do + kubectl wait --for condition=established --timeout=1s crd/installations.operator.tigera.io + result=$? + if [ $result -eq 0 ]; then + break + fi + sleep 1s + done + kubectl create -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42/yaml/calico/installation-default.yaml + kubectl apply -k ${{ github.workspace }}/src/github.com/${{ github.repository }}/calico + kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=5m + - name: Check kind cluster + run: | + kubectl version + kubectl cluster-info + kubectl get pods --all-namespaces + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Integration tests + continue-on-error: true + run: | + go test -count 1 -timeout 1h30m -race -v \ + ./tests_single/basic_test.go \ + ./tests_single/heal_test.go \ + ./tests_single/memory_test.go \ + ./tests_single/observability_test.go \ + ./tests_single/feature_test.go \ + -calico + env: + ARTIFACTS_DIR: calico-logs/${{ env.KUBERNETES_VERSION }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: Calico logs + path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_single/calico-logs - # ### INTERDOMAIN CLUSTER - # interdomain-kind: - # runs-on: ubuntu-latest - # env: - # KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} - # steps: - # - name: Cancel Previous Runs - # uses: styfle/cancel-workflow-action@0.9.0 - # with: - # access_token: ${{ github.token }} - # - uses: actions/setup-go@v1 - # with: - # go-version: 1.16 - # github-token: ${{ github.token }} - # - name: Set go env - # run: | - # echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - # echo GO111MODULE=on >> $GITHUB_ENV - # echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - # - uses: actions/checkout@v2 - # with: - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Get kind - # run: go get sigs.k8s.io/kind@v0.11.1 - # - name: Create kind clusters - # run: | - # if [[ $KUBERNETES_VERSION=="" ]]; then - # KUBERNETES_VERSION="v1.22.1" - # fi - # for (( i = 1; i <= 3; i++ )); do - # kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" - # configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} - # kind get kubeconfig --name "kind-${i}" > ${configPath} - # echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV - # echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV - # done - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Interdomain tests - # run: | - # go test -count 1 -timeout 1h -race -v ./tests_interdomain - # env: - # ARTIFACTS_DIR: interdomain-logs - # working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - # - name: Cleanup resources - # if: ${{ success() || failure() || cancelled() }} - # run: kind delete clusters $(kind get clusters) - # - name: Upload artifacts - # if: ${{ success() || failure() || cancelled() }} - # uses: actions/upload-artifact@v2 - # with: - # name: Interdomain logs - # path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_interdomain/interdomain-logs + ### INTERDOMAIN CLUSTER + interdomain-kind: + runs-on: ubuntu-latest + env: + KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - uses: actions/setup-go@v1 + with: + go-version: 1.16 + github-token: ${{ github.token }} + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Get kind + run: go get sigs.k8s.io/kind@v0.11.1 + - name: Create kind clusters + run: | + if [[ $KUBERNETES_VERSION=="" ]]; then + KUBERNETES_VERSION="v1.22.1" + fi + for (( i = 1; i <= 3; i++ )); do + kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" + configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} + kind get kubeconfig --name "kind-${i}" > ${configPath} + echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV + echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV + done + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Interdomain tests + run: | + go test -count 1 -timeout 1h -race -v ./tests_interdomain + env: + ARTIFACTS_DIR: interdomain-logs + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Cleanup resources + if: ${{ success() || failure() || cancelled() }} + run: kind delete clusters $(kind get clusters) + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: Interdomain logs + path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/tests_interdomain/interdomain-logs From b500b4048f6aad75004fd7b8bfc47c81ab76e960 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 11:10:24 +0700 Subject: [PATCH 11/22] remove unused readme file Signed-off-by: Danil Uzlov --- .../use-cases/k2wireguard2k/README.md | 105 ------------------ 1 file changed, 105 deletions(-) delete mode 100644 performance_testing/use-cases/k2wireguard2k/README.md diff --git a/performance_testing/use-cases/k2wireguard2k/README.md b/performance_testing/use-cases/k2wireguard2k/README.md deleted file mode 100644 index 1f5363022..000000000 --- a/performance_testing/use-cases/k2wireguard2k/README.md +++ /dev/null @@ -1,105 +0,0 @@ -# Test kernel to wireguard to kernel connection - -This example shows that NSC and NSE on the different clusters could find and work with each other. - -NSC and NSE are using the `kernel` mechanism to connect to its local forwarder. -Forwarders are using the `wireguard` mechanism to connect with each other. - -## Requires - -Make sure that you have completed steps from [interdomain](../../) - -## Run - -**1. Deploy endpoint on cluster2** - -```bash -export KUBECONFIG=$KUBECONFIG2 -``` - -```bash -kubectl create ns ns-interdomain-kernel2wireguard2kernel -``` - -Deploy NSE: -```bash -kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/multicluster/usecases/interdomain_Kernel2Wireguard2Kernel/cluster2?ref=v1.7.0-rc.2 -``` - -Wait for applications ready: -```bash -kubectl wait --for=condition=ready --timeout=1m pod -l app=nse-kernel -n ns-interdomain-kernel2wireguard2kernel -``` - -Find NSE pod by labels: -```bash -NSE=$(kubectl get pods -l app=nse-kernel -n ns-interdomain-kernel2wireguard2kernel --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') -[[ ! -z $NSE ]] -``` - -**2. Deploy client on cluster1** - -```bash -export KUBECONFIG=$KUBECONFIG1 -``` - -```bash -kubectl create ns ns-interdomain-kernel2wireguard2kernel -``` - -Deploy NSC: -```bash -kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/multicluster/usecases/interdomain_Kernel2Wireguard2Kernel/cluster1?ref=v1.7.0-rc.2 -``` - -Wait for applications ready: -```bash -kubectl wait --for=condition=ready --timeout=5m pod -l app=alpine -n ns-interdomain-kernel2wireguard2kernel -``` - -Find NSC pod by labels: -```bash -NSC=$(kubectl get pods -l app=alpine -n ns-interdomain-kernel2wireguard2kernel --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') -[[ ! -z $NSC ]] -``` - -**3. Check connectivity** - -Switch to *cluster1*: - -```bash -export KUBECONFIG=$KUBECONFIG1 -``` - -```bash -kubectl exec ${NSC} -n ns-interdomain-kernel2wireguard2kernel -- ping -c 4 172.16.1.2 -``` - -Switch to *cluster2*: - -```bash -export KUBECONFIG=$KUBECONFIG2 -``` - -Ping from NSE to NSC: -```bash -kubectl exec ${NSE} -n ns-interdomain-kernel2wireguard2kernel -- ping -c 4 172.16.1.3 -``` - -## Cleanup - -1. Cleanup resources for *cluster1*: -```bash -export KUBECONFIG=$KUBECONFIG1 -``` -```bash -kubectl delete ns ns-interdomain-kernel2wireguard2kernel -``` - -2. Cleanup resources for *cluster2*: -```bash -export KUBECONFIG=$KUBECONFIG2 -``` -```bash -kubectl delete ns ns-interdomain-kernel2wireguard2kernel -``` From 4107ae0361299caae8cb696759ea6de135e8a762 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 11:42:41 +0700 Subject: [PATCH 12/22] use github branch name to select nsm version Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml index 3967370fe..9cffeb12b 100644 --- a/.github/workflows/perfromance-testing.yaml +++ b/.github/workflows/perfromance-testing.yaml @@ -46,7 +46,7 @@ jobs: run: | performance_testing/scripts/full_ci_run.sh $NSM_VERSION $ARTIFACTS_DIR env: - NSM_VERSION: v1.8.0 + NSM_VERSION: ${{ env.GITHUB_REF_NAME }} ARTIFACTS_DIR: perf-test-results working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Cleanup resources From fb28aebf149360c37e385f52748c845dbf4374c5 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 6 Apr 2023 11:49:24 +0700 Subject: [PATCH 13/22] fix ci config Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml index 9cffeb12b..20ee23835 100644 --- a/.github/workflows/perfromance-testing.yaml +++ b/.github/workflows/perfromance-testing.yaml @@ -46,7 +46,7 @@ jobs: run: | performance_testing/scripts/full_ci_run.sh $NSM_VERSION $ARTIFACTS_DIR env: - NSM_VERSION: ${{ env.GITHUB_REF_NAME }} + NSM_VERSION: ${{ github.ref_name }} ARTIFACTS_DIR: perf-test-results working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Cleanup resources From 68c382b8069ef8fdffea08a6fa9a58a1a5ded02d Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 13 Apr 2023 16:23:50 +0700 Subject: [PATCH 14/22] expose test parameters in script args Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 100 +++++++++--------- performance_testing/scripts/full_ci_run.sh | 18 +++- performance_testing/scripts/run_test_suite.sh | 21 ++-- 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml index 20ee23835..52a628011 100644 --- a/.github/workflows/perfromance-testing.yaml +++ b/.github/workflows/perfromance-testing.yaml @@ -3,7 +3,7 @@ name: performance-testing on: push: branches: - - 'release/**' + - 'release/**' jobs: ### INTERDOMAIN CLUSTER interdomain-kind: @@ -11,50 +11,54 @@ jobs: env: KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.0 - with: - access_token: ${{ github.token }} - - uses: actions/setup-go@v1 - with: - go-version: 1.16 - github-token: ${{ github.token }} - - name: Set go env - run: | - echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV - echo GO111MODULE=on >> $GITHUB_ENV - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - - uses: actions/checkout@v2 - with: - path: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Get kind - run: go get sigs.k8s.io/kind@v0.11.1 - - name: Create kind clusters - run: | - if [[ $KUBERNETES_VERSION=="" ]]; then - KUBERNETES_VERSION="v1.22.1" - fi - for (( i = 1; i <= 2; i++ )); do - kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" - configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} - kind get kubeconfig --name "kind-${i}" > ${configPath} - echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV - echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV - done - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Performance tests - run: | - performance_testing/scripts/full_ci_run.sh $NSM_VERSION $ARTIFACTS_DIR - env: - NSM_VERSION: ${{ github.ref_name }} - ARTIFACTS_DIR: perf-test-results - working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Cleanup resources - if: ${{ success() || failure() || cancelled() }} - run: kind delete clusters $(kind get clusters) - - name: Upload artifacts - if: ${{ success() || failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: Performance tests results and logs - path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/perf-test-results + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - uses: actions/setup-go@v1 + with: + go-version: 1.16 + github-token: ${{ github.token }} + - name: Set go env + run: | + echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV + echo GO111MODULE=on >> $GITHUB_ENV + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: actions/checkout@v2 + with: + path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Get kind + run: go get sigs.k8s.io/kind@v0.11.1 + - name: Create kind clusters + run: | + if [[ $KUBERNETES_VERSION=="" ]]; then + KUBERNETES_VERSION="v1.22.1" + fi + for (( i = 1; i <= 2; i++ )); do + kind create cluster --name "kind-${i}" --config cluster-config-interdomain.yaml --image="kindest/node:$KUBERNETES_VERSION" + configPath=${{ github.workspace }}/src/github.com/${{ github.repository }}/config${i} + kind get kubeconfig --name "kind-${i}" > ${configPath} + echo KUBECONFIG${i}=${configPath} >> $GITHUB_ENV + echo CLUSTER${i}_CIDR="172.18.${i}.128/25" >> $GITHUB_ENV + done + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Performance tests + run: | + performance_testing/scripts/full_ci_run.sh "$NSM_VERSION" "$ARTIFACTS_DIR" "$QPS_LIST" "$DURATION" "$CONNECTIONS" "$ITERATIONS" + env: + NSM_VERSION: ${{ github.ref_name }} + ARTIFACTS_DIR: perf-test-results + QPS_LIST: 100 1000 1000000 + DURATION: 60s + CONNECTIONS: 1 + ITERATIONS: 3 + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Cleanup resources + if: ${{ success() || failure() || cancelled() }} + run: kind delete clusters $(kind get clusters) + - name: Upload artifacts + if: ${{ success() || failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: Performance tests results and logs + path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/perf-test-results diff --git a/performance_testing/scripts/full_ci_run.sh b/performance_testing/scripts/full_ci_run.sh index 4235c8874..5a2ec6483 100755 --- a/performance_testing/scripts/full_ci_run.sh +++ b/performance_testing/scripts/full_ci_run.sh @@ -9,9 +9,17 @@ if [ -z "$2" ]; then echo 2nd arg 'result_folder' is missing; exit 1; fi nsm_version=$1 result_folder=$2 +qps_list=${3:-1000000} +duration=${4:-60s} +connections=${5:-1} +iterations=${6:-3} echo nsm_version: "$nsm_version" echo result_folder: "$result_folder" +echo qps_list: "$qps_list" +echo duration: "$duration" +echo connections: "$connections" +echo iterations: "$iterations" "$parent_path/setup_metallb.sh" || exit @@ -21,23 +29,29 @@ echo result_folder: "$result_folder" "$parent_path/run_test_suite.sh" \ vl3 \ "$result_folder" \ - 3 \ + "$iterations" \ "http://nginx.my-vl3-network:80" \ "$parent_path/../use-cases/vl3/deploy.sh" \ "$parent_path/../use-cases/vl3/clear.sh" \ "$nsm_version" \ "$parent_path/../nsm" \ + "$qps_list" \ + "$duration" \ + "$connections" \ || exit "$parent_path/run_test_suite.sh" \ k2wireguard2k \ "$result_folder" \ - 3 \ + "$iterations" \ "http://172.16.1.2:80" \ "$parent_path/../use-cases/k2wireguard2k/deploy.sh" \ "$parent_path/../use-cases/k2wireguard2k/clear.sh" \ "$nsm_version" \ "$parent_path/../nsm" \ + "$qps_list" \ + "$duration" \ + "$connections" \ || exit "$parent_path/nsm_clear_spire.sh" diff --git a/performance_testing/scripts/run_test_suite.sh b/performance_testing/scripts/run_test_suite.sh index 8818530d6..3cd02a895 100755 --- a/performance_testing/scripts/run_test_suite.sh +++ b/performance_testing/scripts/run_test_suite.sh @@ -13,6 +13,9 @@ if [ -z "$5" ]; then echo 5th arg 'deploy_script' is missing; exit 1; fi if [ -z "$6" ]; then echo 6th arg 'clear_script' is missing; exit 1; fi if [ -z "$7" ]; then echo 7th arg 'nsm_version' is missing; exit 1; fi if [ -z "$8" ]; then echo 8th arg 'nsm_deploy_folder' is missing; exit 1; fi +if [ -z "$9" ]; then echo 9th arg 'qps_list' is missing; exit 1; fi +if [ -z "${10}" ]; then echo 10th arg 'duration' is missing; exit 1; fi +if [ -z "${11}" ]; then echo 10th arg 'connections' is missing; exit 1; fi test_name=test-$(TZ=UTC date +%F-T%H-%M-%S)-$1 result_folder=$2/$test_name @@ -22,6 +25,9 @@ deploy_script=$5 clear_script=$6 nsm_version=$7 nsm_deploy_folder=$8 +qps_list=$9 +duration=${10} +connections=${11} echo "test_name: $test_name" echo "result_folder: $result_folder" @@ -31,18 +37,15 @@ echo "deploy_script: $deploy_script" echo "clear_script: $clear_script" echo "nsm_version: $nsm_version" echo "nsm_deploy_folder: $nsm_deploy_folder" +echo "qps_list: $qps_list" +echo "duration: $duration" +echo "connections: $connections" mkdir -p "$result_folder" || exit -qps1=100 -qps2=1000 -qps3=1000000 -connections1=1 -duration1=60s - echo running tests for "$test_url" # for current_qps in $qps3 -for current_qps in $qps1 $qps2 $qps3 +for current_qps in $qps_list do echo "testing qps $current_qps" "$parent_path/run_test_single.sh" \ @@ -51,8 +54,8 @@ do "$test_iterations" \ "$test_url" \ "$current_qps" \ - "$connections1" \ - "$duration1" \ + "$connections" \ + "$duration" \ "$deploy_script" \ "$clear_script" \ "$nsm_version" \ From b8ae41bb8caf782a425fc6ce382affb19d597eb0 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Thu, 13 Apr 2023 16:24:10 +0700 Subject: [PATCH 15/22] add documentation Signed-off-by: Danil Uzlov --- performance_testing/README.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/performance_testing/README.md b/performance_testing/README.md index 6aff3ceb5..77149079e 100644 --- a/performance_testing/README.md +++ b/performance_testing/README.md @@ -1,5 +1,30 @@ -# Test use-cases +# Performance testing + +This folder contains deployment yaml files and scripts +that deploy, run and clear applications for performance testing. + +# Parameters + +Parameters to be considered are: + +1. `qps_list`: requested load of the system +2. `duration`: duration of a single test +3. `connections`: the amount of simultaneous connections from test client to test server +4. `iterations`: how many times to run each test + +To inspect results you can install Fortio and run `fortio server`. +In the web ui you will be able to see graphs for different runs and compare them. + +Alternatively you can simply open .json files and inspect them for QPS and different latency percentiles. + +# Running the tests manually locally + +Make sure that you have load ballancer in you cluster. +For Kind and bare metal clusters you can use metallb installation script: +```bash +./performance_testing/scripts/setup_metallb.sh +``` Prepare DNS and Spire: ```bash @@ -7,7 +32,7 @@ Prepare DNS and Spire: ./performance_testing/scripts/nsm_setup_spire.sh ``` -Test with vl3 DNS: +Test interdomain vl3: ```bash ./performance_testing/scripts/run_test_suite.sh \ vl3 \ From fc6c91237d9ecfa64bdb5ef573c9e89e7e4c9dd1 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Tue, 18 Apr 2023 13:01:16 +0700 Subject: [PATCH 16/22] print results on ci Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 5 +++ .../scripts/print_all_summaries.sh | 12 +++++++ performance_testing/scripts/print_summary.sh | 35 +++++++++++++++++++ .../scripts/run_test_single.sh | 5 ++- 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 performance_testing/scripts/print_all_summaries.sh create mode 100644 performance_testing/scripts/print_summary.sh diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml index 52a628011..a05f68e24 100644 --- a/.github/workflows/perfromance-testing.yaml +++ b/.github/workflows/perfromance-testing.yaml @@ -52,6 +52,11 @@ jobs: DURATION: 60s CONNECTIONS: 1 ITERATIONS: 3 + - name: Print results + run: | + print_all_summaries.sh "$ARTIFACTS_DIR" + env: + ARTIFACTS_DIR: perf-test-results working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Cleanup resources if: ${{ success() || failure() || cancelled() }} diff --git a/performance_testing/scripts/print_all_summaries.sh b/performance_testing/scripts/print_all_summaries.sh new file mode 100644 index 000000000..895c4ac4b --- /dev/null +++ b/performance_testing/scripts/print_all_summaries.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +result_folder=$1 + +echo showing results from "$result_folder" + +folders="$result_folder"/* + +for folder in $folders +do + print_summary.sh "$folder"/results +done diff --git a/performance_testing/scripts/print_summary.sh b/performance_testing/scripts/print_summary.sh new file mode 100644 index 000000000..48b02850e --- /dev/null +++ b/performance_testing/scripts/print_summary.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +result_folder=$1 + +echo showing results from "$result_folder" + +function printForFile() { + filepath=$1 + + qps=$(cat "$filepath" | jq .ActualQPS) + count=$(cat "$filepath" | jq .DurationHistogram.Count) + min=$(cat "$filepath" | jq .DurationHistogram.Min) + max=$(cat "$filepath" | jq .DurationHistogram.Max) + avg=$(cat "$filepath" | jq .DurationHistogram.Avg) + p50=$(cat "$filepath" | jq .DurationHistogram.Percentiles[0].Value) + p99=$(cat "$filepath" | jq .DurationHistogram.Percentiles[4].Value) + + echo ---------------------- + echo From "$filepath" + echo QPS: "$qps" + echo Total queries: "$count" + echo Min: $(awk '{print $1 * 1000}' <<<"$min") ms + echo Max: $(awk '{print $1 * 1000}' <<<"$max") ms + echo Avg: $(awk '{print $1 * 1000}' <<<"$avg") ms + echo p50: $(awk '{print $1 * 1000}' <<<"$p50") ms + echo p99: $(awk '{print $1 * 1000}' <<<"$p99") ms + echo ---------------------- +} + +files="$result_folder"/* + +for file in $files +do + printForFile "$file" +done diff --git a/performance_testing/scripts/run_test_single.sh b/performance_testing/scripts/run_test_single.sh index 7e857933f..5c133ea44 100755 --- a/performance_testing/scripts/run_test_single.sh +++ b/performance_testing/scripts/run_test_single.sh @@ -85,6 +85,9 @@ function runTest() { deploy_logs=$result_folder/deploy mkdir -p "$deploy_logs" + final_results=$result_folder/results + mkdir -p "$final_results" + echo "config name: $config_name" echo "measure for $iterations iterations" @@ -99,7 +102,7 @@ function runTest() { echo doing warmup run... curl -s -d "$config" "localhost:8080/fortio/rest/run" > "$warmup_results/$test_full_name-warmup.json" echo doing main run... - curl -s -d "$config" "localhost:8080/fortio/rest/run" > "$result_folder/$test_full_name.json" + curl -s -d "$config" "localhost:8080/fortio/rest/run" > "$final_results/$test_full_name.json" result_code=$? echo saving pod layout k1 get pod -A -o wide > "$deploy_logs/$test_full_name-k1-pods.log" From 31f397e60a4924e6132c49033d2f394f557cff64 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Tue, 18 Apr 2023 13:03:25 +0700 Subject: [PATCH 17/22] fix file permissions Signed-off-by: Danil Uzlov --- performance_testing/scripts/print_all_summaries.sh | 0 performance_testing/scripts/print_summary.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 performance_testing/scripts/print_all_summaries.sh mode change 100644 => 100755 performance_testing/scripts/print_summary.sh diff --git a/performance_testing/scripts/print_all_summaries.sh b/performance_testing/scripts/print_all_summaries.sh old mode 100644 new mode 100755 diff --git a/performance_testing/scripts/print_summary.sh b/performance_testing/scripts/print_summary.sh old mode 100644 new mode 100755 From d237a75bde3bfa0fea1157c9889d7a93b3f37a9a Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Tue, 18 Apr 2023 13:28:40 +0700 Subject: [PATCH 18/22] fix print scripts Signed-off-by: Danil Uzlov --- .../scripts/print_all_summaries.sh | 6 +++-- performance_testing/scripts/print_summary.sh | 26 +++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/performance_testing/scripts/print_all_summaries.sh b/performance_testing/scripts/print_all_summaries.sh index 895c4ac4b..312ff20b4 100755 --- a/performance_testing/scripts/print_all_summaries.sh +++ b/performance_testing/scripts/print_all_summaries.sh @@ -1,12 +1,14 @@ #!/bin/bash +parent_path=$( cd "$(dirname "$0")" ; pwd -P ) || exit + result_folder=$1 echo showing results from "$result_folder" -folders="$result_folder"/* +folders="$result_folder/*" for folder in $folders do - print_summary.sh "$folder"/results + "$parent_path/print_summary.sh" "$folder" done diff --git a/performance_testing/scripts/print_summary.sh b/performance_testing/scripts/print_summary.sh index 48b02850e..33f9c4147 100755 --- a/performance_testing/scripts/print_summary.sh +++ b/performance_testing/scripts/print_summary.sh @@ -7,27 +7,27 @@ echo showing results from "$result_folder" function printForFile() { filepath=$1 - qps=$(cat "$filepath" | jq .ActualQPS) - count=$(cat "$filepath" | jq .DurationHistogram.Count) - min=$(cat "$filepath" | jq .DurationHistogram.Min) - max=$(cat "$filepath" | jq .DurationHistogram.Max) - avg=$(cat "$filepath" | jq .DurationHistogram.Avg) - p50=$(cat "$filepath" | jq .DurationHistogram.Percentiles[0].Value) - p99=$(cat "$filepath" | jq .DurationHistogram.Percentiles[4].Value) + qps=$(< "$filepath" jq .ActualQPS) + count=$(< "$filepath" jq .DurationHistogram.Count) + min=$(< "$filepath" jq .DurationHistogram.Min) + max=$(< "$filepath" jq .DurationHistogram.Max) + avg=$(< "$filepath" jq .DurationHistogram.Avg) + p50=$(< "$filepath" jq .DurationHistogram.Percentiles[0].Value) + p99=$(< "$filepath" jq .DurationHistogram.Percentiles[4].Value) echo ---------------------- echo From "$filepath" echo QPS: "$qps" echo Total queries: "$count" - echo Min: $(awk '{print $1 * 1000}' <<<"$min") ms - echo Max: $(awk '{print $1 * 1000}' <<<"$max") ms - echo Avg: $(awk '{print $1 * 1000}' <<<"$avg") ms - echo p50: $(awk '{print $1 * 1000}' <<<"$p50") ms - echo p99: $(awk '{print $1 * 1000}' <<<"$p99") ms + echo Min: "$(awk '{print $1 * 1000}' <<<"$min")" ms + echo Max: "$(awk '{print $1 * 1000}' <<<"$max")" ms + echo Avg: "$(awk '{print $1 * 1000}' <<<"$avg")" ms + echo p50: "$(awk '{print $1 * 1000}' <<<"$p50")" ms + echo p99: "$(awk '{print $1 * 1000}' <<<"$p99")" ms echo ---------------------- } -files="$result_folder"/* +files="$result_folder/*" for file in $files do From 54e3c6e6d062bcf01be1e35aa76814e22ddc9c8e Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Tue, 18 Apr 2023 13:44:07 +0700 Subject: [PATCH 19/22] fix ci config Signed-off-by: Danil Uzlov --- .github/workflows/perfromance-testing.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/perfromance-testing.yaml b/.github/workflows/perfromance-testing.yaml index a05f68e24..2fdaf6241 100644 --- a/.github/workflows/perfromance-testing.yaml +++ b/.github/workflows/perfromance-testing.yaml @@ -52,9 +52,10 @@ jobs: DURATION: 60s CONNECTIONS: 1 ITERATIONS: 3 + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - name: Print results run: | - print_all_summaries.sh "$ARTIFACTS_DIR" + performance_testing/scripts/print_all_summaries.sh "$ARTIFACTS_DIR" env: ARTIFACTS_DIR: perf-test-results working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} From 2fc4608ed9472f5952114829f0c664cda77dee9e Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Tue, 18 Apr 2023 16:13:00 +0700 Subject: [PATCH 20/22] fix print script Signed-off-by: Danil Uzlov --- performance_testing/scripts/print_all_summaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance_testing/scripts/print_all_summaries.sh b/performance_testing/scripts/print_all_summaries.sh index 312ff20b4..3d8d77131 100755 --- a/performance_testing/scripts/print_all_summaries.sh +++ b/performance_testing/scripts/print_all_summaries.sh @@ -10,5 +10,5 @@ folders="$result_folder/*" for folder in $folders do - "$parent_path/print_summary.sh" "$folder" + "$parent_path/print_summary.sh" "$folder"/results done From 7493f7a2a3a07e8c992ac9b296dc85ee56214340 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Wed, 19 Apr 2023 10:50:27 +0700 Subject: [PATCH 21/22] adjust format Signed-off-by: Danil Uzlov --- performance_testing/scripts/print_summary.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/performance_testing/scripts/print_summary.sh b/performance_testing/scripts/print_summary.sh index 33f9c4147..e6a2461a2 100755 --- a/performance_testing/scripts/print_summary.sh +++ b/performance_testing/scripts/print_summary.sh @@ -15,7 +15,7 @@ function printForFile() { p50=$(< "$filepath" jq .DurationHistogram.Percentiles[0].Value) p99=$(< "$filepath" jq .DurationHistogram.Percentiles[4].Value) - echo ---------------------- + echo ----------------------- echo From "$filepath" echo QPS: "$qps" echo Total queries: "$count" @@ -24,7 +24,7 @@ function printForFile() { echo Avg: "$(awk '{print $1 * 1000}' <<<"$avg")" ms echo p50: "$(awk '{print $1 * 1000}' <<<"$p50")" ms echo p99: "$(awk '{print $1 * 1000}' <<<"$p99")" ms - echo ---------------------- + echo ----------------------- } files="$result_folder/*" From d2d7371d9412ff47e1ce19fe97a685af69f4d229 Mon Sep 17 00:00:00 2001 From: Danil Uzlov Date: Wed, 19 Apr 2023 16:29:18 +0700 Subject: [PATCH 22/22] add current results Signed-off-by: Danil Uzlov --- performance_testing/known-results.md | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 performance_testing/known-results.md diff --git a/performance_testing/known-results.md b/performance_testing/known-results.md new file mode 100644 index 000000000..8456c3723 --- /dev/null +++ b/performance_testing/known-results.md @@ -0,0 +1,57 @@ + +# Known results + +This file contains info about results we already have. + +There are several different QPS targets. Each target has its own result expectations. + +# NSM v1.8.0, vl3 + +vl3 tests in v1.8.0 seems to be CPU throttled by github, which affects max latency. + +1. Target QPS == 100 + Actual QPS: 100 + Min latency: 0.3-0.35 ms + Max latency: 100-250 ms + Avg latency: 4-5 ms + p50 latency: 1.5-2.5 ms + p99 atency: 90-150 ms +2. Target QPS == 1000 + Actual QPS: 350-450 + Min latency: 0.25-0.3 ms + Max latency: 100-300 ms + Avg latency: 2-2.5 ms + p50 latency: 0.7-1.3 ms + p99 atency: 40-80 ms +3. Target QPS == 1000000 + Actual QPS: 350-450 + Min latency: 0.25-0.3 ms + Max latency: 100-300 + Avg latency: 2-2.5 ms + p50 latency: 0.7-1.3 ms + p99 atency: 40-80 ms + +# NSM v1.8.0, wireguard + +1. Target QPS == 100 + Actual QPS: 100 + Min latency: 0.3 ms + Max latency: 20-50 ms + Avg latency: 1-2 ms + p50 latency: 0.6 ms + p99 atency: 15-35 ms +2. Target QPS == 1000 + Actual QPS: 1000 + Min latency: 0.2 ms + Max latency: 30-50 ms + Avg latency: 0.8 ms + p50 latency: 0.4-0.5 ms + p99 atency: 12-15 ms +3. Target QPS == 1000000 + Actual QPS: 1200-1400 + Min latency: 0.2 ms + Max latency: 40-50 ms + Avg latency: 0.7 ms + p50 latency: 0.4 ms + p99 atency: 12-15 ms +