✅ K6 fault tolerance (#1071)

* Warn if scenario completed before restarts finished * Configurable sleep during create holders to slow down req/s * Collections to run all holder & issuer operations * Refactor getWalletIndex function
didx-xyz · Sep 26, 2024 · 3884838 · 3884838
1 parent b8ff92a
commit 3884838
Show file tree

Hide file tree

Showing 11 changed files with 141 additions and 78 deletions.
diff --git a/scripts/k6/libs/functions.js b/scripts/k6/libs/functions.js
@@ -7,6 +7,11 @@ import { Counter, Trend } from "k6/metrics";
 import sse from "k6/x/sse";
 // let customDuration = new Trend('custom_duration', true);
 
+// Helper function to generate a unique, zero-based index for even distribution of operations
+export function getWalletIndex(vu, iter) {
+  return (vu - 1) * (iter + 1) + iter;
+}
+
 function logError(response, requestBody) {
   console.error(`Response status: ${response.status}`);
   console.error(`Response body: ${response.body}`);

diff --git a/scripts/k6/scenarios/create-credentials.js b/scripts/k6/scenarios/create-credentials.js
@@ -2,19 +2,13 @@
 /* eslint-disable no-undefined, no-console, camelcase */
 
 import { check, sleep } from "k6";
-// import { shuffle } from "k6/data";
 import { Counter, Trend } from "k6/metrics";
 import file from "k6/x/file";
 import { getBearerToken } from "../libs/auth.js";
-import { acceptCredential, createCredential, genericWaitForSSEEvent, getCredentialIdByThreadId } from "../libs/functions.js";
-import { bootstrapIssuer } from "../libs/setup.js";
+import { acceptCredential, createCredential, genericWaitForSSEEvent, getCredentialIdByThreadId, getWalletIndex } from "../libs/functions.js";
 
 const vus = Number.parseInt(__ENV.VUS, 10);
 const iterations = Number.parseInt(__ENV.ITERATIONS, 10);
-const issuerPrefix = __ENV.ISSUER_PREFIX;
-const schemaName = __ENV.SCHEMA_NAME;
-const schemaVersion = __ENV.SCHEMA_VERSION;
-const numIssuers = __ENV.NUM_ISSUERS;
 
 export const options = {
   scenarios: {
@@ -64,44 +58,23 @@ function shuffleArray(array) {
   return array;
 }
 
-
 export function setup() {
   const bearerToken = getBearerToken();
 
   file.writeString(outputFilepath, "");
   let holders = data.trim().split("\n").map(JSON.parse);
   holders = shuffleArray(holders); // Randomize the order of holders
 
-  // const walletName = issuerPrefix;
-  // const credDefTag = walletName;
-  // const issuers = bootstrapIssuer(numIssuers, walletName, credDefTag, schemaName, schemaVersion);
-
-  // if (!issuers || issuers.length === 0) {
-  //   console.error("Failed to bootstrap issuers.");
-  // }
-
   return { bearerToken, holders };
 }
 
-const iterationsPerVU = options.scenarios.default.iterations;
-// Helper function to calculate the wallet index based on VU and iteration
-function getWalletIndex(vu, iter) {
-  const walletIndex = (vu - 1) * iterationsPerVU + (iter - 1);
-  return walletIndex;
-}
-
-//random number between 0 and 100 (including 0 and 100 as options)
-function getRandomInt() {
-  return Math.floor(Math.random() * 101);
-}
-
 export default function (data) {
   const bearerToken = data.bearerToken;
   const holders = data.holders;
-  const walletIndex = getWalletIndex(__VU, __ITER + 1); // __ITER starts from 0, adding 1 to align with the logic
+  const walletIndex = getWalletIndex(__VU, __ITER);
   const wallet = holders[walletIndex];
 
-  // console.log(`VU: ${__VU}, Iteration: ${__ITER}, Issuer Wallet ID: ${wallet.issuer_wallet_id}`);
+  // console.log(`VU: ${__VU}, Iteration: ${__ITER}, Wallet Index: ${walletIndex}, Issuer Wallet ID: ${wallet.issuer_wallet_id}`);
 
   let createCredentialResponse;
   try {

diff --git a/scripts/k6/scenarios/create-holders.js b/scripts/k6/scenarios/create-holders.js
@@ -1,7 +1,7 @@
 /* global __ENV, __ITER, __VU */
 /* eslint-disable no-undefined, no-console, camelcase */
 
-import { check } from "k6";
+import { check, sleep } from "k6";
 import { SharedArray } from "k6/data";
 import { Counter, Trend } from "k6/metrics";
 import file from "k6/x/file";
@@ -11,6 +11,7 @@ import { createTenant } from "../libs/functions.js";
 const vus = Number.parseInt(__ENV.VUS, 10);
 const iterations = Number.parseInt(__ENV.ITERATIONS, 10);
 const holderPrefix = __ENV.HOLDER_PREFIX;
+const sleepDuration = Number.parseInt(__ENV.SLEEP_DURATION, 0);
 
 export const options = {
   scenarios: {
@@ -101,6 +102,6 @@ export default function (data) {
   const duration = end - start;
   // console.log(`Duration for iteration ${__ITER}: ${duration} ms`);
   mainIterationDuration.add(duration);
-  // sleep(1);
+  sleep(sleepDuration);
   testFunctionReqs.add(1);
 }
diff --git a/scripts/k6/scenarios/create-invitations.js b/scripts/k6/scenarios/create-invitations.js
@@ -5,7 +5,7 @@ import { check } from "k6";
 import { Counter, Trend } from "k6/metrics";
 import file from "k6/x/file";
 import { getBearerToken } from "../libs/auth.js";
-import { acceptInvitation, createInvitation, genericWaitForSSEEvent } from "../libs/functions.js";
+import { acceptInvitation, createInvitation, genericWaitForSSEEvent, getWalletIndex } from "../libs/functions.js";
 import { bootstrapIssuer } from "../libs/setup.js";
 // import bootstrapIssuer from "./bootstrap-issuer.js";
 
@@ -68,13 +68,6 @@ export function setup() {
   return { bearerToken, issuers, holders };
 }
 
-const iterationsPerVU = options.scenarios.default.iterations;
-// Helper function to calculate the wallet index based on VU and iteration
-function getWalletIndex(vu, iter) {
-  const walletIndex = (vu - 1) * iterationsPerVU + (iter - 1);
-  return walletIndex;
-}
-
 function getIssuerIndex(vu, iter) {
   const numIssuers = __ENV.NUM_ISSUERS;
   return (vu + iter - 2) % numIssuers;
@@ -90,7 +83,7 @@ export default function (data) {
   const start = Date.now();
   const bearerToken = data.bearerToken;
   const issuers = data.issuers;
-  const walletIndex = getWalletIndex(__VU, __ITER + 1); // __ITER starts from 0, adding 1 to align with the logic
+  const walletIndex = getWalletIndex(__VU, __ITER);
 
   const holders = data.holders;
   const wallet = holders[walletIndex];
@@ -99,7 +92,7 @@ export default function (data) {
   const issuerIndex = getIssuerIndex(__VU, __ITER + 1);
   const issuer = issuers[issuerIndex];
 
-  // console.log(`VU: ${__VU}, Iteration: ${__ITER}, Issuer Index: ${issuerIndex}, Issuer Wallet ID: ${issuer.walletId}`);
+  // console.log(`VU: ${__VU}, Iteration: ${__ITER}, Wallet Index: ${walletIndex}, Issuer Index: ${issuerIndex}, Issuer Wallet ID: ${issuer.walletId}`);
 
   const createInvitationResponse = createInvitation(bearerToken, issuer.accessToken);
   check(createInvitationResponse, {

diff --git a/scripts/k6/scenarios/create-proof.js b/scripts/k6/scenarios/create-proof.js
@@ -10,15 +10,12 @@ import {
   getProof,
   getProofIdByThreadId,
   getProofIdCredentials,
+  getWalletIndex,
   sendProofRequest
 } from "../libs/functions.js";
-import { bootstrapIssuer } from "../libs/setup.js";
 
 const vus = Number.parseInt(__ENV.VUS, 10);
 const iterations = Number.parseInt(__ENV.ITERATIONS, 10);
-const issuerPrefix = __ENV.ISSUER_PREFIX;
-const schemaName = __ENV.SCHEMA_NAME;
-const schemaVersion = __ENV.SCHEMA_VERSION;
 
 export const options = {
   scenarios: {
@@ -71,34 +68,15 @@ export function setup() {
   return { bearerToken, tenants };
 }
 
-const iterationsPerVU = options.scenarios.default.iterations;
-// Helper function to calculate the wallet index based on VU and iteration
-function getWalletIndex(vu, iter) {
-  const walletIndex = (vu - 1) * iterationsPerVU + (iter - 1);
-  return walletIndex;
-}
-
-//random number between 0 and 100 (including 0 and 100 as options)
-function getRandomInt() {
-  return Math.floor(Math.random() * 101);
-}
-
 export default function (data) {
-  // const start = Date.now();
   const bearerToken = data.bearerToken;
   const tenants = data.tenants;
-  // const holders = data.holders;
-  const walletIndex = getWalletIndex(__VU, __ITER + 1); // __ITER starts from 0, adding 1 to align with the logic
+  const walletIndex = getWalletIndex(__VU, __ITER);
   const wallet = tenants[walletIndex];
 
-  // const issuerIndex = 0;
-  // const issuer = issuers[issuerIndex];
-
-  // console.log(`isser.accessToken: ${issuer.accessToken}`);
-  // console.log(`issuer.credentialDefinitionId: ${issuer.credentialDefinitionId}`);
   // console.log(`wallet.issuer_connection_id: ${wallet.issuer_connection_id}`);
   // const sendProofRequestResponse = sendProofRequest(issuer.accessToken, wallet.issuer_connection_id);
-  console.log(`VU: ${__VU}, Iteration: ${__ITER}, Issuer Wallet ID: ${wallet.issuer_wallet_id}`);
+  // console.log(`VU: ${__VU}, Iteration: ${__ITER}, Issuer Wallet ID: ${wallet.issuer_wallet_id}`);
   let sendProofRequestResponse;
   try {
     sendProofRequestResponse = sendProofRequest(wallet.issuer_access_token, wallet.issuer_connection_id);

diff --git a/scripts/k6/scenarios/revoke-credentials.js b/scripts/k6/scenarios/revoke-credentials.js
@@ -6,7 +6,8 @@ import { Counter } from "k6/metrics";
 import { getBearerToken } from "../libs/auth.js";
 import {
   checkRevoked,
-  revokeCredentialAutoPublish,
+  getWalletIndex,
+  revokeCredentialAutoPublish
 } from "../libs/functions.js";
 
 const inputFilepath = "../output/create-credentials.json";
@@ -47,16 +48,9 @@ export function setup() {
   return { bearerToken, tenants }; // eslint-disable-line no-eval
 }
 
-const iterationsPerVU = options.scenarios.default.iterations;
-// Helper function to calculate the wallet index based on VU and iteration
-function getWalletIndex(vu, iter) {
-  const walletIndex = (vu - 1) * iterationsPerVU + (iter - 1);
-  return walletIndex;
-}
-
 export default function (data) {
   const tenants = data.tenants;
-  const walletIndex = getWalletIndex(__VU, __ITER + 1); // __ITER starts from 0, adding 1 to align with the logic
+  const walletIndex = getWalletIndex(__VU, __ITER);
   const wallet = tenants[walletIndex];
 
   const revokeCredentialResponse = revokeCredentialAutoPublish(wallet.issuer_access_token, wallet.credential_exchange_id);

diff --git a/scripts/k6/scripts/collection_credential.sh b/scripts/k6/scripts/collection_credential.sh
@@ -9,7 +9,7 @@ config() {
     export ITERATIONS=10
     export HOLDER_PREFIX="k6_holder_credential"
     export ISSUER_PREFIX="k6_issuer_credential"
-    export NUM_ISSUERS=1
+    export NUM_ISSUERS=2
 }
 
 init() {

diff --git a/scripts/k6/scripts/collection_holderops.sh b/scripts/k6/scripts/collection_holderops.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+config() {
+  export VUS=10
+  export ITERATIONS=25
+  export ISSUER_PREFIX="k6_issuer_credential_ops"
+  export HOLDER_PREFIX="k6_holder_credential_ops"
+  export NUM_ISSUERS=2
+}
+
+init() {
+  xk6 run ./scenarios/bootstrap-issuer.js -e ITERATIONS=1 -e VUS=1
+}
+
+scenario_create_holders() {
+  export SLEEP_DURATION=0.75
+  local iterations=$((ITERATIONS * VUS))
+  local vus=1
+  xk6 run ./scenarios/create-holders.js -e ITERATIONS=${iterations} -e VUS=${vus}
+}
+
+senario_create_invitations() {
+  run_test ./scenarios/create-invitations.js
+}
+
+scemario_create_credentials() {
+  run_test ./scenarios/create-credentials.js
+}
+
+scenario_create_proof_verified() {
+  run_test ./scenarios/create-proof.js
+}
+
+scenario_revoke_credentials() {
+  local iterations=$((ITERATIONS * VUS))
+  local vus=1
+  xk6 run ./scenarios/revoke-credentials.js -e ITERATIONS=${iterations} -e VUS=${vus}
+}
+
+scenario_create_proof_unverified() {
+  export IS_REVOKED=true
+  run_test ./scenarios/create-proof.js
+}
+
+cleanup() {
+  log "Cleaning up..."
+  xk6 run ./scenarios/delete-holders.js
+  # xk6 run ./scenarios/delete-issuers.js -e ITERATIONS="${NUM_ISSUERS}" -e VUS=1
+}
+
+run_collection() {
+  local deployments="$1"
+
+  config
+  init
+  run_ha_iterations "${deployments}" scenario_create_holders
+  run_ha_iterations "${deployments}" senario_create_invitations
+  run_ha_iterations "${deployments}" scemario_create_credentials
+  run_ha_iterations "${deployments}" scenario_create_proof_verified
+  run_ha_iterations "${deployments}" scenario_revoke_credentials
+  run_ha_iterations "${deployments}" scenario_create_proof_unverified
+
+  cleanup
+}
diff --git a/scripts/k6/scripts/collection_issuerops.sh b/scripts/k6/scripts/collection_issuerops.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+config() {
+  export VUS=10
+  export ITERATIONS=10
+  export ISSUER_PREFIX="k6_issuer_issuer_ops"
+  export HOLDER_PREFIX="k6_holder_issuer_ops"
+}
+
+init() {
+  log "No init function specified"
+}
+
+scenario_create_issuers() {
+  run_test ./scenarios/create-issuers.js
+}
+
+scenario_create_schemas() {
+  export VUS=10
+  export ITERATIONS=40
+    export SCHEMA_PREFIX="k6_issuer_ops2"
+  run_test ./scenarios/create-schemas.js
+}
+
+cleanup() {
+  log "Cleaning up..."
+  xk6 run ./scenarios/delete-issuers.js
+}
+
+run_collection() {
+  local deployments="$1"
+
+  config
+  init
+  run_ha_iterations "${deployments}" scenario_create_issuers
+  run_ha_iterations "${deployments}" scenario_create_schemas
+
+  cleanup
+}
diff --git a/scripts/k6/scripts/common.sh b/scripts/k6/scripts/common.sh
@@ -52,7 +52,7 @@ run_ha_iterations() {
     local scenario_pid=$!
 
     if [[ -n "${deployments}" ]]; then
-      for ((j=1; j<=2; j++)); do
+      for ((j=1; j<=RESTART_ITERATIONS; j++)); do
         local deployment_pids=()
         for deployment in ${deployments}; do
           restart_deployment "${deployment}" &
@@ -67,6 +67,13 @@ run_ha_iterations() {
           fi
         done
       done
+
+        # Check if scenario is still running after deployments restart
+        if kill -0 "${scenario_pid}" 2>/dev/null; then
+            log "Scenario is still running after all deployments were restarted"
+        else
+            wrn "WARNING: Scenario completed too quickly, before all deployments were restarted"
+        fi
     else
       log "No stack specified. Skipping restarts."
     fi

diff --git a/scripts/k6/scripts/config.sh b/scripts/k6/scripts/config.sh
@@ -13,6 +13,7 @@ export SERVICE="governance-endorser tails-server governance-trust-registry waypo
 export AUTH="inquisitor"
 export INVALID="governance-webhooks-web"
 export HA_TEST_ITERATIONS=1 # Configurable number of HA test iterations
+export RESTART_ITERATIONS=1 # Configurable number of restart iterations
 export NUM_ISSUERS=1
 
 # Combine all stacks into one variable