initial benchmark test case and flow. (#2)

.github/workflows/benchmark-test.yml

@@ -0,0 +1,53 @@
+name: Run Benchmark Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      graph:
+        description: 'Graph'
+        required: true
+      user-count:
+        description: 'Number of users'
+        required: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Prepare input graph
+        id: prepare-graph
+        run: echo "::set-output name=graph::${{ github.event.inputs.graph }}"
+
+      - name: Run tests
+        run: |
+          yarn test:flow
+        env:
+          INPUT_GRAPH: ${{ steps.prepare-graph.outputs.graph }}
+          INPUT_USER_COUNT: ${{ github.event.inputs.user-count }}
+          FUSIONAUTH_BASE_URL: ${{ secrets.FUSIONAUTH_BASE_URL }}
+          FUSIONAUTH_API_KEY: ${{ secrets.FUSIONAUTH_API_KEY }}
+          SOCKET_URL: ${{ secrets.SOCKET_URL }}
+          AI_TOOLS_BASE_URL: ${{ secrets.AI_TOOLS_BASE_URL }}
+          AI_TOOLS_AUTH_HEADER: ${{ secrets.AI_TOOLS_AUTH_HEADER }}
+          APPLICATION_ID: ${{ secrets.APPLICATION_ID }}
+          REQUEST_TIMEOUT_DURATION: ${{ secrets.REQUEST_TIMEOUT_DURATION }}
+          HASURA_SECRET: ${{ secrets.HASURA_SECRET }}
+          HASURA_BASE_URL: ${{ secrets.HASURA_BASE_URL }}
+
+      - name: Upload test result
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-result
+          path: testResult.json

package.json

@@ -16,12 +16,22 @@
     "babel-loader": "8.2.2",
     "clean-webpack-plugin": "4.0.0-alpha.0",
     "copy-webpack-plugin": "^9.0.1",
+    "ts-node": "10.9.1",
     "typescript": "4.2.4",
     "webpack": "5.35.1",
     "webpack-cli": "4.6.0",
     "webpack-glob-entries": "^1.0.1"
   },
   "scripts": {
-    "start": "webpack"
+    "start": "webpack",
+    "test:flow": "ts-node src/e2e/flow.ts",
+    "test:socket-connections": "ts-node src/e2e/socket-connections.ts"
+  },
+  "dependencies": {
+    "axios": "^1.4.0",
+    "dotenv-webpack": "^8.0.1",
+    "ml-matrix": "^6.10.4",
+    "socket.io-client": "^4.6.1",
+    "uuid": "^9.0.0"
   }
 }

src/e2e/flow.ts

@@ -0,0 +1,11 @@
+import { testRunner } from "./testRunner";
+import dotenv from 'dotenv';
+dotenv.config();
+
+(async ()=>{
+    console.log(process.env.INPUT_GRAPH,process.env.INPUT_USER_COUNT)
+    testRunner(
+        process.env.INPUT_GRAPH || '',
+        parseInt(process.env.INPUT_USER_COUNT || '10')
+    )
+})()

src/e2e/graph.ts

@@ -0,0 +1,165 @@
+import UserSocket from "./socket";
+//@ts-ignore
+import { v4 as uuid } from 'uuid';
+import { Matrix } from 'ml-matrix';
+import { getEmbedding } from "./utils/ai-tools";
+import { logger } from "./utils/logger";
+
+type TreeNode = string;
+type WeightedEdge = [TreeNode, TreeNode, number];
+
+interface Graph {
+    [key: string]: { [key: string]: number };
+}
+
+class Parser {
+    private diagram: string;
+    public firstNode: string;
+
+    constructor(diagram: string) {
+        this.diagram = diagram;
+        this.firstNode = this.diagram.split("     ")[1].trim()
+    }
+
+    private parse(): WeightedEdge[] {
+        const edges = this.diagram.split("     ").filter(line => line.includes('--'));
+        let ret_edges: WeightedEdge[] =  edges.map(edge => {
+            const [from, toWithWeight] = edge.split('-->');
+            const [_, weight, to] = toWithWeight.split('|');
+
+            return [from.trim(), to.trim(), Number(weight) / 100];
+        });
+        return  ret_edges
+    }
+
+    public toGraph(): Graph {
+        const graph: Graph = {};
+        const edges = this.parse();
+        for (const [from, to, weight] of edges) {
+            if (!(from in graph)) {
+                graph[from] = {};
+            }
+
+            graph[from][to] = weight;
+        }
+
+        return graph;
+    }
+}
+
+class Walker {
+    private graph: Graph;
+    private socket: UserSocket;
+    private conversationId: string;
+
+
+    constructor(graph: Graph, socket: UserSocket) {
+        this.graph = graph;
+        this.socket = socket
+        this.conversationId = uuid()
+    }
+
+    public async walk(start: TreeNode): Promise<any> {
+        let result: any = {pathTaken:[]}
+        let [currentNode, messageFull] = start.split("[");
+        let [message, expectedMessage] = messageFull.split(":");
+        expectedMessage = expectedMessage.replace("]", "");
+        const pathTaken: string[] = []
+        while (true) {
+            pathTaken.push(currentNode)
+            logger.logProcess(`User${this.socket.index}`,`: Path taken - ${pathTaken}`)
+            message = message.trim();
+            expectedMessage = expectedMessage.trim();
+            let messageResult: any = {
+                node: currentNode,
+                query: message,
+                expectedResponse: expectedMessage
+            }
+            const startTime = new Date().getTime()
+
+            const reply = await Promise.race([
+                this.socket.emitEvent("botRequest", {
+                    content: {
+                        text: message,
+                        userId: this.socket.deviceId,
+                        appId: "AKAI_App_Id",
+                        channel: "AKAI",
+                        from: this.socket.deviceId,
+                        context: null,
+                        accessToken: null,
+                        conversationId: this.conversationId
+                    },
+                    to: this.socket.deviceId, 
+                    conversationId: this.conversationId
+                }),
+                new Promise((resolve, _) => {
+                    setTimeout(() => {
+                        resolve({
+                            content:{
+                                title: `No response received in ${parseInt(process.env.REQUEST_TIMEOUT_DURATION || '60000')/1000} sec`
+                            }
+                        });
+                    }, parseInt(process.env.REQUEST_TIMEOUT_DURATION || '60000'));
+                })
+            ]);
+            messageResult['timeTaken'] = new Date().getTime() - startTime
+            messageResult['receivedResponse'] = reply.content.title
+            messageResult['similarity'] =  await this.checkSimilarity(reply.content.title, message)
+            messageResult['similarityResult'] = parseFloat(messageResult['similarity']) > 0.97 ? "Positive": "Negative"
+            result.pathTaken.push(messageResult)
+            const neighbors = this.graph[currentNode];
+            if (!neighbors) {
+                return result
+            }
+            const neighborsArr = Object.keys(neighbors);
+            const weights = neighborsArr.map(neighbor => neighbors[neighbor]);
+            let choice = this.weightedRandom(neighborsArr, weights);
+            [currentNode, messageFull] = choice.split("[");
+            [message, expectedMessage] = messageFull.split(":");
+            expectedMessage = expectedMessage.replace("]", "");
+        }
+    }
+
+    private weightedRandom(options: string[], weights: number[]): string {
+        let i, pickedIndex,
+            totalWeight = weights.reduce((prev, curr) => prev + curr, 0);
+
+        let randomNum = Math.random() * totalWeight;
+
+        for (i = 0; i < options.length; i++) {
+            randomNum -= weights[i];
+
+            if (randomNum < 0) {
+                pickedIndex = i;
+                break;
+            }
+        }
+
+        let pickedNode = options[pickedIndex || 0];
+        return pickedNode
+    }
+
+    private async checkSimilarity(receivedResponse: string, expectedResponse: string): Promise<any> {
+        function cosineSimilarity(vector1: number[], vector2: number[]): number {
+          const matrix1 = Matrix.columnVector(vector1);
+          const matrix2 = Matrix.columnVector(vector2);
+
+          const dotProduct = matrix1.transpose().mmul(matrix2).get(0, 0);
+          const norm1 = matrix1.norm("frobenius");
+          const norm2 = matrix2.norm("frobenius");
+
+          return dotProduct / (norm1 * norm2);
+        }
+
+        const vector1 = await getEmbedding(receivedResponse)
+        const vector2 = await getEmbedding(expectedResponse)
+
+        const similarity = cosineSimilarity(vector1[0].embedding, vector2[0].embedding);
+        return similarity;
+      }
+
+}
+
+export default { Walker, Parser }
+
+

src/e2e/socket-connections.ts

@@ -0,0 +1,22 @@
+import dotenv from 'dotenv';
+dotenv.config();
+import UserSocket from "./socket";
+import { createUser } from "./utils/user";
+
+(async () => {
+    let userCount = 100
+    let sockets = []
+    for(let i=1;i<=userCount;i++){
+        let data = await createUser(`tester${i}@amakrushi.ai`,`tester-amakrushi-${i}`, i)
+        if(!data) {console.error('unable to create user', `tester${i}@amakrushi.ai`); continue;}
+        const userSocket = new UserSocket(data.token,data.user.id, i)
+        userSocket.connect()
+        userSocket.mockEvent("mock","mock message")
+        sockets.push(userSocket)
+        console.log("connections =",i)
+    }
+    for(let i=0;i<sockets.length;i++){
+        let userSocket = sockets[i]
+        await userSocket.close()
+    }
+})();

src/e2e/socket.ts

@@ -0,0 +1,103 @@
+import { Socket, io } from 'socket.io-client';
+import {logger} from '../e2e/utils/logger'
+import { deleteUser, deleteUserQueries } from './utils/user';
+import * as fs from 'fs';
+
+class UserSocket {
+  private socket: Socket;
+  public deviceId: string;
+  public index: number;
+  public logger: any;
+
+  constructor(apiKey: string, deviceId: string, index: number) {
+    this.index = index
+    this.deviceId = deviceId
+    const socketOptions = {
+      transportOptions: {
+        polling: {
+          extraHeaders: {
+            Authorization: `Bearer ${apiKey}`,
+            channel: 'akai',
+          },
+        },
+      },
+      query: {
+        deviceId: deviceId,
+      },
+      autoConnect: false,
+      upgrade: false
+    };
+    this.socket = io(process.env.SOCKET_URL || '', socketOptions);
+    this.socket.on('connect', () => {
+      logger.logProcess(`User${this.index}`,`: Connected to WebSocket server`);
+    });
+    this.socket.on('disconnect', (reason) => {
+      logger.logProcess(`User${this.index}`,`: ${this.deviceId} - Disconnected from WebSocket server: ${reason}`);
+    });
+    this.socket.on('error', (error) => {
+      logger.logProcess(`User${this.index}`,`: WebSocket error: ${error}`);
+    });
+  }
+
+  connect(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      this.socket.connect();
+      this.socket.on('connect', () => {
+        resolve();
+      });
+    });
+  }
+
+  disconnect(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      this.socket.disconnect();
+      this.socket.on('disconnect', () => {
+        resolve();
+      });
+    });
+  }
+
+  emitEvent(event: string, data: any): Promise<any> {
+    return new Promise<any>((resolve) => {
+      this.socket.emit(event, data);
+      this.socket.once('botResponse', (response: any) => {
+        resolve(response);
+      });
+    });
+  }
+
+  mockEvent(event: string, data: any): Promise<any> {
+    return new Promise<any>((resolve) => {
+      this.socket.emit(event, data);
+      resolve("mock response");
+    });
+  }
+
+  async close(finalResult: any = null): Promise<void> {
+    if(finalResult) {
+      let additionalParams: any = {
+        totalTimeTaken: finalResult.userFlows.reduce((sum: number, path: any) => sum + path.totalTimeTaken, 0),
+        totalPositiveResponse: finalResult.userFlows.reduce((sum: number, path: any) => sum + path.totalPositveResponse, 0),
+        totalNegativeResponse: finalResult.userFlows.reduce((sum: number, path: any) => sum + path.totalNegativeResponse, 0)
+      }
+      additionalParams['averageTimeTaken'] = additionalParams['totalTimeTaken'] / finalResult.userFlows.length;
+      additionalParams['totalTimeTaken'] = `${additionalParams['totalTimeTaken']/1000} sec`
+      additionalParams['averageTimeTaken'] = `${additionalParams['averageTimeTaken']/1000} sec`
+      finalResult = Object.assign(additionalParams, finalResult);
+      const jsonData = JSON.stringify(finalResult, null, 2);
+      const filePath = './testResult.json';
+      fs.writeFile(filePath, jsonData, (err) => {
+          if (err) {
+          console.error('Error writing JSON data:', err);
+          return;
+          }
+          console.log('Result written to file:', filePath);
+      });
+    }
+    await deleteUser(this.deviceId)
+    await deleteUserQueries(this.deviceId)
+    this.socket.close();
+  }
+}
+
+export default UserSocket;