Add Performance Tests (#421)

* Have a working dockerfile to run perf tests and report the times they take. We can also capture stdout/stderr with it for further information, especially for tools that report internal latencies.

* Slight changes to the perf test script, a perf.yml for the github action

.github/workflows/perf.yml

@@ -0,0 +1,26 @@
+name: DiskANN Nightly Performance Metrics
+on:
+  schedule:
+    - cron: "41 14 * * *"  # 14:41 UTC, 7:41 PDT, 8:41 PST, 08:11 IST
+jobs:
+  perf-test:
+    name: Run Perf Test from main
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Build Perf Container
+        run: |
+          docker build --build-arg GIT_COMMIT_ISH="$GITHUB_SHA" -t perf -f scripts/perf/Dockerfile scripts
+      - name: Performance Tests
+        run: |
+          mkdir metrics
+          docker run -v ./metrics:/app/logs perf &> ./metrics/combined_stdouterr.log
+      - name: Upload Metrics Logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: metrics
+          path: |
+            ./metrics/**

scripts/dev/install-dev-deps-ubuntu.bash

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-apt install cmake \
+DEBIAN_FRONTEND=noninteractive apt install -y cmake \
   g++ \
   libaio-dev \
   libgoogle-perftools-dev \

scripts/perf/Dockerfile

@@ -0,0 +1,31 @@
+#Copyright(c) Microsoft Corporation.All rights reserved.
+#Licensed under the MIT license.
+
+FROM ubuntu:jammy
+
+# Can be provided at build to point to a specific commit-ish, by default builds from HEAD
+ARG GIT_COMMIT_ISH=HEAD
+
+RUN apt update
+RUN apt install -y software-properties-common
+RUN add-apt-repository -y ppa:git-core/ppa
+RUN apt update
+RUN DEBIAN_FRONTEND=noninteractive apt install -y git time
+
+COPY dev/install-dev-deps-ubuntu.bash /app/fallback/install-dev-deps-ubuntu.bash
+WORKDIR /app
+RUN git clone https://github.com/microsoft/DiskANN.git
+WORKDIR /app/DiskANN
+RUN git checkout $GIT_COMMIT_ISH
+
+# we would prefer to use the deps requested at the same commit. if the script doesn't exist we'll use the current one.
+RUN bash scripts/dev/install-dev-deps-ubuntu.bash || bash /app/fallback/install-dev-deps-ubuntu.bash
+
+RUN mkdir build
+RUN cmake -S . -B build  -DCMAKE_BUILD_TYPE=Release -DUNIT_TEST=True
+RUN cmake --build build -- -j
+
+RUN mkdir /app/logs
+COPY perf/perf_test.sh /app/DiskANN/perf_test.sh
+
+ENTRYPOINT bash perf_test.sh

scripts/perf/README.md

@@ -0,0 +1,20 @@
+#Performance Tests
+
+The bash scripts in this folder are responsible for running a suite of performance 
+tests.
+
+The timing and recall metrics reported by these tests when run periodically can then
+be used to identify performance improvements or regressions as 
+development continues.
+
+## Usage
+
+`docker build` must be run with the context directory set to `scripts`, but the Dockerfile set to `scripts/perf/Dockerfile` as in:
+```bash
+docker build [--build-arg GIT_COMMIT_ISH=<rev>] -f scripts/perf/Dockerfile scripts
+```
+
+We prefer to install the dependencies from the commit-ish that we're building against, but as the deps were not stored 
+in a known file in all commits, we will fall back to the one currently in HEAD if one is not found already.
+
+The `--build-arg GIT_COMMIT_ISH=<rev>` is optional, with a default value of HEAD if not otherwise specified.

scripts/perf/perf_test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+function json_time {
+  command="$@"
+  echo "Executing $command"
+  /usr/bin/time --quiet -o /app/logs/time.log -a --format '{"command":"%C", "wallclock": %e, "user": %U, "sys": %S}' $command
+  ret=$?
+  if [ $ret -ne 0 ]; then
+    echo "{\"command\": \""$command"\", \"status_code\": $ret}" >> /app/logs/time.log
+  fi
+}
+
+mkdir data
+rm /app/logs/time.log
+touch /app/logs/time.log
+chmod 666 /app/logs/time.log
+
+if [ -d "build/apps" ]; then
+	export BASE_PATH="build/apps"
+else
+	export BASE_PATH="build/tests"
+fi
+
+json_time $BASE_PATH/utils/rand_data_gen --data_type float --output_file data/rand_float_768D_1M_norm1.0.bin -D 768 -N 1000000 --norm 1.0
+json_time $BASE_PATH/utils/rand_data_gen --data_type float --output_file data/rand_float_768D_10K_norm1.0.bin -D 768 -N 10000 --norm 1.0
+
+json_time $BASE_PATH/utils/compute_groundtruth  --data_type float --dist_fn l2 --base_file data/rand_float_768D_1M_norm1.0.bin --query_file data/rand_float_768D_10K_norm1.0.bin --gt_file data/l2_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 --K 100
+json_time $BASE_PATH/utils/compute_groundtruth  --data_type float --dist_fn mips --base_file data/rand_float_768D_1M_norm1.0.bin --query_file data/rand_float_768D_10K_norm1.0.bin --gt_file data/mips_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 --K 100
+json_time $BASE_PATH/utils/compute_groundtruth  --data_type float --dist_fn cosine --base_file data/rand_float_768D_1M_norm1.0.bin --query_file data/rand_float_768D_10K_norm1.0.bin --gt_file data/cosine_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 --K 100
+
+json_time $BASE_PATH/build_memory_index --data_type float --dist_fn l2 --data_path data/rand_float_768D_1M_norm1.0.bin --index_path_prefix data/index_l2_rand_float_768D_1M_norm1.0
+json_time $BASE_PATH/search_memory_index --data_type float --dist_fn l2 --index_path_prefix data/index_l2_rand_float_768D_1M_norm1.0 --query_file data/rand_float_768D_10K_norm1.0.bin --recall_at 10 --result_path temp --gt_file data/l2_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 -L 100 32
+json_time $BASE_PATH/search_memory_index --data_type float --dist_fn fast_l2 --index_path_prefix data/index_l2_rand_float_768D_1M_norm1.0 --query_file data/rand_float_768D_10K_norm1.0.bin --recall_at 10 --result_path temp --gt_file data/l2_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 -L 100 32
+
+json_time $BASE_PATH/build_memory_index --data_type float --dist_fn mips --data_path data/rand_float_768D_1M_norm1.0.bin --index_path_prefix data/index_mips_rand_float_768D_1M_norm1.0
+json_time $BASE_PATH/search_memory_index --data_type float --dist_fn mips --index_path_prefix data/index_l2_rand_float_768D_1M_norm1.0 --query_file data/rand_float_768D_10K_norm1.0.bin --recall_at 10 --result_path temp --gt_file data/mips_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 -L 100 32
+
+json_time $BASE_PATH/build_memory_index --data_type float --dist_fn cosine --data_path data/rand_float_768D_1M_norm1.0.bin --index_path_prefix data/index_cosine_rand_float_768D_1M_norm1.0
+json_time $BASE_PATH/search_memory_index --data_type float --dist_fn cosine --index_path_prefix data/index_l2_rand_float_768D_1M_norm1.0 --query_file data/rand_float_768D_10K_norm1.0.bin --recall_at 10 --result_path temp --gt_file data/cosine_rand_float_768D_1M_norm1.0_768D_10K_norm1.0_gt100 -L 100 32
+