ci: benchmark

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2.0
+version: 2.1
 
 aliases:
   make_out_dirs: &make_out_dirs
@@ -17,6 +17,30 @@ aliases:
           - ~/go/pkg/mod
           - ~/.cache/go-build/
 
+commands:
+  benchmark:
+    parameters:
+      output:
+        type: string
+      branch:
+        type: string
+        default: HEAD
+    steps:
+      - checkout
+      - run: git checkout << parameters.branch >>
+      - *make_out_dirs
+      - *restore_gomod
+      - run:
+          command: go test -tags nofuse -run=NONE -bench=. ./... | tee /tmp/circleci-workspace/<< parameters.output >>
+          environment:
+            IPFS_LOGGING: critical
+            CGO_ENABLED: 0
+      - persist_to_workspace:
+          root: /tmp/circleci-workspace
+          paths:
+            - << parameters.output >>
+      - *store_gomod
+
 
 defaults: &defaults
   working_directory: ~/ipfs/go-ipfs
@@ -141,6 +165,40 @@ jobs:
           paths:
             - bin/ipfs
       - *store_gomod
+  benchmark-release:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - benchmark:
+          output: benchmark-release.txt
+          branch: release
+  benchmark-before:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - benchmark:
+          output: benchmark-before.txt
+          branch: master
+  benchmark-after:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - benchmark:
+          output: benchmark-after.txt
+  benchmark-compare:
+    docker:
+      - image: circleci/golang:1.12
+    <<: *defaults
+    steps:
+      - checkout
+      - *make_out_dirs
+      - attach_workspace:
+          at: /tmp/circleci-workspace
+      - run: bin/diff-benchmarks /tmp/circleci-workspace/benchmark-before.txt /tmp/circleci-workspace/benchmark-after.txt
+      - run: bin/diff-benchmarks /tmp/circleci-workspace/benchmark-release.txt /tmp/circleci-workspace/benchmark-after.txt
   interop:
     docker:
       - image: circleci/node:10
@@ -249,3 +307,11 @@ workflows:
     - go-ipfs-http-client:
         requires:
           - build
+    - benchmark-before
+    - benchmark-after
+    - benchmark-release
+    - benchmark-compare:
+        requires:
+          - benchmark-after
+          - benchmark-before
+          - benchmark-release

bin/diff-benchmarks

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+before="$1"
+after="$2"
+
+parse() {
+    sed -n \
+        -e 's/ *\t */\t/g' \
+        -e '/^Benchmark/p' |
+        column -s'	' --json \
+               --table-columns name,count,time,rate \
+               --table-name "results" |
+        jq '.results[] | {name: .name, time: (.time | rtrimstr(" ns/op") | tonumber)}'
+}
+
+benchcmp "$1" "$2"
+
+echo ""
+echo "Result:"
+
+{
+    parse < "$1"
+    parse < "$2"
+} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[1].time / .[0].time)} | select(.speedup < 0.95) | "\(.name)\t\(.speedup)x"'
+
+if [[ $? -ne 4 ]]; then
+    echo ""
+    echo "FAIL"
+    exit 1
+else
+    echo "PASS"
+fi

test/integration/bench_test.go

@@ -1,23 +1,25 @@
 package integrationtest
 
 import (
+	"runtime"
 	"testing"
 
 	"github.com/ipfs/go-ipfs/thirdparty/unit"
 	testutil "github.com/libp2p/go-libp2p-testing/net"
 )
 
 func benchmarkAddCat(numBytes int64, conf testutil.LatencyConfig, b *testing.B) {
-
-	b.StopTimer()
-	b.SetBytes(numBytes)
 	data := RandomBytes(numBytes) // we don't want to measure the time it takes to generate this data
-	b.StartTimer()
 
+	b.SetBytes(numBytes)
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
 		if err := DirectAddCat(data, conf); err != nil {
 			b.Fatal(err)
 		}
+		b.StopTimer()
+		runtime.GC()
+		b.StartTimer()
 	}
 }