Feat/benchmarks

.gitignore

@@ -2,6 +2,11 @@
 **/*.log
 test/repo-tests*
 
+.DS_Store
+
+# Benchmarks
+benchmarks/reports/out/*
+
 # Logs
 logs
 *.log

benchmarks/aggregate.js

@@ -0,0 +1,131 @@
+'use strict'
+
+const join = require('path').join
+const fs = require('fs')
+
+const reportDir = join(__dirname, 'reports', 'out')
+
+const OK_ERRORS = ['ENOENT', 'ENOTDIR']
+
+const reports = fs.readdirSync(reportDir)
+  .map((report) => {
+    const path = join(reportDir, report, 'results.json')
+    try {
+      return {
+        path: join('out', report, 'report.html'),
+        content: JSON.parse(fs.readFileSync(path, { encoding: 'utf8' }))
+      }
+    } catch(err) {
+      if (OK_ERRORS.indexOf(err.code) > -1) {
+        return null
+      }
+      throw err
+    }
+  })
+  .filter(Boolean)
+
+// console.log('reports.length:', reports.length)
+// console.log('reports:', JSON.stringify(reports, null, '  '))
+
+const suites = {}
+reports.forEach((report) => {
+  // console.log('report.length:', report.length)
+  report.content.forEach((s) => {
+    let suite = suites[s.name]
+    if (!suite) {
+      suite = suites[s.name] = {
+        name: s.name,
+        benchmarks: {}
+      }
+    }
+    s.benchmarks.forEach((b) => {
+      let benchmark = suite.benchmarks[b.name]
+      if (!benchmark) {
+        benchmark = suite.benchmarks[b.name] = {
+          name: b.name,
+          runs: []
+        }
+      }
+      // console.log('adding run')
+      benchmark.runs.push({
+        when: b.now,
+        path: report.path,
+        whenPretty: new Date(b.now).toISOString(),
+        count: b.count,
+        hz: b.hz,
+        stats: {
+          moe: b.stats.moe,
+          rme: b.stats.rme,
+          sem: b.stats.sem,
+          deviation: b.stats.deviation,
+          mean: b.stats.mean,
+          variance: b.stats.variance
+        }
+      })
+    })
+  })
+})
+
+const results = Object.keys(suites).map(sn => suites[sn]).map((suite) => {
+  const benchmarks =
+    Object.keys(suite.benchmarks)
+      .map(bn => suite.benchmarks[bn])
+      .map((bm) => {
+        return {
+          name: bm.name,
+          runs: bm.runs.sort(sortBenchmarkRuns),
+          colors: {
+            mean: '#3772FF',
+            variance: '#EF709D',
+            deviation: '#E2EF70',
+            MOE: '#CA3CFF',
+            RME: '#70E4EF',
+            SEM: '#DEF6CA',
+            count: '#F8BDC4',
+            hz: '#F497DA'
+          },
+          metric: bm.runs.sort(sortBenchmarkRuns).reduce(reduceBenchmarkRuns, {
+            mean: [],
+            variance: [],
+            deviation: [],
+            MOE: [],
+            RME: [],
+            SEM: [],
+            count: [],
+            hz: []
+          })
+        }
+      })
+  // const runs = benchmarks.runs.forEach()
+  const s = {
+    suite: suite.name,
+    benchmarks: benchmarks
+  }
+
+  return s
+})
+
+const viewPath = join(__dirname, 'reports', 'views.json')
+
+process.stdout.write(JSON.stringify(results, null, '  '))
+
+function sortBenchmarkRuns (a, b) {
+  return a.when - b.when
+}
+
+function reduceBenchmarkRuns (acc, run) {
+  acc.count.push({ when: run.when, value: run.count })
+  acc.hz.push({ when: run.when, value: run.hz })
+  acc.MOE.push({ when: run.when, value: run.stats.moe })
+  acc.RME.push({ when: run.when, value: run.stats.rme })
+  acc.SEM.push({ when: run.when, value: run.stats.sem })
+  acc.deviation.push({ when: run.when, value: run.stats.deviation })
+  acc.mean.push({ when: run.when, value: run.stats.mean })
+  acc.variance.push({ when: run.when, value: run.stats.variance })
+
+  return acc
+}
+
+function generateColor () {
+
+}

benchmarks/generate-aggregate-report.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const concat = require('concat-stream')
+const handlebars = require('handlebars')
+const fs = require('fs')
+const join = require('path').join
+const open = require('opn')
+
+const template = handlebars.compile(
+  fs.readFileSync(join(__dirname, 'report-templates', 'aggregate.html'), 'utf8'))
+
+process.stdin.pipe(concat(gotResult))
+
+function gotResult(str) {
+  outputSuites(JSON.parse(str))
+}
+
+function outputSuites(suites) {
+  const html = template({ suites: suites })
+  const outPath = join(__dirname, 'reports', 'aggregate.html')
+  fs.writeFileSync(outPath, html)
+  // console.log(html)
+  // return;
+
+  console.log('Aggregate report written to %s', outPath)
+  open(outPath, { wait: false })
+}

benchmarks/generate-report.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const concat = require('concat-stream')
+const handlebars = require('handlebars')
+const fs = require('fs')
+const join = require('path').join
+const template = handlebars.compile(
+  fs.readFileSync(join(__dirname, 'report-templates', 'results.html'), 'utf8'))
+
+process.stdin.pipe(concat(gotResult))
+
+function gotResult(str) {
+  const result = JSON.parse(str)
+  outputReport(result)
+}
+
+function outputReport(result) {
+  const html = template({ results: result })
+  process.stdout.write(html)
+}

benchmarks/index.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const runner = require('./runner')
+
+let suites = process.argv.slice(2)
+
+if (!suites.length) {
+  suites = undefined
+}
+
+runner.run(suites, (err, results) => {
+  if (err) {
+    throw err
+  }
+
+  console.log(JSON.stringify(results, null, '  '))
+})

benchmarks/profile.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const profiler = require('./profiler')
+const argv = require('yargs').argv
+
+const suites = argv._
+const outDir = argv.out
+if (!outDir) {
+  throw new Error('please provide an out dir')
+}
+
+profiler(outDir, suites, (err, resultLink) => {
+  if (err) {
+    throw err
+  }
+  process.stdout.write(resultLink)
+})

benchmarks/profiler.js

@@ -0,0 +1,43 @@
+'use strict'
+
+const spawn = require('child_process').spawn
+const split = require('split')
+const url = require('url')
+const path = require('path')
+
+module.exports = profile
+
+function profile (outDir, suites, callback) {
+  const args = ['--output-dir', outDir].concat([process.argv[0], __dirname].concat(suites))
+
+  const options = {
+    // stdio: 'pipe'
+  }
+  const child = spawn('0x', args, options)
+  let lastLine = ''
+  let error
+  let errorOut = ''
+
+  child.stderr.pipe(split())
+    .on('data', (line) => {
+      if (line) {
+        lastLine = line
+        errorOut += line + '\n'
+      }
+    })
+    .once('end', () => {
+      if (!error) {
+        const matched = lastLine.match(/file:\/\/.*\/flamegraph.html/)
+        if (matched) {
+          const resultPath = url.parse(matched[0].replace('file://', 'file:///')).pathname
+          callback(null, path.relative(outDir, resultPath))
+        } else {
+          callback(new Error('0x unexpected output:\n' + errorOut))
+        }
+      }
+    })
+    .once('error', (err) => {
+      error = err
+      callback(err)
+    })
+}

benchmarks/report-templates/aggregate.html

@@ -0,0 +1,104 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>IPFS Benchmarks Aggregate Repor</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href='css/bootstrap.css' rel='stylesheet' type='text/css'>
+    <script src="js/Chart.bundle.min.js"></script>
+  </head>
+  <body class="container">
+    <h1 class="title">IPFS Benchmarks Views</h1>
+
+    <h1>Index</h1>
+    <ul>
+      {{#each suites}}
+        <li><a href="#{{suite}}">{{suite}}</a>
+          <ul>
+            {{#each benchmarks}}
+              <li><a href="#{{../suite}}/{{name}}">{{name}}</a></li>
+            {{/each}}
+          </ul>
+        </li>
+      {{/each}}
+    </ul>
+
+    <h1>Suites</h1>
+
+    <section>
+      {{#each suites}}
+        <h2><a id="{{suite}}"></a>{{suite}}</h2>
+        {{#each benchmarks}}
+          <div class="clearfix">
+            <h3><a id="{{../suite}}/{{name}}"></a>{{../suite}}/{{name}}</h3>
+            <table class="table table-bordered">
+              <thead>
+                <tr>
+                  <th>When</th>
+                  <th>Mean</th>
+                  <th>Variance</th>
+                  <th>Deviation</th>
+                  <th>Count</th>
+                  <th>Hz</th>
+                  <th>MOE</th>
+                  <th>RME</th>
+                  <th>SEM</th>
+                </tr>
+              </thead>
+              <tbody>
+                {{#each runs}}
+                  <tr>
+                    <td>{{whenPretty}} <a href="{{path}}">(detail)</a></td>
+                    <td><b>{{stats.mean}}</b></td>
+                    <td>{{stats.variance}}</td>
+                    <td>{{stats.deviation}}</td>
+                    <td>{{count}}</td>
+                    <td>{{hz}}</td>
+                    <td>{{stats.moe}}</td>
+                    <td>{{stats.rme}}</td>
+                    <td>{{stats.sem}}</td>
+                  </tr>
+                {{/each}}
+              </tbody>
+            </table>
+
+            {{#each metric}}
+              <div style="float: left; width: 350px">
+                <span>{{@key}}</span>
+                <canvas height="350" id="{{../../suite}}/{{../name}}/{{@key}}"></canvas>
+                <script>
+                  var canvas = document.getElementById('{{../../suite}}/{{../name}}/{{@key}}')
+                  var options = {
+                    type: 'line',
+                    data: {
+                      datasets: [{
+                        label: '{{@key}}',
+                        data: [
+                          {{#each this}}
+                            {
+                              x: {{when}},
+                              y: {{value}}
+                            }{{#unless @last}},{{/unless}}
+                          {{/each}}
+                        ],
+                        backgroundColor: '{{lookup ../colors @key}}'
+                      }]
+                    },
+                    options: {
+                      scales: {
+                        xAxes: [{
+                          type: 'time',
+                          position: 'bottom'
+                        }]
+                      }
+                    }
+                  }
+                  var chart = new Chart(canvas, options)
+                </script>
+              </div>
+            {{/each}}
+          </div>
+        {{/each}}
+      {{/each}}
+    </section>
+  </body>
+</html>