From f61c71b53391b0ecc1b14a6e20b1fae18a4168cf Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 16 Jan 2017 02:59:13 +0800 Subject: [PATCH] benchmark: add progress indicator to compare.js * Print the progress bar and the current benchmark to stderr when stderr is TTY and stdout is not. * Allow cli arguments without values via setting.boolArgs * Add --no-progress option PR-URL: https://github.com/nodejs/node/pull/10823 Fixes: https://github.com/nodejs/node/issues/8659 Reviewed-By: Andreas Madsen --- benchmark/_benchmark_progress.js | 120 +++++++++++++++++++++++++++++++ benchmark/_cli.js | 10 +-- benchmark/common.js | 11 ++- benchmark/compare.js | 63 +++++++++++----- benchmark/run.js | 3 + benchmark/scatter.js | 4 ++ 6 files changed, 187 insertions(+), 24 deletions(-) create mode 100644 benchmark/_benchmark_progress.js diff --git a/benchmark/_benchmark_progress.js b/benchmark/_benchmark_progress.js new file mode 100644 index 00000000000000..2a2a458c5c4e4b --- /dev/null +++ b/benchmark/_benchmark_progress.js @@ -0,0 +1,120 @@ +'use strict'; + +const readline = require('readline'); + +function pad(input, minLength, fill) { + var result = input + ''; + return fill.repeat(Math.max(0, minLength - result.length)) + result; +} + +function fraction(numerator, denominator) { + const fdenominator = denominator + ''; + const fnumerator = pad(numerator, fdenominator.length, ' '); + return `${fnumerator}/${fdenominator}`; +} + +function getTime(diff) { + const time = Math.ceil(diff[0] + diff[1] / 1e9); + const seconds = pad(time % 60, 2, '0'); + const minutes = pad(Math.floor(time / 60) % (60 * 60), 2, '0'); + const hours = pad(Math.floor(time / (60 * 60)), 2, '0'); + return `${hours}:${minutes}:${seconds}`; +} + +// A run is an item in the job queue: { binary, filename, iter } +// A config is an item in the subqueue: { binary, filename, iter, configs } +class BenchmarkProgress { + constructor(queue, benchmarks) { + this.queue = queue; // Scheduled runs. + this.benchmarks = benchmarks; // Filenames of scheduled benchmarks. + this.completedRuns = 0; // Number of completed runs. + this.scheduledRuns = queue.length; // Number of scheduled runs. + // Time when starting to run benchmarks. + this.startTime = process.hrtime(); + // Number of times each file will be run (roughly). + this.runsPerFile = queue.length / benchmarks.length; + this.currentFile = ''; // Filename of current benchmark. + this.currentFileConfig; // Configurations for current file + // Number of configurations already run for the current file. + this.completedConfig = 0; + // Total number of configurations for the current file + this.scheduledConfig = 0; + this.interval = 0; // result of setInterval for updating the elapsed time + } + + startQueue(index) { + this.kStartOfQueue = index; + this.currentFile = this.queue[index].filename; + this.interval = setInterval(() => { + if (this.completedRuns === this.scheduledRuns) { + clearInterval(this.interval); + } else { + this.updateProgress(); + } + }, 1000); + } + + startSubqueue(data, index) { + // This subqueue is generated by a new benchmark + if (data.name !== this.currentFile || index === this.kStartOfQueue) { + this.currentFile = data.name; + this.scheduledConfig = data.queueLength; + } + this.completedConfig = 0; + this.updateProgress(); + } + + completeConfig(data) { + this.completedConfig++; + this.updateProgress(); + } + + completeRun(job) { + this.completedRuns++; + this.updateProgress(); + } + + getProgress() { + // Get time as soon as possible. + const diff = process.hrtime(this.startTime); + + const completedRuns = this.completedRuns; + const scheduledRuns = this.scheduledRuns; + const finished = completedRuns === scheduledRuns; + + // Calculate numbers for fractions. + const runsPerFile = this.runsPerFile; + const completedFiles = Math.floor(completedRuns / runsPerFile); + const scheduledFiles = this.benchmarks.length; + const completedRunsForFile = finished ? runsPerFile : + completedRuns % runsPerFile; + const completedConfig = this.completedConfig; + const scheduledConfig = this.scheduledConfig; + + // Calculate the percentage. + let runRate = 0; // Rate of current incomplete run. + if (completedConfig !== scheduledConfig) { + runRate = completedConfig / scheduledConfig; + } + const completedRate = ((completedRuns + runRate) / scheduledRuns); + const percent = pad(Math.floor(completedRate * 100), 3, ' '); + + const caption = finished ? 'Done\n' : this.currentFile; + return `[${getTime(diff)}|% ${percent}` + + `| ${fraction(completedFiles, scheduledFiles)} files ` + + `| ${fraction(completedRunsForFile, runsPerFile)} runs ` + + `| ${fraction(completedConfig, scheduledConfig)} configs]` + + `: ${caption}`; + } + + updateProgress(finished) { + if (!process.stderr.isTTY || process.stdout.isTTY) { + return; + } + readline.clearLine(process.stderr); + readline.cursorTo(process.stderr, 0); + process.stderr.write(this.getProgress()); + } +} + +module.exports = BenchmarkProgress; diff --git a/benchmark/_cli.js b/benchmark/_cli.js index be2f7ffff83624..17718c4c4d757a 100644 --- a/benchmark/_cli.js +++ b/benchmark/_cli.js @@ -45,13 +45,13 @@ function CLI(usage, settings) { currentOptional = arg.slice(1); } - // Default the value to true - if (!settings.arrayArgs.includes(currentOptional)) { + if (settings.boolArgs && settings.boolArgs.includes(currentOptional)) { this.optional[currentOptional] = true; + mode = 'both'; + } else { + // expect the next value to be option related (either -- or the value) + mode = 'option'; } - - // expect the next value to be option related (either -- or the value) - mode = 'option'; } else if (mode === 'option') { // Optional arguments value diff --git a/benchmark/common.js b/benchmark/common.js index 4ce9501dd9cff7..6a9b2ba0f71681 100644 --- a/benchmark/common.js +++ b/benchmark/common.js @@ -128,6 +128,14 @@ Benchmark.prototype.http = function(options, cb) { Benchmark.prototype._run = function() { const self = this; + // If forked, report to the parent. + if (process.send) { + process.send({ + type: 'config', + name: this.name, + queueLength: this.queue.length + }); + } (function recursive(queueIndex) { const config = self.queue[queueIndex]; @@ -217,7 +225,8 @@ Benchmark.prototype.report = function(rate, elapsed) { name: this.name, conf: this.config, rate: rate, - time: elapsed[0] + elapsed[1] / 1e9 + time: elapsed[0] + elapsed[1] / 1e9, + type: 'report' }); }; diff --git a/benchmark/compare.js b/benchmark/compare.js index ea431b18cb4cd9..af36d1c4239edf 100644 --- a/benchmark/compare.js +++ b/benchmark/compare.js @@ -3,6 +3,7 @@ const fork = require('child_process').fork; const path = require('path'); const CLI = require('./_cli.js'); +const BenchmarkProgress = require('./_benchmark_progress.js'); // // Parse arguments @@ -13,13 +14,15 @@ const cli = CLI(`usage: ./node compare.js [options] [--] ... The output is formatted as csv, which can be processed using for example 'compare.R'. - --new ./new-node-binary new node binary (required) - --old ./old-node-binary old node binary (required) - --runs 30 number of samples - --filter pattern string to filter benchmark scripts - --set variable=value set benchmark variable (can be repeated) + --new ./new-node-binary new node binary (required) + --old ./old-node-binary old node binary (required) + --runs 30 number of samples + --filter pattern string to filter benchmark scripts + --set variable=value set benchmark variable (can be repeated) + --no-progress don't show benchmark progress indicator `, { - arrayArgs: ['set'] + arrayArgs: ['set'], + boolArgs: ['no-progress'] }); if (!cli.optional.new || !cli.optional.old) { @@ -39,6 +42,9 @@ if (benchmarks.length === 0) { // Create queue from the benchmarks list such both node versions are tested // `runs` amount of times each. +// Note: BenchmarkProgress relies on this order to estimate +// how much runs remaining for a file. All benchmarks generated from +// the same file must be run consecutively. const queue = []; for (const filename of benchmarks) { for (let iter = 0; iter < runs; iter++) { @@ -47,10 +53,20 @@ for (const filename of benchmarks) { } } } +// queue.length = binary.length * runs * benchmarks.length // Print csv header console.log('"binary", "filename", "configuration", "rate", "time"'); +const kStartOfQueue = 0; + +const showProgress = !cli.optional['no-progress']; +let progress; +if (showProgress) { + progress = new BenchmarkProgress(queue, benchmarks); + progress.startQueue(kStartOfQueue); +} + (function recursive(i) { const job = queue[i]; @@ -59,18 +75,26 @@ console.log('"binary", "filename", "configuration", "rate", "time"'); }); child.on('message', function(data) { - // Construct configuration string, " A=a, B=b, ..." - let conf = ''; - for (const key of Object.keys(data.conf)) { - conf += ' ' + key + '=' + JSON.stringify(data.conf[key]); - } - conf = conf.slice(1); + if (data.type === 'report') { + // Construct configuration string, " A=a, B=b, ..." + let conf = ''; + for (const key of Object.keys(data.conf)) { + conf += ' ' + key + '=' + JSON.stringify(data.conf[key]); + } + conf = conf.slice(1); + // Escape quotes (") for correct csv formatting + conf = conf.replace(/"/g, '""'); - // Escape quotes (") for correct csv formatting - conf = conf.replace(/"/g, '""'); - - console.log(`"${job.binary}", "${job.filename}", "${conf}", ` + - `${data.rate}, ${data.time}`); + console.log(`"${job.binary}", "${job.filename}", "${conf}", ` + + `${data.rate}, ${data.time}`); + if (showProgress) { + // One item in the subqueue has been completed. + progress.completeConfig(data); + } + } else if (showProgress && data.type === 'config') { + // The child has computed the configurations, ready to run subqueue. + progress.startSubqueue(data, i); + } }); child.once('close', function(code) { @@ -78,10 +102,13 @@ console.log('"binary", "filename", "configuration", "rate", "time"'); process.exit(code); return; } + if (showProgress) { + progress.completeRun(job); + } // If there are more benchmarks execute the next if (i + 1 < queue.length) { recursive(i + 1); } }); -})(0); +})(kStartOfQueue); diff --git a/benchmark/run.js b/benchmark/run.js index c04824866710a8..cb4f8cc00487fe 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -44,6 +44,9 @@ if (format === 'csv') { } child.on('message', function(data) { + if (data.type !== 'report') { + return; + } // Construct configuration string, " A=a, B=b, ..." let conf = ''; for (const key of Object.keys(data.conf)) { diff --git a/benchmark/scatter.js b/benchmark/scatter.js index 3003616b58eee4..65d1a5f604877d 100644 --- a/benchmark/scatter.js +++ b/benchmark/scatter.js @@ -42,6 +42,10 @@ function csvEncodeValue(value) { const child = fork(path.resolve(__dirname, filepath), cli.optional.set); child.on('message', function(data) { + if (data.type !== 'report') { + return; + } + // print csv header if (printHeader) { const confHeader = Object.keys(data.conf)