From 7e512e1205f0570eaacebb8fc8b9d16eba93ed6f Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 15 Jan 2020 15:25:45 -0500 Subject: [PATCH 01/10] add process profiler for debugging --- packages/server/index.js | 4 + packages/server/lib/util/process_profiler.ts | 140 +++++++++++++++++++ packages/server/package.json | 1 + 3 files changed, 145 insertions(+) create mode 100644 packages/server/lib/util/process_profiler.ts diff --git a/packages/server/index.js b/packages/server/index.js index 7fe552991e8b..b80314c8c07a 100644 --- a/packages/server/index.js +++ b/packages/server/index.js @@ -15,6 +15,10 @@ require('graceful-fs').gracefulify(require('fs')) require('@packages/ts/register') require('@packages/coffee/register') +if (process.env.CYPRESS_ENV) { + require('./lib/util/process_profiler').start() +} + require && require.extensions && delete require.extensions['.litcoffee'] require && require.extensions && delete require.extensions['.coffee.md'] diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts new file mode 100644 index 000000000000..417486e721b3 --- /dev/null +++ b/packages/server/lib/util/process_profiler.ts @@ -0,0 +1,140 @@ +import _ from 'lodash' +import { concatStream } from '@packages/network' +import Debug from 'debug' +import la from 'lazy-ass' +import si from 'systeminformation' + +const debug = Debug('cypress:server:util:process_profiler') +const debugVerbose = Debug('cypress-verbose:server:util:process_profiler') + +const interval = Number(process.env.CYPRESS_PROCESS_PROFILER_INTERVAL) || 10000 +let started = false + +let groupsOverTime = {} + +function checkProcesses () { + si.processes() + .then(({ list }) => { + let knownParents: number[] = [process.pid] + let cyProcesses: Set = new Set() + + const thisProcess = _.find(list, { pid: process.pid }) + + la(thisProcess, 'expected to find current pid in process list') + cyProcesses.add(thisProcess!) + + function findNewChildren () { + return _.filter(list, (v) => { + return knownParents.includes(v.parentPid) && !cyProcesses.has(v) + }) + } + + let newChildren: si.Systeminformation.ProcessesProcessData[] = [] + + // build the proc tree one layer at a time until no new children can be found + do { + newChildren = findNewChildren() + newChildren.forEach((child) => { + cyProcesses.add(child) + knownParents.push(child.pid) + }) + } while (newChildren.length > 0) + + return Array.from(cyProcesses.values()) + }) + .then((processes) => { + debugVerbose('all Cypress-launched processes: %o', processes) + + const consoleBuffer = concatStream(debug) + // eslint-disable-next-line no-console + const buffedConsole = new console.Console(consoleBuffer) + + const groupTotals = _.chain(processes) + .groupBy((proc) => proc!.name.split(' ')[0]) + .mapValues((groupedProcesses, groupName) => { + return { + groupName, + processCount: groupedProcesses.length, + pids: _.map(groupedProcesses, _.property('pid')), + totalCpuPercent: _.sumBy(groupedProcesses, 'pcpu'), + totalMemRssMb: _.sumBy(groupedProcesses, 'mem_rss') / 1024, + } + }) + .values() + .sortBy('totalMemRssMb') + .reverse() + .value() + + buffedConsole.log('current & mean memory and CPU usage by process group:') + groupTotals.push(_.reduce(groupTotals, (acc, val) => { + acc.processCount += val.processCount + acc.totalCpuPercent += val.totalCpuPercent + acc.totalMemRssMb += val.totalMemRssMb + + return acc + }, { groupName: '[TOTAL]', processCount: 0, pids: [], totalCpuPercent: 0, totalMemRssMb: 0 })) + + groupTotals.forEach((total) => { + if (!groupsOverTime[total.groupName]) { + groupsOverTime[total.groupName] = [] + } + + const measurements = groupsOverTime[total.groupName] + + measurements.push(total) + + _.merge(total, { + meanProcessCount: _.meanBy(measurements, 'processCount'), + meanCpuPercent: _.meanBy(measurements, 'totalCpuPercent'), + meanMemRssMb: _.meanBy(measurements, 'totalMemRssMb'), + maxMemRssMb: _.max(_.map(measurements, _.property('totalMemRssMb'))), + }) + + _.forEach(total, (v, k) => { + // round all numbers to 100ths precision + if (_.isNumber(v)) { + total[k] = _.round(v, 2) + } + }) + }) + + buffedConsole.table(groupTotals, [ + 'groupName', + 'processCount', + 'meanProcessCount', + 'pids', + 'totalCpuPercent', + 'meanCpuPercent', + 'totalMemRssMb', + 'meanMemRssMb', + 'maxMemRssMb', + ]) + + consoleBuffer.end() + }) + .then(scheduleProcessCheck) + .catch((err) => { + debug('error running process profiler: %o', err) + }) +} + +function scheduleProcessCheck () { + // not setinterval, since checkProcesses is asynchronous + setTimeout(checkProcesses, interval) +} + +export function start () { + if (!debug.enabled) { + debug('process profiler not enabled') + + return + } + + if (started) { + return + } + + checkProcesses() + + started = true +} diff --git a/packages/server/package.json b/packages/server/package.json index 88875a24283d..9fcd2b45f049 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -121,6 +121,7 @@ "squirrelly": "7.7.0", "strip-ansi": "3.0.1", "syntax-error": "1.4.0", + "systeminformation": "4.19.1", "term-size": "2.1.0", "tough-cookie": "3.0.1", "trash": "5.2.0", From 33db7935ecf897c2cc8bd08a25e2f89c917e4ef1 Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Tue, 21 Jan 2020 10:34:14 -0500 Subject: [PATCH 02/10] remove existing CYPRESS_MEMORY check --- packages/server/lib/project.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/server/lib/project.js b/packages/server/lib/project.js index c21f3e5d3f43..cb0d5387503e 100644 --- a/packages/server/lib/project.js +++ b/packages/server/lib/project.js @@ -55,7 +55,6 @@ class Project extends EE { this.spec = null this.browser = null this.server = null - this.memoryCheck = null this.automation = null this.getConfig = this.getConfig.bind(this) @@ -78,15 +77,6 @@ class Project extends EE { debug('project options %o', options) this.options = options - if (process.env.CYPRESS_MEMORY) { - const logMemory = () => { - // eslint-disable-next-line no-console - return console.log('memory info', process.memoryUsage()) - } - - this.memoryCheck = setInterval(logMemory, 1000) - } - this.onWarning = options.onWarning return this.getConfig(options) @@ -205,10 +195,6 @@ class Project extends EE { close () { debug('closing project instance %s', this.projectRoot) - if (this.memoryCheck) { - clearInterval(this.memoryCheck) - } - this.cfg = null this.spec = null this.browser = null From 2acca59a3a26d4f07689dec9e4612ab67be1b20d Mon Sep 17 00:00:00 2001 From: Brian Mann Date: Fri, 24 Jan 2020 17:59:23 -0500 Subject: [PATCH 03/10] rename some columns, format PIDs a little nicer, start when debug is enabled or verbosedebug is enabled --- packages/server/lib/util/process_profiler.ts | 42 +++++++++++--------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts index 417486e721b3..bc787151ab83 100644 --- a/packages/server/lib/util/process_profiler.ts +++ b/packages/server/lib/util/process_profiler.ts @@ -1,8 +1,9 @@ -import _ from 'lodash' -import { concatStream } from '@packages/network' import Debug from 'debug' import la from 'lazy-ass' +import _ from 'lodash' import si from 'systeminformation' +import util from 'util' +import { concatStream } from '@packages/network' const debug = Debug('cypress:server:util:process_profiler') const debugVerbose = Debug('cypress-verbose:server:util:process_profiler') @@ -12,8 +13,15 @@ let started = false let groupsOverTime = {} +const formatPidDisplay = (groupedProcesses) => { + const pids = _.chain(groupedProcesses).map('pid').map(_.toNumber).value() + + // slice off the starting '[ ' and the ending '] ' array brackets + return util.inspect(pids, { maxArrayLength: 10 }).slice(2, -2) +} + function checkProcesses () { - si.processes() + return si.processes() .then(({ list }) => { let knownParents: number[] = [process.pid] let cyProcesses: Set = new Set() @@ -55,24 +63,24 @@ function checkProcesses () { return { groupName, processCount: groupedProcesses.length, - pids: _.map(groupedProcesses, _.property('pid')), - totalCpuPercent: _.sumBy(groupedProcesses, 'pcpu'), - totalMemRssMb: _.sumBy(groupedProcesses, 'mem_rss') / 1024, + pids: formatPidDisplay(groupedProcesses), + cpuPercent: _.sumBy(groupedProcesses, 'pcpu'), + memRssMb: _.sumBy(groupedProcesses, 'mem_rss') / 1024, } }) .values() - .sortBy('totalMemRssMb') + .sortBy('memRssMb') .reverse() .value() buffedConsole.log('current & mean memory and CPU usage by process group:') groupTotals.push(_.reduce(groupTotals, (acc, val) => { acc.processCount += val.processCount - acc.totalCpuPercent += val.totalCpuPercent - acc.totalMemRssMb += val.totalMemRssMb + acc.cpuPercent += val.cpuPercent + acc.memRssMb += val.memRssMb return acc - }, { groupName: '[TOTAL]', processCount: 0, pids: [], totalCpuPercent: 0, totalMemRssMb: 0 })) + }, { groupName: 'TOTAL', processCount: 0, pids: '-', cpuPercent: 0, memRssMb: 0 })) groupTotals.forEach((total) => { if (!groupsOverTime[total.groupName]) { @@ -84,10 +92,9 @@ function checkProcesses () { measurements.push(total) _.merge(total, { - meanProcessCount: _.meanBy(measurements, 'processCount'), - meanCpuPercent: _.meanBy(measurements, 'totalCpuPercent'), - meanMemRssMb: _.meanBy(measurements, 'totalMemRssMb'), - maxMemRssMb: _.max(_.map(measurements, _.property('totalMemRssMb'))), + meanCpuPercent: _.meanBy(measurements, 'cpuPercent'), + meanMemRssMb: _.meanBy(measurements, 'memRssMb'), + maxMemRssMb: _.max(_.map(measurements, _.property('memRssMb'))), }) _.forEach(total, (v, k) => { @@ -101,11 +108,10 @@ function checkProcesses () { buffedConsole.table(groupTotals, [ 'groupName', 'processCount', - 'meanProcessCount', 'pids', - 'totalCpuPercent', + 'cpuPercent', 'meanCpuPercent', - 'totalMemRssMb', + 'memRssMb', 'meanMemRssMb', 'maxMemRssMb', ]) @@ -124,7 +130,7 @@ function scheduleProcessCheck () { } export function start () { - if (!debug.enabled) { + if (!debug.enabled && !debugVerbose.enabled) { debug('process profiler not enabled') return From a06160bd355bbdb1a5ceef53cad4beedcd715b5e Mon Sep 17 00:00:00 2001 From: Brian Mann Date: Fri, 24 Jan 2020 17:59:56 -0500 Subject: [PATCH 04/10] use shared method for determining if this is the electron process running --- packages/server/index.js | 10 ++++++++-- packages/server/lib/cypress.js | 4 ++-- packages/server/lib/modes/run.js | 4 ++-- .../util/{electron_app.js => electron-app.js} | 19 +++++++++++++------ .../server/test/integration/cypress_spec.js | 3 ++- 5 files changed, 27 insertions(+), 13 deletions(-) rename packages/server/lib/util/{electron_app.js => electron-app.js} (69%) diff --git a/packages/server/index.js b/packages/server/index.js index b80314c8c07a..ac8614b1b9bf 100644 --- a/packages/server/index.js +++ b/packages/server/index.js @@ -1,13 +1,19 @@ // override tty if we're being forced to require('./lib/util/tty').override() -if (process.env.CY_NET_PROFILE && process.env.CYPRESS_ENV) { +const electronApp = require('./lib/util/electron-app') + +// are we in the main node process or the electron process? +const isRunningElectron = electronApp.isRunning() + +if (process.env.CY_NET_PROFILE && isRunningElectron) { const netProfiler = require('./lib/util/net_profiler')() process.stdout.write(`Network profiler writing to ${netProfiler.logPath}\n`) } process.env.UV_THREADPOOL_SIZE = 128 + require('graceful-fs').gracefulify(require('fs')) // if running in production mode (CYPRESS_ENV) // all transpile should have been done already @@ -15,7 +21,7 @@ require('graceful-fs').gracefulify(require('fs')) require('@packages/ts/register') require('@packages/coffee/register') -if (process.env.CYPRESS_ENV) { +if (isRunningElectron) { require('./lib/util/process_profiler').start() } diff --git a/packages/server/lib/cypress.js b/packages/server/lib/cypress.js index bdcee462e4c4..1f621267a72f 100644 --- a/packages/server/lib/cypress.js +++ b/packages/server/lib/cypress.js @@ -45,7 +45,7 @@ const exitErr = (err) => { module.exports = { isCurrentlyRunningElectron () { - return !!(process.versions && process.versions.electron) + return require('./util/electron-app').isRunning() }, runElectron (mode, options) { @@ -170,7 +170,7 @@ module.exports = { // to force retina screens to not // upsample their images when offscreen // rendering - require('./util/electron_app').scale() + require('./util/electron-app').scale() } // make sure we have the appData folder diff --git a/packages/server/lib/modes/run.js b/packages/server/lib/modes/run.js index 94cfb770acd1..1b024f40dfb2 100644 --- a/packages/server/lib/modes/run.js +++ b/packages/server/lib/modes/run.js @@ -26,7 +26,7 @@ const newlines = require('../util/newlines') const terminal = require('../util/terminal') const specsUtil = require('../util/specs') const humanTime = require('../util/human_time') -const electronApp = require('../util/electron_app') +const electronApp = require('../util/electron-app') const settings = require('../util/settings') const chromePolicyCheck = require('../util/chrome_policy_check') @@ -1395,7 +1395,7 @@ module.exports = { run (options) { return electronApp - .ready() + .waitForReady() .then(() => { return this.ready(options) }) diff --git a/packages/server/lib/util/electron_app.js b/packages/server/lib/util/electron-app.js similarity index 69% rename from packages/server/lib/util/electron_app.js rename to packages/server/lib/util/electron-app.js index 9180b559e28b..ecb76fdcfb1b 100644 --- a/packages/server/lib/util/electron_app.js +++ b/packages/server/lib/util/electron-app.js @@ -1,5 +1,3 @@ -const debug = require('debug')('cypress:server:electron_app') - const scale = () => { try { const { app } = require('electron') @@ -10,7 +8,9 @@ const scale = () => { } } -const ready = () => { +const waitForReady = () => { + const debug = require('debug')('cypress:server:electron-app') + const Promise = require('bluebird') const { app } = require('electron') @@ -21,20 +21,27 @@ const ready = () => { debug('all BrowserWindows closed, not exiting') }) - const waitForReady = () => { + const onReadyEvent = () => { return new Promise((resolve) => { app.on('ready', resolve) }) } return Promise.any([ - waitForReady(), + onReadyEvent(), Promise.delay(500), ]) } +const isRunning = () => { + // are we in the electron or the node process? + return Boolean(process.versions && process.versions.electron) +} + module.exports = { scale, - ready, + waitForReady, + + isRunning, } diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index 366f07c2e883..091435ae5c2d 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -44,6 +44,7 @@ const env = require(`${root}lib/util/env`) const v = require(`${root}lib/util/validation`) const system = require(`${root}lib/util/system`) const appData = require(`${root}lib/util/app_data`) +const electronApp = require('../../lib/util/electron-app') const { formStatePath } = require(`${root}lib/util/saved_state`) const TYPICAL_BROWSERS = [ @@ -118,7 +119,7 @@ describe('lib/cypress', () => { // spawning a separate process sinon.stub(videoCapture, 'start').resolves({}) sinon.stub(plugins, 'init').resolves(undefined) - sinon.stub(cypress, 'isCurrentlyRunningElectron').returns(true) + sinon.stub(electronApp, 'isRunning').returns(true) sinon.stub(extension, 'setHostAndPath').resolves() sinon.stub(launcher, 'detect').resolves(TYPICAL_BROWSERS) sinon.stub(process, 'exit') From a13bd3687f37d9ab92759039baf6ed0ad2ccfa78 Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 29 Jan 2020 14:40:17 -0500 Subject: [PATCH 05/10] finish adding enhanced grouping --- packages/server/lib/browsers/electron.coffee | 10 +- packages/server/lib/browsers/index.js | 6 + packages/server/lib/plugins/index.coffee | 4 + packages/server/lib/util/process_profiler.ts | 145 ++++++++++++++----- packages/server/lib/video_capture.js | 12 +- 5 files changed, 142 insertions(+), 35 deletions(-) diff --git a/packages/server/lib/browsers/electron.coffee b/packages/server/lib/browsers/electron.coffee index b54ffc8a01cf..d45e35868a1d 100644 --- a/packages/server/lib/browsers/electron.coffee +++ b/packages/server/lib/browsers/electron.coffee @@ -21,6 +21,8 @@ ELECTRON_DEBUG_EVENTS = [ 'unresponsive' ] +instance = null + tryToCall = (win, method) -> try if not win.isDestroyed() @@ -72,6 +74,9 @@ module.exports = { _win.on "close", -> if not child.isDestroyed() child.destroy() + + ## add this pid to list of pids + instance?.pid?.push(tryToCall(child.webContents, 'getOSProcessId')) } _.defaultsDeep({}, options, defaults) @@ -253,9 +258,12 @@ module.exports = { events.emit("exit") - return _.extend events, { + instance = _.extend events, { + pid: [tryToCall(win.webContents, 'getOSProcessId')] browserWindow: win kill: -> tryToCall(win, "destroy") removeAllListeners: -> tryToCall(win, "removeAllListeners") } + + return instance } diff --git a/packages/server/lib/browsers/index.js b/packages/server/lib/browsers/index.js index 16907779eee1..26adf7036d06 100644 --- a/packages/server/lib/browsers/index.js +++ b/packages/server/lib/browsers/index.js @@ -125,6 +125,12 @@ module.exports = { close: kill, + // note: does not guarantee that `browser` is still running + // note: electron will return a list of pids for each webContent + getBrowserPid () { + return instance ? instance.pid : undefined + }, + getAllBrowsersWith (nameOrPath) { debug('getAllBrowsersWith %o', { nameOrPath }) if (nameOrPath) { diff --git a/packages/server/lib/plugins/index.coffee b/packages/server/lib/plugins/index.coffee index ab5c725f13b0..50fefb7993f4 100644 --- a/packages/server/lib/plugins/index.coffee +++ b/packages/server/lib/plugins/index.coffee @@ -22,6 +22,10 @@ register = (event, callback) -> registeredEvents[event] = callback module.exports = { + getPluginPid: () -> + if pluginsProcess + return pluginsProcess.pid + registerHandler: (handler) -> handlers.push(handler) diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts index bc787151ab83..cb6b11172025 100644 --- a/packages/server/lib/util/process_profiler.ts +++ b/packages/server/lib/util/process_profiler.ts @@ -2,9 +2,17 @@ import Debug from 'debug' import la from 'lazy-ass' import _ from 'lodash' import si from 'systeminformation' -import util from 'util' import { concatStream } from '@packages/network' +const { getBrowserPid } = require('../browsers') +const { getPluginPid } = require('../plugins') +const { getFfmpegPid } = require('../video_capture') + +type Group = 'browser' | 'cypress' | 'plugin' | 'desktop-gui' | 'ffmpeg' | 'electron-shared' | 'other' +type Process = si.Systeminformation.ProcessesProcessData & { + group?: Group +} + const debug = Debug('cypress:server:util:process_profiler') const debugVerbose = Debug('cypress-verbose:server:util:process_profiler') @@ -14,54 +22,125 @@ let started = false let groupsOverTime = {} const formatPidDisplay = (groupedProcesses) => { - const pids = _.chain(groupedProcesses).map('pid').map(_.toNumber).value() + const pids = _.map(groupedProcesses, 'pid') + const maxArrayLength = 6 + + let display = pids.slice(0, maxArrayLength).join(', ') + + if (pids.length > maxArrayLength) { + display += ` ... ${pids.length - maxArrayLength} more items` + } - // slice off the starting '[ ' and the ending '] ' array brackets - return util.inspect(pids, { maxArrayLength: 10 }).slice(2, -2) + return display } function checkProcesses () { return si.processes() .then(({ list }) => { - let knownParents: number[] = [process.pid] - let cyProcesses: Set = new Set() - - const thisProcess = _.find(list, { pid: process.pid }) + const cyProcesses: Process[] = [] + const thisProcess: Process = _.find(list, { pid: process.pid })! la(thisProcess, 'expected to find current pid in process list') - cyProcesses.add(thisProcess!) - function findNewChildren () { - return _.filter(list, (v) => { - return knownParents.includes(v.parentPid) && !cyProcesses.has(v) - }) + const isParentProcessInGroup = (proc: Process, group: Group) => { + return _.chain(cyProcesses).filter({ group }).map('pid').includes(proc.parentPid).value() } - let newChildren: si.Systeminformation.ProcessesProcessData[] = [] + // is this a browser process launched to run Cypress tests? + const isBrowserProcess = (proc: Process): boolean => { + // electron will return a list of pids, since it's not a hierarchy + const pid: number | number[] = getBrowserPid() - // build the proc tree one layer at a time until no new children can be found - do { - newChildren = findNewChildren() - newChildren.forEach((child) => { - cyProcesses.add(child) - knownParents.push(child.pid) - }) - } while (newChildren.length > 0) + debug({ pid }) + + return (Array.isArray(pid) ? (pid as number[]).includes(proc.pid) : proc.pid === pid) + || isParentProcessInGroup(proc, 'browser') + } + + const isPluginProcess = (proc: Process): boolean => { + return proc.pid === getPluginPid() + || isParentProcessInGroup(proc, 'plugin') + } + + // is this the renderer for the desktop-gui? + const isDesktopGuiProcess = (proc: Process): boolean => { + return proc.params.includes('--type=renderer') + && !isBrowserProcess(proc) + } - return Array.from(cyProcesses.values()) + // these processes may be shared between the AUT and desktop-gui + // rather than treat them as part of the `browser` in `run` mode and have + // their usage in `open` mode be ambiguous, just put them in their own group + const isElectronSharedProcess = (proc: Process): boolean => { + return proc.params.includes('--type=broker') + || proc.params.includes('--type=gpu-process') + || proc.params.includes('--type=utility') + || proc.params.includes('--type=zygote') + } + + const getProcessGroup = (proc: Process): Group => { + if (proc === thisProcess) { + return 'cypress' + } + + if (isBrowserProcess(proc)) { + return 'browser' + } + + if (isPluginProcess(proc)) { + return 'plugin' + } + + if (isDesktopGuiProcess(proc)) { + return 'desktop-gui' + } + + if (proc.pid === getFfmpegPid()) { + return 'ffmpeg' + } + + if (isElectronSharedProcess(proc)) { + return 'electron-shared' + } + + return 'other' + } + + const classifyProcess = (proc: Process) => { + const classify = (group: Group) => { + proc.group = group + cyProcesses.push(proc) + + // queue all children + _.chain(list) + .filter({ parentPid: proc.pid }) + .map(classifyProcess) + .value() + } + + classify(getProcessGroup(proc)) + } + + classifyProcess(thisProcess) + + return cyProcesses }) .then((processes) => { - debugVerbose('all Cypress-launched processes: %o', processes) + debugVerbose('all Cypress-launched processes: %s', require('util').inspect(processes)) + + const consoleBuffer = concatStream((buf) => { + // get rid of trailing newline + debug(String(buf).trim()) + }) - const consoleBuffer = concatStream(debug) // eslint-disable-next-line no-console const buffedConsole = new console.Console(consoleBuffer) const groupTotals = _.chain(processes) - .groupBy((proc) => proc!.name.split(' ')[0]) - .mapValues((groupedProcesses, groupName) => { + .groupBy('group') + .mapValues((groupedProcesses, group) => { return { - groupName, + group, processCount: groupedProcesses.length, pids: formatPidDisplay(groupedProcesses), cpuPercent: _.sumBy(groupedProcesses, 'pcpu'), @@ -80,14 +159,14 @@ function checkProcesses () { acc.memRssMb += val.memRssMb return acc - }, { groupName: 'TOTAL', processCount: 0, pids: '-', cpuPercent: 0, memRssMb: 0 })) + }, { group: 'TOTAL', processCount: 0, pids: '-', cpuPercent: 0, memRssMb: 0 })) groupTotals.forEach((total) => { - if (!groupsOverTime[total.groupName]) { - groupsOverTime[total.groupName] = [] + if (!groupsOverTime[total.group]) { + groupsOverTime[total.group] = [] } - const measurements = groupsOverTime[total.groupName] + const measurements = groupsOverTime[total.group] measurements.push(total) @@ -106,7 +185,7 @@ function checkProcesses () { }) buffedConsole.table(groupTotals, [ - 'groupName', + 'group', 'processCount', 'pids', 'cpuPercent', diff --git a/packages/server/lib/video_capture.js b/packages/server/lib/video_capture.js index cb87d949a039..956f869a0ec6 100644 --- a/packages/server/lib/video_capture.js +++ b/packages/server/lib/video_capture.js @@ -11,6 +11,8 @@ const fs = require('./util/fs') // extra verbose logs for logging individual frames const debugFrames = require('debug')('cypress-verbose:server:video:frames') +let currentPid + debug('using ffmpeg from %s', ffmpegPath) ffmpeg.setFfmpegPath(ffmpegPath) @@ -27,6 +29,10 @@ const deferredPromise = function () { } module.exports = { + getFfmpegPid () { + return currentPid + }, + getMsFromDuration (duration) { return utils.timemarkToSeconds(duration) * 1000 }, @@ -133,6 +139,8 @@ module.exports = { .videoCodec('libx264') .outputOptions('-preset ultrafast') .on('start', (command) => { + currentPid = cmd.ffmpegProc.pid + debug('capture started %o', { command }) return resolve({ @@ -180,7 +188,7 @@ module.exports = { debug('processing video from %s to %s video compression %o', name, cname, videoCompression) - ffmpeg() + const cmd = ffmpeg() .input(name) .videoCodec('libx264') .outputOptions([ @@ -188,6 +196,8 @@ module.exports = { `-crf ${videoCompression}`, ]) .on('start', (command) => { + currentPid = cmd.ffmpegProc.pid + return debug('compression started %o', { command }) }).on('codecData', (data) => { debug('compression codec data: %o', data) From 0fe1aa93e117bffcc0f0d0f7a40428eee7dc9b48 Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 29 Jan 2020 16:31:47 -0500 Subject: [PATCH 06/10] cleanup --- packages/server/lib/browsers/electron.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/lib/browsers/electron.coffee b/packages/server/lib/browsers/electron.coffee index d45e35868a1d..6a19d46554a5 100644 --- a/packages/server/lib/browsers/electron.coffee +++ b/packages/server/lib/browsers/electron.coffee @@ -76,7 +76,8 @@ module.exports = { child.destroy() ## add this pid to list of pids - instance?.pid?.push(tryToCall(child.webContents, 'getOSProcessId')) + tryToCall child, -> + instance?.pid?.push(child.webContents.getOSProcessId()) } _.defaultsDeep({}, options, defaults) @@ -259,7 +260,7 @@ module.exports = { events.emit("exit") instance = _.extend events, { - pid: [tryToCall(win.webContents, 'getOSProcessId')] + pid: [tryToCall(win, -> win.webContents.getOSProcessId())] browserWindow: win kill: -> tryToCall(win, "destroy") removeAllListeners: -> tryToCall(win, "removeAllListeners") From f95dfc038861fc8c207e102e50ea252490a8376c Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 29 Jan 2020 16:32:03 -0500 Subject: [PATCH 07/10] add tests for changes to other parts of cypress --- packages/server/lib/browsers/index.js | 5 +++ packages/server/lib/plugins/index.coffee | 4 +++ .../test/unit/browsers/browsers_spec.coffee | 9 ++++++ .../test/unit/browsers/electron_spec.coffee | 31 +++++++++++++++++-- .../test/unit/plugins/index_spec.coffee | 16 ++++++++++ 5 files changed, 63 insertions(+), 2 deletions(-) diff --git a/packages/server/lib/browsers/index.js b/packages/server/lib/browsers/index.js index 26adf7036d06..d7444ef5182c 100644 --- a/packages/server/lib/browsers/index.js +++ b/packages/server/lib/browsers/index.js @@ -125,6 +125,11 @@ module.exports = { close: kill, + _setInstance (_instance) { + // for testing + instance = _instance + }, + // note: does not guarantee that `browser` is still running // note: electron will return a list of pids for each webContent getBrowserPid () { diff --git a/packages/server/lib/plugins/index.coffee b/packages/server/lib/plugins/index.coffee index 50fefb7993f4..0ea612935ece 100644 --- a/packages/server/lib/plugins/index.coffee +++ b/packages/server/lib/plugins/index.coffee @@ -22,6 +22,10 @@ register = (event, callback) -> registeredEvents[event] = callback module.exports = { + ## for testing + _setPluginsProcess: (_pluginsProcess) -> + pluginsProcess = _pluginsProcess + getPluginPid: () -> if pluginsProcess return pluginsProcess.pid diff --git a/packages/server/test/unit/browsers/browsers_spec.coffee b/packages/server/test/unit/browsers/browsers_spec.coffee index 295a2e07e84b..227c468db0cb 100644 --- a/packages/server/test/unit/browsers/browsers_spec.coffee +++ b/packages/server/test/unit/browsers/browsers_spec.coffee @@ -6,6 +6,15 @@ browsers = require("#{root}../lib/browsers") utils = require("#{root}../lib/browsers/utils") describe "lib/browsers/index", -> + context ".getBrowserPid", -> + it "returns instance.pid if instance", -> + browsers._setInstance({ pid: 1234 }) + expect(browsers.getBrowserPid()).to.eq(1234) + + it "returns undefined if no instance", -> + browsers._setInstance(null) + expect(browsers.getBrowserPid()).to.be.undefined + context ".isBrowserFamily", -> it "allows only known browsers", -> expect(browsers.isBrowserFamily("chrome")).to.be.true diff --git a/packages/server/test/unit/browsers/electron_spec.coffee b/packages/server/test/unit/browsers/electron_spec.coffee index bb6c19f867e9..660c3d4c2ad1 100644 --- a/packages/server/test/unit/browsers/electron_spec.coffee +++ b/packages/server/test/unit/browsers/electron_spec.coffee @@ -12,6 +12,8 @@ electron = require("#{root}../lib/browsers/electron") savedState = require("#{root}../lib/saved_state") Automation = require("#{root}../lib/automation") +ELECTRON_PID = 10001 + describe "lib/browsers/electron", -> beforeEach -> @url = "https://foo.com" @@ -34,6 +36,7 @@ describe "lib/browsers/electron", -> remove: sinon.stub() } } + getOSProcessId: sinon.stub().returns(ELECTRON_PID) "debugger": { attach: sinon.stub().returns() sendCommand: sinon.stub().resolves() @@ -42,8 +45,7 @@ describe "lib/browsers/electron", -> } }) - context ".open", -> - beforeEach -> + @stubForOpen = -> sinon.stub(electron, "_render").resolves(@win) sinon.stub(plugins, "has") sinon.stub(plugins, "execute") @@ -53,6 +55,10 @@ describe "lib/browsers/electron", -> la(check.fn(state.get), "state is missing .get to stub", state) sinon.stub(state, "get").resolves(@state) + context ".open", -> + beforeEach -> + @stubForOpen() + it "calls render with url, state, and options", -> electron.open("electron", @url, @options, @automation) .then => @@ -76,6 +82,9 @@ describe "lib/browsers/electron", -> expect(obj.kill).to.be.a("function") expect(obj.removeAllListeners).to.be.a("function") + expect(@win.webContents.getOSProcessId).to.be.calledOnce + expect(obj.pid).to.deep.eq([ELECTRON_PID]) + it "is noop when before:browser:launch yields null", -> plugins.has.returns(true) plugins.execute.resolves(null) @@ -239,6 +248,24 @@ describe "lib/browsers/electron", -> event, @url, parentWindow, @options.projectRoot, @state, @options ) + it "adds pid of new BrowserWindow to pid list", -> + opts = electron._defaultOptions(@options.projectRoot, @state, @options) + + NEW_WINDOW_PID = ELECTRON_PID * 2 + + child = _.cloneDeep(@win) + child.webContents.getOSProcessId = sinon.stub().returns(NEW_WINDOW_PID) + + electron._launchChild.resolves(child) + + @stubForOpen() + .then => + electron.open("electron", @url, opts, @automation) + .then (instance) => + opts.onNewWindow.call(@win, {}, @url) + .then -> + expect(instance.pid).to.deep.eq([ELECTRON_PID, NEW_WINDOW_PID]) + ## TODO: these all need to be updated context.skip "._launchChild", -> beforeEach -> diff --git a/packages/server/test/unit/plugins/index_spec.coffee b/packages/server/test/unit/plugins/index_spec.coffee index c23c1ee2afcf..fc65f6751865 100644 --- a/packages/server/test/unit/plugins/index_spec.coffee +++ b/packages/server/test/unit/plugins/index_spec.coffee @@ -5,6 +5,8 @@ cp = require("child_process") util = require("#{root}../lib/plugins/util") plugins = require("#{root}../lib/plugins") +PLUGIN_PID = 77777 + describe "lib/plugins/index", -> beforeEach -> plugins._reset() @@ -13,6 +15,7 @@ describe "lib/plugins/index", -> send: sinon.spy() on: sinon.stub() kill: sinon.spy() + pid: PLUGIN_PID } sinon.stub(cp, "fork").returns(@pluginsProcess) @@ -200,3 +203,16 @@ describe "lib/plugins/index", -> plugins.register("foo", foo) plugins.execute("foo", "arg1", "arg2") expect(foo).to.be.calledWith("arg1", "arg2") + + context "#getPluginPid", -> + beforeEach -> + plugins._setPluginsProcess(null) + + it "returns the pid if there is a plugins process", -> + @ipc.on.withArgs("loaded").yields([]) + plugins.init({ pluginsFile: "cypress-plugin" }) + .then -> + expect(plugins.getPluginPid()).to.eq(PLUGIN_PID) + + it "returns undefined if there is no plugins process", -> + expect(plugins.getPluginPid()).to.be.undefined From 095d3a533ed1a85a40f1fa9b40532fbf9455b55d Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 29 Jan 2020 17:19:50 -0500 Subject: [PATCH 08/10] add tests for process_profiler --- packages/server/lib/util/process_profiler.ts | 282 +++++++++--------- .../test/unit/util/process_profiler_spec.ts | 140 +++++++++ 2 files changed, 285 insertions(+), 137 deletions(-) create mode 100644 packages/server/test/unit/util/process_profiler_spec.ts diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts index cb6b11172025..7790f31408dc 100644 --- a/packages/server/lib/util/process_profiler.ts +++ b/packages/server/lib/util/process_profiler.ts @@ -4,9 +4,9 @@ import _ from 'lodash' import si from 'systeminformation' import { concatStream } from '@packages/network' -const { getBrowserPid } = require('../browsers') -const { getPluginPid } = require('../plugins') -const { getFfmpegPid } = require('../video_capture') +const browsers = require('../browsers') +const plugins = require('../plugins') +const videoCapture = require('../video_capture') type Group = 'browser' | 'cypress' | 'plugin' | 'desktop-gui' | 'ffmpeg' | 'electron-shared' | 'other' type Process = si.Systeminformation.ProcessesProcessData & { @@ -34,178 +34,186 @@ const formatPidDisplay = (groupedProcesses) => { return display } -function checkProcesses () { - return si.processes() - .then(({ list }) => { - const cyProcesses: Process[] = [] - const thisProcess: Process = _.find(list, { pid: process.pid })! +export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) => { + const cyProcesses: Process[] = [] + const thisProcess: Process = _.find(list, { pid: process.pid })! + + la(thisProcess, 'expected to find current pid in process list', process.pid) + + const isParentProcessInGroup = (proc: Process, group: Group) => { + return _.chain(cyProcesses).filter({ group }).map('pid').includes(proc.parentPid).value() + } + + // is this a browser process launched to run Cypress tests? + const isBrowserProcess = (proc: Process): boolean => { + // electron will return a list of pids, since it's not a hierarchy + const pid: number | number[] = browsers.getBrowserPid() + + debug({ pid }) + + return (Array.isArray(pid) ? (pid as number[]).includes(proc.pid) : proc.pid === pid) + || isParentProcessInGroup(proc, 'browser') + } + + const isPluginProcess = (proc: Process): boolean => { + return proc.pid === plugins.getPluginPid() + || isParentProcessInGroup(proc, 'plugin') + } - la(thisProcess, 'expected to find current pid in process list') + // is this the renderer for the desktop-gui? + const isDesktopGuiProcess = (proc: Process): boolean => { + return proc.params.includes('--type=renderer') + && !isBrowserProcess(proc) + } - const isParentProcessInGroup = (proc: Process, group: Group) => { - return _.chain(cyProcesses).filter({ group }).map('pid').includes(proc.parentPid).value() + // these processes may be shared between the AUT and desktop-gui + // rather than treat them as part of the `browser` in `run` mode and have + // their usage in `open` mode be ambiguous, just put them in their own group + const isElectronSharedProcess = (proc: Process): boolean => { + const isType = (type) => { + return proc.params.includes(`--type=${type}`) } - // is this a browser process launched to run Cypress tests? - const isBrowserProcess = (proc: Process): boolean => { - // electron will return a list of pids, since it's not a hierarchy - const pid: number | number[] = getBrowserPid() + return isType('broker') + || isType('gpu-process') + || isType('utility') + || isType('zygote') + } - debug({ pid }) + const getProcessGroup = (proc: Process): Group => { + if (proc === thisProcess) { + return 'cypress' + } - return (Array.isArray(pid) ? (pid as number[]).includes(proc.pid) : proc.pid === pid) - || isParentProcessInGroup(proc, 'browser') + if (isBrowserProcess(proc)) { + return 'browser' } - const isPluginProcess = (proc: Process): boolean => { - return proc.pid === getPluginPid() - || isParentProcessInGroup(proc, 'plugin') + if (isPluginProcess(proc)) { + return 'plugin' } - // is this the renderer for the desktop-gui? - const isDesktopGuiProcess = (proc: Process): boolean => { - return proc.params.includes('--type=renderer') - && !isBrowserProcess(proc) + if (isDesktopGuiProcess(proc)) { + return 'desktop-gui' } - // these processes may be shared between the AUT and desktop-gui - // rather than treat them as part of the `browser` in `run` mode and have - // their usage in `open` mode be ambiguous, just put them in their own group - const isElectronSharedProcess = (proc: Process): boolean => { - return proc.params.includes('--type=broker') - || proc.params.includes('--type=gpu-process') - || proc.params.includes('--type=utility') - || proc.params.includes('--type=zygote') + if (proc.pid === videoCapture.getFfmpegPid()) { + return 'ffmpeg' } - const getProcessGroup = (proc: Process): Group => { - if (proc === thisProcess) { - return 'cypress' - } + if (isElectronSharedProcess(proc)) { + return 'electron-shared' + } - if (isBrowserProcess(proc)) { - return 'browser' - } + return 'other' + } - if (isPluginProcess(proc)) { - return 'plugin' - } + const classifyProcess = (proc: Process) => { + const classify = (group: Group) => { + proc.group = group + cyProcesses.push(proc) - if (isDesktopGuiProcess(proc)) { - return 'desktop-gui' - } + // queue all children + _.chain(list) + .filter({ parentPid: proc.pid }) + .map(classifyProcess) + .value() + } - if (proc.pid === getFfmpegPid()) { - return 'ffmpeg' - } + classify(getProcessGroup(proc)) + } - if (isElectronSharedProcess(proc)) { - return 'electron-shared' - } + classifyProcess(thisProcess) - return 'other' - } + return cyProcesses +} - const classifyProcess = (proc: Process) => { - const classify = (group: Group) => { - proc.group = group - cyProcesses.push(proc) +export const _printGroupedProcesses = (processes) => { + debugVerbose('all Cypress-launched processes: %s', require('util').inspect(processes)) - // queue all children - _.chain(list) - .filter({ parentPid: proc.pid }) - .map(classifyProcess) - .value() - } + const consoleBuffer = concatStream((buf) => { + // get rid of trailing newline + debug(String(buf).trim()) + }) - classify(getProcessGroup(proc)) + // eslint-disable-next-line no-console + const buffedConsole = new console.Console(consoleBuffer) + + const groupTotals = _.chain(processes) + .groupBy('group') + .mapValues((groupedProcesses, group) => { + return { + group, + processCount: groupedProcesses.length, + pids: formatPidDisplay(groupedProcesses), + cpuPercent: _.sumBy(groupedProcesses, 'pcpu'), + memRssMb: _.sumBy(groupedProcesses, 'mem_rss') / 1024, + } + }) + .values() + .sortBy('memRssMb') + .reverse() + .value() + + buffedConsole.log('current & mean memory and CPU usage by process group:') + groupTotals.push(_.reduce(groupTotals, (acc, val) => { + acc.processCount += val.processCount + acc.cpuPercent += val.cpuPercent + acc.memRssMb += val.memRssMb + + return acc + }, { group: 'TOTAL', processCount: 0, pids: '-', cpuPercent: 0, memRssMb: 0 })) + + groupTotals.forEach((total) => { + if (!groupsOverTime[total.group]) { + groupsOverTime[total.group] = [] } - classifyProcess(thisProcess) + const measurements = groupsOverTime[total.group] - return cyProcesses - }) - .then((processes) => { - debugVerbose('all Cypress-launched processes: %s', require('util').inspect(processes)) + measurements.push(total) - const consoleBuffer = concatStream((buf) => { - // get rid of trailing newline - debug(String(buf).trim()) + _.merge(total, { + meanCpuPercent: _.meanBy(measurements, 'cpuPercent'), + meanMemRssMb: _.meanBy(measurements, 'memRssMb'), + maxMemRssMb: _.max(_.map(measurements, _.property('memRssMb'))), }) - // eslint-disable-next-line no-console - const buffedConsole = new console.Console(consoleBuffer) - - const groupTotals = _.chain(processes) - .groupBy('group') - .mapValues((groupedProcesses, group) => { - return { - group, - processCount: groupedProcesses.length, - pids: formatPidDisplay(groupedProcesses), - cpuPercent: _.sumBy(groupedProcesses, 'pcpu'), - memRssMb: _.sumBy(groupedProcesses, 'mem_rss') / 1024, + _.forEach(total, (v, k) => { + // round all numbers to 100ths precision + if (_.isNumber(v)) { + total[k] = _.round(v, 2) } }) - .values() - .sortBy('memRssMb') - .reverse() - .value() - - buffedConsole.log('current & mean memory and CPU usage by process group:') - groupTotals.push(_.reduce(groupTotals, (acc, val) => { - acc.processCount += val.processCount - acc.cpuPercent += val.cpuPercent - acc.memRssMb += val.memRssMb - - return acc - }, { group: 'TOTAL', processCount: 0, pids: '-', cpuPercent: 0, memRssMb: 0 })) - - groupTotals.forEach((total) => { - if (!groupsOverTime[total.group]) { - groupsOverTime[total.group] = [] - } - - const measurements = groupsOverTime[total.group] - - measurements.push(total) - - _.merge(total, { - meanCpuPercent: _.meanBy(measurements, 'cpuPercent'), - meanMemRssMb: _.meanBy(measurements, 'memRssMb'), - maxMemRssMb: _.max(_.map(measurements, _.property('memRssMb'))), - }) + }) - _.forEach(total, (v, k) => { - // round all numbers to 100ths precision - if (_.isNumber(v)) { - total[k] = _.round(v, 2) - } - }) - }) + buffedConsole.table(groupTotals, [ + 'group', + 'processCount', + 'pids', + 'cpuPercent', + 'meanCpuPercent', + 'memRssMb', + 'meanMemRssMb', + 'maxMemRssMb', + ]) + + consoleBuffer.end() +} - buffedConsole.table(groupTotals, [ - 'group', - 'processCount', - 'pids', - 'cpuPercent', - 'meanCpuPercent', - 'memRssMb', - 'meanMemRssMb', - 'maxMemRssMb', - ]) - - consoleBuffer.end() - }) - .then(scheduleProcessCheck) +function _checkProcesses () { + return si.processes() + .then(_groupCyProcesses) + .then(_printGroupedProcesses) + .then(_scheduleProcessCheck) .catch((err) => { debug('error running process profiler: %o', err) }) } -function scheduleProcessCheck () { +function _scheduleProcessCheck () { // not setinterval, since checkProcesses is asynchronous - setTimeout(checkProcesses, interval) + setTimeout(_checkProcesses, interval) } export function start () { @@ -219,7 +227,7 @@ export function start () { return } - checkProcesses() + _checkProcesses() started = true } diff --git a/packages/server/test/unit/util/process_profiler_spec.ts b/packages/server/test/unit/util/process_profiler_spec.ts new file mode 100644 index 000000000000..823ab0bde707 --- /dev/null +++ b/packages/server/test/unit/util/process_profiler_spec.ts @@ -0,0 +1,140 @@ +import '../../spec_helper' + +import _ from 'lodash' +import si from 'systeminformation' +import { expect } from 'chai' +import { _groupCyProcesses } from '../../../lib/util/process_profiler' +import sinon from 'sinon' + +const browsers = require('../../../lib/browsers') +const plugins = require('../../../lib/plugins') +const videoCapture = require('../../../lib/video_capture') + +const BROWSER_PID = 11111 +const SUB_BROWSER_PID = 11112 +const GUI_PID = 77777 +const PLUGIN_PID = 22222 +const SUB_PLUGIN_PID = 22223 +const FFMPEG_PID = 33333 +const MAIN_PID = process.pid +const OTHER_PID = 66666 +const ANOTHER_PID = 88888 +const LAUNCHER_PID = 55555 +const SHARED_BROKER_PID = 99990 +const SHARED_GPU_PID = 99991 +const SHARED_UTILITY_PID = 99992 +const SHARED_ZYGOTE_PID = 99993 + +const PROCESSES: Partial[] = [ + { + pid: MAIN_PID, + parentPid: LAUNCHER_PID, + params: '', + name: 'Cypress', + }, + { + pid: BROWSER_PID, + parentPid: MAIN_PID, + params: '', + name: 'firefox', + }, + { + pid: SUB_BROWSER_PID, + parentPid: BROWSER_PID, + params: '', + name: 'firefox-bin', + }, + { + pid: GUI_PID, + parentPid: MAIN_PID, + params: '--type=renderer', + name: 'Cypress', + }, + { + pid: PLUGIN_PID, + parentPid: MAIN_PID, + params: 'plugin.js', + name: 'node', + }, + { + pid: SUB_PLUGIN_PID, + parentPid: PLUGIN_PID, + params: '', + name: 'msword.exe', + }, + { + pid: FFMPEG_PID, + parentPid: MAIN_PID, + params: '', + name: 'ffmpeg', + }, + { + pid: OTHER_PID, + parentPid: MAIN_PID, + params: '', + name: 'foo', + }, + { + pid: ANOTHER_PID, + parentPid: MAIN_PID, + params: '', + name: 'bar', + }, + { + pid: SHARED_GPU_PID, + parentPid: MAIN_PID, + params: '--type=gpu-process', + name: 'Cypress', + }, + { + pid: SHARED_BROKER_PID, + parentPid: MAIN_PID, + params: '--type=broker', + name: 'Cypress', + }, + { + pid: SHARED_UTILITY_PID, + parentPid: MAIN_PID, + params: '--type=utility', + name: 'Cypress', + }, + { + pid: SHARED_ZYGOTE_PID, + parentPid: MAIN_PID, + params: '--type=zygote', + name: 'Cypress', + }, +] + +describe('lib/util/process_profiler', function () { + context('._groupCyProcesses', () => { + it('groups correctly', () => { + sinon.stub(browsers, 'getBrowserPid').returns(BROWSER_PID) + sinon.stub(plugins, 'getPluginPid').returns(PLUGIN_PID) + sinon.stub(videoCapture, 'getFfmpegPid').returns(FFMPEG_PID) + + // @ts-ignore + const groupedProcesses = _groupCyProcesses({ list: PROCESSES }) + + const checkGroup = (pid, group) => { + expect(_.find(groupedProcesses, { pid })) + .to.have.property('group') + .eq(group) + } + + checkGroup(BROWSER_PID, 'browser') + checkGroup(SUB_BROWSER_PID, 'browser') + checkGroup(GUI_PID, 'desktop-gui') + checkGroup(PLUGIN_PID, 'plugin') + checkGroup(SUB_PLUGIN_PID, 'plugin') + checkGroup(FFMPEG_PID, 'ffmpeg') + checkGroup(MAIN_PID, 'cypress') + checkGroup(OTHER_PID, 'other') + checkGroup(ANOTHER_PID, 'other') + checkGroup(SHARED_GPU_PID, 'electron-shared') + checkGroup(SHARED_BROKER_PID, 'electron-shared') + checkGroup(SHARED_UTILITY_PID, 'electron-shared') + checkGroup(SHARED_ZYGOTE_PID, 'electron-shared') + }) + }) +}) From d2d70dbf4f90376b25ab26f150299317ff0e0ba7 Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 29 Jan 2020 17:30:40 -0500 Subject: [PATCH 09/10] use browser displayname to group --- packages/server/lib/browsers/index.js | 6 +++-- packages/server/lib/util/process_profiler.ts | 23 +++++++++++++--- .../test/unit/browsers/browsers_spec.coffee | 13 +++++----- .../test/unit/util/process_profiler_spec.ts | 26 +++++++++++++++++-- 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/packages/server/lib/browsers/index.js b/packages/server/lib/browsers/index.js index d7444ef5182c..79deacd2ff76 100644 --- a/packages/server/lib/browsers/index.js +++ b/packages/server/lib/browsers/index.js @@ -132,8 +132,8 @@ module.exports = { // note: does not guarantee that `browser` is still running // note: electron will return a list of pids for each webContent - getBrowserPid () { - return instance ? instance.pid : undefined + getBrowserInstance () { + return instance }, getAllBrowsersWith (nameOrPath) { @@ -171,6 +171,8 @@ module.exports = { // TODO: bind to process.exit here // or move this functionality into cypress-core-launder + i.browser = browser + instance = i // TODO: normalizing opening and closing / exiting diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts index 7790f31408dc..a44c4668d655 100644 --- a/packages/server/lib/util/process_profiler.ts +++ b/packages/server/lib/util/process_profiler.ts @@ -46,10 +46,9 @@ export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) // is this a browser process launched to run Cypress tests? const isBrowserProcess = (proc: Process): boolean => { + const instance = browsers.getBrowserInstance() // electron will return a list of pids, since it's not a hierarchy - const pid: number | number[] = browsers.getBrowserPid() - - debug({ pid }) + const pid: number | number[] = instance && instance.pid return (Array.isArray(pid) ? (pid as number[]).includes(proc.pid) : proc.pid === pid) || isParentProcessInGroup(proc, 'browser') @@ -128,6 +127,23 @@ export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) return cyProcesses } +export const _renameBrowserGroup = (processes) => { + const instance = browsers.getBrowserInstance() + const displayName = _.get(instance, 'browser.displayName') + + processes.forEach((proc) => { + if (!displayName) { + return + } + + if (proc.group === 'browser') { + proc.group = displayName + } + }) + + return processes +} + export const _printGroupedProcesses = (processes) => { debugVerbose('all Cypress-launched processes: %s', require('util').inspect(processes)) @@ -204,6 +220,7 @@ export const _printGroupedProcesses = (processes) => { function _checkProcesses () { return si.processes() .then(_groupCyProcesses) + .then(_renameBrowserGroup) .then(_printGroupedProcesses) .then(_scheduleProcessCheck) .catch((err) => { diff --git a/packages/server/test/unit/browsers/browsers_spec.coffee b/packages/server/test/unit/browsers/browsers_spec.coffee index 227c468db0cb..00dae1acebeb 100644 --- a/packages/server/test/unit/browsers/browsers_spec.coffee +++ b/packages/server/test/unit/browsers/browsers_spec.coffee @@ -6,14 +6,15 @@ browsers = require("#{root}../lib/browsers") utils = require("#{root}../lib/browsers/utils") describe "lib/browsers/index", -> - context ".getBrowserPid", -> - it "returns instance.pid if instance", -> - browsers._setInstance({ pid: 1234 }) - expect(browsers.getBrowserPid()).to.eq(1234) + context ".getBrowserInstance", -> + it "returns instance", -> + instance = { pid: 1234 } + browsers._setInstance(instance) + expect(browsers.getBrowserInstance()).to.eq(instance) it "returns undefined if no instance", -> - browsers._setInstance(null) - expect(browsers.getBrowserPid()).to.be.undefined + browsers._setInstance() + expect(browsers.getBrowserInstance()).to.be.undefined context ".isBrowserFamily", -> it "allows only known browsers", -> diff --git a/packages/server/test/unit/util/process_profiler_spec.ts b/packages/server/test/unit/util/process_profiler_spec.ts index 823ab0bde707..090aba659b16 100644 --- a/packages/server/test/unit/util/process_profiler_spec.ts +++ b/packages/server/test/unit/util/process_profiler_spec.ts @@ -3,7 +3,7 @@ import '../../spec_helper' import _ from 'lodash' import si from 'systeminformation' import { expect } from 'chai' -import { _groupCyProcesses } from '../../../lib/util/process_profiler' +import { _groupCyProcesses, _renameBrowserGroup } from '../../../lib/util/process_profiler' import sinon from 'sinon' const browsers = require('../../../lib/browsers') @@ -109,7 +109,7 @@ const PROCESSES: Partial[] = [ describe('lib/util/process_profiler', function () { context('._groupCyProcesses', () => { it('groups correctly', () => { - sinon.stub(browsers, 'getBrowserPid').returns(BROWSER_PID) + sinon.stub(browsers, 'getBrowserInstance').returns({ pid: BROWSER_PID }) sinon.stub(plugins, 'getPluginPid').returns(PLUGIN_PID) sinon.stub(videoCapture, 'getFfmpegPid').returns(FFMPEG_PID) @@ -137,4 +137,26 @@ describe('lib/util/process_profiler', function () { checkGroup(SHARED_ZYGOTE_PID, 'electron-shared') }) }) + + context('._renameBrowserGroup', () => { + it('renames browser-grouped processes to correct name', () => { + sinon.stub(browsers, 'getBrowserInstance').returns({ browser: { displayName: 'FooBrowser' } }) + + const processes = [ + { group: 'foo' }, + { group: 'bar' }, + { group: 'browser', pid: 1 }, + { group: 'browser', pid: 2 }, + ] + + const expected = [ + { group: 'foo' }, + { group: 'bar' }, + { group: 'FooBrowser', pid: 1 }, + { group: 'FooBrowser', pid: 2 }, + ] + + expect(_renameBrowserGroup(processes)).to.deep.eq(expected) + }) + }) }) From ae6d2d913285cf1ea4f3daab325be1bdf7169957 Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Thu, 30 Jan 2020 10:43:46 -0500 Subject: [PATCH 10/10] fix ffmpeg and add tests --- .../__snapshots__/process_profiler_spec.ts.js | 82 +++++++++++++++++++ packages/server/lib/util/process_profiler.ts | 45 ++++++---- packages/server/lib/video_capture.js | 12 +-- .../test/unit/util/process_profiler_spec.ts | 40 ++++++++- 4 files changed, 149 insertions(+), 30 deletions(-) create mode 100644 packages/server/__snapshots__/process_profiler_spec.ts.js diff --git a/packages/server/__snapshots__/process_profiler_spec.ts.js b/packages/server/__snapshots__/process_profiler_spec.ts.js new file mode 100644 index 000000000000..8143ca5eb524 --- /dev/null +++ b/packages/server/__snapshots__/process_profiler_spec.ts.js @@ -0,0 +1,82 @@ +exports['lib/util/process_profiler ._aggregateGroups aggregates groups as expected 1'] = [ + { + "group": "electron-shared", + "processCount": 4, + "pids": "99991, 99990, 99992, 99993", + "cpuPercent": 80, + "memRssMb": 40, + "meanCpuPercent": 80, + "meanMemRssMb": 40, + "maxMemRssMb": 40 + }, + { + "group": "other", + "processCount": 2, + "pids": "66666, 88888", + "cpuPercent": 40, + "memRssMb": 20, + "meanCpuPercent": 40, + "meanMemRssMb": 20, + "maxMemRssMb": 20 + }, + { + "group": "plugin", + "processCount": 2, + "pids": "22222, 22223", + "cpuPercent": 40, + "memRssMb": 20, + "meanCpuPercent": 40, + "meanMemRssMb": 20, + "maxMemRssMb": 20 + }, + { + "group": "browser", + "processCount": 2, + "pids": "11111, 11112", + "cpuPercent": 40, + "memRssMb": 20, + "meanCpuPercent": 40, + "meanMemRssMb": 20, + "maxMemRssMb": 20 + }, + { + "group": "ffmpeg", + "processCount": 1, + "pids": "33333", + "cpuPercent": 20, + "memRssMb": 10, + "meanCpuPercent": 20, + "meanMemRssMb": 10, + "maxMemRssMb": 10 + }, + { + "group": "desktop-gui", + "processCount": 1, + "pids": "77777", + "cpuPercent": 20, + "memRssMb": 10, + "meanCpuPercent": 20, + "meanMemRssMb": 10, + "maxMemRssMb": 10 + }, + { + "group": "cypress", + "processCount": 1, + "pids": "111111111", + "cpuPercent": 20, + "memRssMb": 10, + "meanCpuPercent": 20, + "meanMemRssMb": 10, + "maxMemRssMb": 10 + }, + { + "group": "TOTAL", + "processCount": 13, + "pids": "-", + "cpuPercent": 260, + "memRssMb": 130, + "meanCpuPercent": 260, + "meanMemRssMb": 130, + "maxMemRssMb": 130 + } +] diff --git a/packages/server/lib/util/process_profiler.ts b/packages/server/lib/util/process_profiler.ts index a44c4668d655..15bad220880e 100644 --- a/packages/server/lib/util/process_profiler.ts +++ b/packages/server/lib/util/process_profiler.ts @@ -6,7 +6,6 @@ import { concatStream } from '@packages/network' const browsers = require('../browsers') const plugins = require('../plugins') -const videoCapture = require('../video_capture') type Group = 'browser' | 'cypress' | 'plugin' | 'desktop-gui' | 'ffmpeg' | 'electron-shared' | 'other' type Process = si.Systeminformation.ProcessesProcessData & { @@ -21,6 +20,10 @@ let started = false let groupsOverTime = {} +export const _reset = () => { + groupsOverTime = {} +} + const formatPidDisplay = (groupedProcesses) => { const pids = _.map(groupedProcesses, 'pid') const maxArrayLength = 6 @@ -74,9 +77,14 @@ export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) } return isType('broker') - || isType('gpu-process') - || isType('utility') - || isType('zygote') + || isType('gpu-process') + || isType('utility') + || isType('zygote') + } + + const isFfmpegProcess = (proc: Process): boolean => { + return proc.parentPid === thisProcess.pid + && /ffmpeg/i.test(proc.name) } const getProcessGroup = (proc: Process): Group => { @@ -96,7 +104,7 @@ export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) return 'desktop-gui' } - if (proc.pid === videoCapture.getFfmpegPid()) { + if (isFfmpegProcess(proc)) { return 'ffmpeg' } @@ -127,7 +135,7 @@ export const _groupCyProcesses = ({ list }: si.Systeminformation.ProcessesData) return cyProcesses } -export const _renameBrowserGroup = (processes) => { +export const _renameBrowserGroup = (processes: Process[]) => { const instance = browsers.getBrowserInstance() const displayName = _.get(instance, 'browser.displayName') @@ -144,17 +152,9 @@ export const _renameBrowserGroup = (processes) => { return processes } -export const _printGroupedProcesses = (processes) => { +export const _aggregateGroups = (processes: Process[]) => { debugVerbose('all Cypress-launched processes: %s', require('util').inspect(processes)) - const consoleBuffer = concatStream((buf) => { - // get rid of trailing newline - debug(String(buf).trim()) - }) - - // eslint-disable-next-line no-console - const buffedConsole = new console.Console(consoleBuffer) - const groupTotals = _.chain(processes) .groupBy('group') .mapValues((groupedProcesses, group) => { @@ -171,7 +171,6 @@ export const _printGroupedProcesses = (processes) => { .reverse() .value() - buffedConsole.log('current & mean memory and CPU usage by process group:') groupTotals.push(_.reduce(groupTotals, (acc, val) => { acc.processCount += val.processCount acc.cpuPercent += val.cpuPercent @@ -203,6 +202,19 @@ export const _printGroupedProcesses = (processes) => { }) }) + return groupTotals +} + +export const _printGroupedProcesses = (groupTotals) => { + const consoleBuffer = concatStream((buf) => { + // get rid of trailing newline + debug(String(buf).trim()) + }) + + // eslint-disable-next-line no-console + const buffedConsole = new console.Console(consoleBuffer) + + buffedConsole.log('current & mean memory and CPU usage by process group:') buffedConsole.table(groupTotals, [ 'group', 'processCount', @@ -221,6 +233,7 @@ function _checkProcesses () { return si.processes() .then(_groupCyProcesses) .then(_renameBrowserGroup) + .then(_aggregateGroups) .then(_printGroupedProcesses) .then(_scheduleProcessCheck) .catch((err) => { diff --git a/packages/server/lib/video_capture.js b/packages/server/lib/video_capture.js index 956f869a0ec6..cb87d949a039 100644 --- a/packages/server/lib/video_capture.js +++ b/packages/server/lib/video_capture.js @@ -11,8 +11,6 @@ const fs = require('./util/fs') // extra verbose logs for logging individual frames const debugFrames = require('debug')('cypress-verbose:server:video:frames') -let currentPid - debug('using ffmpeg from %s', ffmpegPath) ffmpeg.setFfmpegPath(ffmpegPath) @@ -29,10 +27,6 @@ const deferredPromise = function () { } module.exports = { - getFfmpegPid () { - return currentPid - }, - getMsFromDuration (duration) { return utils.timemarkToSeconds(duration) * 1000 }, @@ -139,8 +133,6 @@ module.exports = { .videoCodec('libx264') .outputOptions('-preset ultrafast') .on('start', (command) => { - currentPid = cmd.ffmpegProc.pid - debug('capture started %o', { command }) return resolve({ @@ -188,7 +180,7 @@ module.exports = { debug('processing video from %s to %s video compression %o', name, cname, videoCompression) - const cmd = ffmpeg() + ffmpeg() .input(name) .videoCodec('libx264') .outputOptions([ @@ -196,8 +188,6 @@ module.exports = { `-crf ${videoCompression}`, ]) .on('start', (command) => { - currentPid = cmd.ffmpegProc.pid - return debug('compression started %o', { command }) }).on('codecData', (data) => { debug('compression codec data: %o', data) diff --git a/packages/server/test/unit/util/process_profiler_spec.ts b/packages/server/test/unit/util/process_profiler_spec.ts index 090aba659b16..e55679c65c17 100644 --- a/packages/server/test/unit/util/process_profiler_spec.ts +++ b/packages/server/test/unit/util/process_profiler_spec.ts @@ -3,12 +3,17 @@ import '../../spec_helper' import _ from 'lodash' import si from 'systeminformation' import { expect } from 'chai' -import { _groupCyProcesses, _renameBrowserGroup } from '../../../lib/util/process_profiler' +import { + _groupCyProcesses, + _renameBrowserGroup, + _aggregateGroups, + _reset, +} from '../../../lib/util/process_profiler' import sinon from 'sinon' +import snapshot from 'snap-shot-it' const browsers = require('../../../lib/browsers') const plugins = require('../../../lib/plugins') -const videoCapture = require('../../../lib/video_capture') const BROWSER_PID = 11111 const SUB_BROWSER_PID = 11112 @@ -107,11 +112,14 @@ const PROCESSES: Partial[] = [ ] describe('lib/util/process_profiler', function () { + beforeEach(() => { + _reset() + }) + context('._groupCyProcesses', () => { it('groups correctly', () => { sinon.stub(browsers, 'getBrowserInstance').returns({ pid: BROWSER_PID }) sinon.stub(plugins, 'getPluginPid').returns(PLUGIN_PID) - sinon.stub(videoCapture, 'getFfmpegPid').returns(FFMPEG_PID) // @ts-ignore const groupedProcesses = _groupCyProcesses({ list: PROCESSES }) @@ -156,7 +164,33 @@ describe('lib/util/process_profiler', function () { { group: 'FooBrowser', pid: 2 }, ] + // @ts-ignore expect(_renameBrowserGroup(processes)).to.deep.eq(expected) }) }) + + context('._aggregateGroups', () => { + it('aggregates groups as expected', () => { + sinon.stub(browsers, 'getBrowserInstance').returns({ pid: BROWSER_PID }) + sinon.stub(plugins, 'getPluginPid').returns(PLUGIN_PID) + + const processes = _.cloneDeep(PROCESSES) + .map((proc) => { + // add some dummy measurements so there is data to aggregate + proc.mem_rss = 10 * 1024 // 10mb + proc.pcpu = 20 + + return proc + }) + + // @ts-ignore + const result = _aggregateGroups(_groupCyProcesses({ list: processes })) + + // main process will have variable pid, replace it w constant for snapshotting + _.find(result, { pids: String(MAIN_PID) }).pids = '111111111' + + // @ts-ignore + snapshot(result) + }) + }) })