Refactor test entry by runner

Over the last few years we've increased the number of test runners.
Entry points by test type have become too lage of a category for unique
names, so this moves top level test scripts under yarn and replaces test
types with the runner name.

e.g. `yarn test:browser` -> `yarn test:karma`

Closes elastic#41133

CONTRIBUTING.md

@@ -193,7 +193,7 @@ yarn es snapshot
 ##### Keeping data between snapshots
 
 If you want to keep the data inside your Elasticsearch between usages of this command,
-you should use the following command, to keep your data folder outside the downloaded snapshot 
+you should use the following command, to keep your data folder outside the downloaded snapshot
 folder:
 
 ```bash
@@ -490,37 +490,19 @@ Test runner arguments:
 
 The standard `yarn test` task runs several sub tasks and can take several minutes to complete, making debugging failures pretty painful. In order to ease the pain specialized tasks provide alternate methods for running the tests.
 
-To execute both server and browser tests, but skip linting, use `yarn test:quick`.
-
-```bash
-yarn test:quick
-```
-
-Use `yarn test:mocha` when you want to run the mocha tests.
-
-```bash
-yarn test:mocha
-```
-
-When you'd like to execute individual server-side test files, you can use the command below. Note that this command takes care of configuring Mocha with Babel compilation for you, and you'll be better off avoiding a globally installed `mocha` package. This command is great for development and for quickly identifying bugs.
-
-```bash
-node scripts/mocha <file>
-```
-
 You could also add the `--debug` option so that `node` is run using the `--debug-brk` flag. You'll need to connect a remote debugger such as [`node-inspector`](https://github.com/node-inspector/node-inspector) to proceed in this mode.
 
 ```bash
 node scripts/mocha --debug <file>
 ```
 
-With `yarn test:browser`, you can run only the browser tests. Coverage reports are available for browser tests by running `yarn test:coverage`. You can find the results under the `coverage/` directory that will be created upon completion.
+With `yarn test:karma`, you can run only the browser tests. Coverage reports are available for browser tests by running `yarn test:coverage`. You can find the results under the `coverage/` directory that will be created upon completion.
 
 ```bash
-yarn test:browser
+yarn test:karma
 ```
 
-Using `yarn test:dev` initializes an environment for debugging the browser tests. Includes an dedicated instance of the kibana server for building the test bundle, and a karma server. When running this task the build is optimized for the first time and then a karma-owned instance of the browser is opened. Click the "debug" button to open a new tab that executes the unit tests.
+Using `yarn test:karma:debug` initializes an environment for debugging the browser tests. Includes an dedicated instance of the kibana server for building the test bundle, and a karma server. When running this task the build is optimized for the first time and then a karma-owned instance of the browser is opened. Click the "debug" button to open a new tab that executes the unit tests.
 
 ```bash
 yarn test:dev

package.json

@@ -40,23 +40,21 @@
     "kbn": "node scripts/kbn",
     "es": "node scripts/es",
     "test": "grunt test",
-    "test:dev": "grunt test:dev",
-    "test:quick": "grunt test:quick",
-    "test:browser": "grunt test:browser",
+    "test:karma": "grunt test:karma",
+    "test:karma:debug": "grunt test:karmaDebug",
     "test:jest": "node scripts/jest",
     "test:mocha": "node scripts/mocha",
     "test:mocha:coverage": "grunt test:mochaCoverage",
-    "test:ui": "node scripts/functional_tests",
-    "test:ui:server": "node scripts/functional_tests_server",
-    "test:ui:runner": "node scripts/functional_test_runner",
+    "test:ftr": "node scripts/functional_tests",
+    "test:ftr:server": "node scripts/functional_tests_server",
+    "test:ftr:runner": "node scripts/functional_test_runner",
     "test:coverage": "grunt test:coverage",
     "typespec": "typings-tester --config x-pack/legacy/plugins/canvas/public/lib/aeroelastic/tsconfig.json x-pack/legacy/plugins/canvas/public/lib/aeroelastic/__fixtures__/typescript/typespec_tests.ts",
     "checkLicenses": "node scripts/check_licenses --dev",
     "build": "node scripts/build --all-platforms",
     "start": "node --trace-warnings --throw-deprecation scripts/kibana --dev",
     "debug": "node --nolazy --inspect scripts/kibana --dev",
     "debug-break": "node --nolazy --inspect-brk scripts/kibana --dev",
-    "karma": "karma start",
     "lint": "yarn run lint:es && yarn run lint:sass",
     "lint:es": "node scripts/eslint",
     "lint:sass": "node scripts/sasslint",

packages/kbn-plugin-generator/integration_tests/generate_plugin.test.js

@@ -69,7 +69,7 @@ describe(`running the plugin-generator via 'node scripts/generate_plugin.js plug
   });
 
   describe(`then running`, () => {
-    it(`'yarn test:browser' should exit 0`, async () => {
+    it(`'yarn test:karma' should exit 0`, async () => {
       await execa('yarn', ['test:browser'], {
         cwd: generatedPath,
         env: {
@@ -78,8 +78,8 @@ describe(`running the plugin-generator via 'node scripts/generate_plugin.js plug
       });
     });
 
-    it(`'yarn test:server' should exit 0`, async () => {
-      await execa('yarn', ['test:server'], {
+    it(`'yarn test:mocha' should exit 0`, async () => {
+      await execa('yarn', ['test:mocha'], {
         cwd: generatedPath,
         env: {
           DISABLE_JUNIT_REPORTER: '1',

packages/kbn-plugin-generator/sao_template/template/README.md

@@ -28,7 +28,7 @@ See the [kibana contributing guide](https://github.com/elastic/kibana/blob/maste
 
     Build a distributable archive of your plugin.
 
-  - `yarn test:browser`
+  - `yarn test:karma`
 
     Run the browser tests in a real web browser.
 

packages/kbn-plugin-generator/sao_template/template/package_template.json

@@ -13,8 +13,8 @@
     "es": "node ../../scripts/es",
     "lint": "eslint .",
     "start": "plugin-helpers start",
-    "test:server": "plugin-helpers test:server",
-    "test:browser": "plugin-helpers test:browser",
+    "test:mocha": "plugin-helpers test:mocha",
+    "test:karma": "plugin-helpers test:karma",
     "build": "plugin-helpers build"
   },
   <%_ if (generateTranslations) { _%>

packages/kbn-plugin-helpers/README.md

@@ -30,8 +30,8 @@ $ plugin-helpers help
     start                       Start kibana and have it include this plugin
     build [options] [files...]  Build a distributable archive
     test                        Run the server and browser tests
-    test:browser [options]      Run the browser tests in a real web browser
-    test:server [files...]      Run the server tests using mocha
+    test:karma [options]      Run the browser tests in a real web browser
+    test:mocha [files...]      Run the server tests using mocha
 
   Options:
 

packages/kbn-plugin-helpers/cli.js

@@ -66,24 +66,24 @@ program
   .action(createCommanderAction('testAll'));
 
 program
-  .command('test:browser')
+  .command('test:karma')
   .description('Run the browser tests in a real web browser')
   .option('--dev', 'Enable dev mode, keeps the test server running')
   .option('-p, --plugins <plugin-ids>', "Manually specify which plugins' test bundles to run")
-  .on('--help', docs('test/browser'))
+  .on('--help', docs('test/karma'))
   .action(
-    createCommanderAction('testBrowser', command => ({
+    createCommanderAction('testKarma', command => ({
       dev: Boolean(command.dev),
       plugins: command.plugins,
     }))
   );
 
 program
-  .command('test:server [files...]')
+  .command('test:mocha [files...]')
   .description('Run the server tests using mocha')
-  .on('--help', docs('test/server'))
+  .on('--help', docs('test/mocha'))
   .action(
-    createCommanderAction('testServer', (command, files) => ({
+    createCommanderAction('testMocha', (command, files) => ({
       files: files,
     }))
   );

packages/kbn-plugin-helpers/lib/index.d.ts

@@ -21,6 +21,6 @@ export function babelRegister(): void;
 export function resolveKibanaPath(path: string): string;
 export function readFtrConfigFile(path: string): any;
 export function run(
-  task: 'build' | 'start' | 'testAll' | 'testBrowser' | 'testServer' | 'postinstall',
+  task: 'build' | 'start' | 'testAll' | 'testBrowser' | 'testMocha' | 'postinstall',
   options: any
 ): Promise<void>;

packages/kbn-plugin-helpers/lib/tasks.js

@@ -20,15 +20,15 @@
 const buildTask = require('../tasks/build');
 const startTask = require('../tasks/start');
 const testAllTask = require('../tasks/test/all');
-const testBrowserTask = require('../tasks/test/browser');
-const testServerTask = require('../tasks/test/server');
+const testKarmaTask = require('../tasks/test/karma');
+const testMochaTask = require('../tasks/test/mocha');
 const postinstallTask = require('../tasks/postinstall');
 
 module.exports = {
   build: buildTask,
   start: startTask,
   testAll: testAllTask,
-  testBrowser: testBrowserTask,
-  testServer: testServerTask,
+  testKarma: testKarmaTask,
+  testMocha: testMochaTask,
   postinstall: postinstallTask,
 };

packages/kbn-plugin-helpers/tasks/test/all/README.md

@@ -1,3 +1,3 @@
-Runs both the server and browser tests, in that order. 
+Runs both the mocha and karma tests, in that order.
 
-This is just a simple caller to both `test/server` and `test/browser`
+This is just a simple caller to both `test/mocha` and `test/karma`

packages/kbn-plugin-helpers/tasks/test/all/test_all_action.js

@@ -18,6 +18,6 @@
  */
 
 module.exports = function testAllAction(plugin, run) {
-  run('testServer');
-  run('testBrowser');
+  run('testMocha');
+  run('testKarma');
 };

packages/kbn-plugin-helpers/tasks/test/browser/test_browser_action.js → packages/kbn-plugin-helpers/tasks/test/karma/test_karma_action.js

@@ -20,7 +20,7 @@
 const execFileSync = require('child_process').execFileSync;
 const winCmd = require('../../../lib/win_cmd');
 
-module.exports = function testBrowserAction(plugin, run, options) {
+module.exports = function testKarmaAction(plugin, run, options) {
   options = options || {};
 
   const kbnServerArgs = ['--kbnServer.plugin-path=' + plugin.root];
@@ -31,7 +31,7 @@ module.exports = function testBrowserAction(plugin, run, options) {
     kbnServerArgs.push('--kbnServer.tests_bundle.pluginId=' + plugin.id);
   }
 
-  const task = options.dev ? 'test:dev' : 'test:browser';
+  const task = options.dev ? 'test:karma:debug' : 'test:karma';
   const args = [task].concat(kbnServerArgs);
   execFileSync(winCmd('yarn'), args, {
     cwd: plugin.kibanaRoot,

packages/kbn-plugin-helpers/tasks/test/server/README.md → packages/kbn-plugin-helpers/tasks/test/mocha/README.md

@@ -20,14 +20,14 @@ Server tests are written just like browser tests, they are just executed differe
 running the tests
 =================
 
-Running the server tests is simple, just execute `yarn test:server` in your terminal
+Running the server tests is simple, just execute `yarn test:mocha` in your terminal
 and all of the tests in your server will be run.
 
 By default, the runner will look for tests in `server/**/__tests__/**/*.js`. If you'd prefer to
-use a different collection of globs and files, you can specify them after the `yarn test:server`
+use a different collection of globs and files, you can specify them after the `yarn test:mocha`
 task, like so:
 
-`yarn test:server 'plugins/myplugins/server/__tests__/**/*.js'`
+`yarn test:mocha 'plugins/myplugins/server/__tests__/**/*.js'`
 
 NOTE: quoting the glob pattern is not required, but helps to avoid issues with globbing expansion
 in your shell.

tasks/config/karma.js

@@ -176,7 +176,7 @@ module.exports = function(grunt) {
    *  (&shard_num=Y), are added to the testing bundle url and read by the
    *  test_harness/setup_test_sharding[1] module. This allows us to use a
    *  different number of shards in different scenarios (ie. running
-   *  `yarn test:browser` runs the tests in a single shard, effectively
+   *  `yarn test:karma` runs the tests in a single shard, effectively
    *  disabling sharding)
    *
    *  These same parameters can also be defined in the URL/query string of the

tasks/config/run.js

@@ -54,7 +54,7 @@ module.exports = function(grunt) {
     };
   }
 
-  const browserTestServerFlags = [
+  const karmaTestServerFlags = [
     '--env.name=development',
     '--plugins.initialize=false',
     '--optimize.bundleFilter=tests',
@@ -161,25 +161,25 @@ module.exports = function(grunt) {
 
     // used by the test:browser task
     //    runs the kibana server to serve the browser test bundle
-    browserTestServer: createKbnServerTask({
-      flags: [...browserTestServerFlags],
+    karmaTestServer: createKbnServerTask({
+      flags: [...karmaTestServerFlags],
     }),
     browserSCSS: createKbnServerTask({
-      flags: [...browserTestServerFlags, '--optimize', '--optimize.enabled=false'],
+      flags: [...karmaTestServerFlags, '--optimize', '--optimize.enabled=false'],
     }),
 
     // used by the test:coverage task
     //    runs the kibana server to serve the instrumented version of the browser test bundle
-    browserTestCoverageServer: createKbnServerTask({
-      flags: [...browserTestServerFlags, '--tests_bundle.instrument=true'],
+    karmaTestCoverageServer: createKbnServerTask({
+      flags: [...karmaTestServerFlags, '--tests_bundle.instrument=true'],
     }),
 
     // used by the test:dev task
     //    runs the kibana server to serve the browser test bundle, but listens for changes
     //    to the public/browser code and rebuilds the test bundle on changes
-    devBrowserTestServer: createKbnServerTask({
+    karmaTestDebugServer: createKbnServerTask({
       flags: [
-        ...browserTestServerFlags,
+        ...karmaTestServerFlags,
         '--dev',
         '--no-dev-config',
         '--no-watch',
@@ -293,7 +293,7 @@ module.exports = function(grunt) {
       'test:jest_integration'
     ),
     test_projects: gruntTaskWithGithubChecks('Project tests', 'test:projects'),
-    test_browser_ci: gruntTaskWithGithubChecks('Browser tests', 'test:browser-ci'),
+    test_karma_ci: gruntTaskWithGithubChecks('Browser tests', 'test:karma-ci'),
 
     ...getFunctionalTestGroupRunConfigs({
       kibanaInstallDir: KIBANA_INSTALL_DIR,

tasks/jenkins.js

@@ -35,7 +35,7 @@ module.exports = function(grunt) {
     'run:test_jest',
     'run:test_jest_integration',
     'run:test_projects',
-    'run:test_browser_ci',
+    'run:test_karma_ci',
     'run:apiIntegrationTests',
   ]);
 };

tasks/test.js

@@ -31,21 +31,21 @@ module.exports = function(grunt) {
     }
   );
 
-  grunt.registerTask('test:browser', [
+  grunt.registerTask('test:karma', [
     'checkPlugins',
     'run:browserSCSS',
-    'run:browserTestServer',
+    'run:karmaTestServer',
     'karma:unit',
   ]);
 
-  grunt.registerTask('test:browser-ci', () => {
+  grunt.registerTask('test:karma-ci', () => {
     const ciShardTasks = keys(grunt.config.get('karma'))
       .filter(key => key.startsWith('ciShard-'))
       .map(key => `karma:${key}`);
 
     grunt.log.ok(`Running UI tests in ${ciShardTasks.length} shards`);
     grunt.task.run(['run:browserSCSS']);
-    grunt.task.run(['run:browserTestServer', ...ciShardTasks]);
+    grunt.task.run(['run:karmaTestServer', ...ciShardTasks]);
   });
 
   grunt.registerTask('test:coverage', ['run:testCoverageServer', 'karma:coverage']);
@@ -61,7 +61,7 @@ module.exports = function(grunt) {
     'run:apiIntegrationTests',
   ]);
 
-  grunt.registerTask('test:dev', ['checkPlugins', 'run:devBrowserTestServer', 'karma:dev']);
+  grunt.registerTask('test:karmaDebug', ['checkPlugins', 'run:karmaDebugServer', 'karma:dev']);
   grunt.registerTask('test:mochaCoverage', ['run:mochaCoverage']);
 
   grunt.registerTask('test', subTask => {

test/scripts/jenkins_xpack.sh

@@ -7,29 +7,29 @@ export TEST_BROWSER_HEADLESS=1
 if [[ -z "$CODE_COVERAGE" ]] ; then
   echo " -> Running mocha tests"
   cd "$XPACK_DIR"
-  checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser
+  checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:karma
   echo ""
   echo ""
-    
+
   echo " -> Running jest tests"
   cd "$XPACK_DIR"
   checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose
   echo ""
   echo ""
-    
+
   echo " -> Running SIEM cyclic dependency test"
   cd "$XPACK_DIR"
   checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps
   echo ""
   echo ""
-    
+
   # FAILING: https://github.com/elastic/kibana/issues/44250
   # echo " -> Running jest contracts tests"
   # cd "$XPACK_DIR"
   # SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose
   # echo ""
   # echo ""
-    
+
   # echo " -> Running jest integration tests"
   # cd "$XPACK_DIR"
   # node scripts/jest_integration --ci --verbose