Merge branch 'master' into remove-unnecessary-configs-kp

docs/api/saved-objects/create.asciidoc

@@ -13,7 +13,7 @@ experimental[] Create {kib} saved objects.
 
 `POST <kibana host>:<port>/api/saved_objects/<type>/<id>`
 
-`POST <kibana host>:<port>/s/<space_id>/api/saved_objects/<type>`
+`POST <kibana host>:<port>/s/<space_id>/saved_objects/<type>`
 
 [[saved-objects-api-create-path-params]]
 ==== Path parameters

docs/developer/getting-started/development-plugin-resources.asciidoc

@@ -33,7 +33,7 @@ To enable TypeScript support, create a `tsconfig.json` file at the root of your
 ["source","js"]
 -----------
 {
-  // extend {kib}'s tsconfig, or use your own settings
+  // extend Kibana's tsconfig, or use your own settings
   "extends": "../../kibana/tsconfig.json",
 
   // tell the TypeScript compiler where to find your source files

package.json

@@ -227,9 +227,9 @@
   "devDependencies": {
     "@babel/parser": "^7.11.2",
     "@babel/types": "^7.11.0",
-    "@elastic/apm-rum": "^5.5.0",
+    "@elastic/apm-rum": "^5.6.0",
     "@elastic/charts": "21.1.2",
-    "@elastic/ems-client": "7.9.3",
+    "@elastic/ems-client": "7.10.0",
     "@elastic/eslint-config-kibana": "0.15.0",
     "@elastic/eslint-plugin-eui": "0.0.2",
     "@elastic/filesaver": "1.1.2",

packages/kbn-pm/src/commands/bootstrap.test.ts

@@ -19,6 +19,7 @@
 
 jest.mock('../utils/scripts');
 jest.mock('../utils/link_project_executables');
+jest.mock('../utils/validate_yarn_lock');
 
 import { resolve } from 'path';
 

packages/kbn-pm/src/commands/bootstrap.ts

@@ -25,6 +25,8 @@ import { Project } from '../utils/project';
 import { ICommand } from './';
 import { getAllChecksums } from '../utils/project_checksums';
 import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
+import { readYarnLock } from '../utils/yarn_lock';
+import { validateYarnLock } from '../utils/validate_yarn_lock';
 
 export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
@@ -54,6 +56,10 @@ export const BootstrapCommand: ICommand = {
       }
     }
 
+    const yarnLock = await readYarnLock(kbn);
+
+    await validateYarnLock(kbn, yarnLock);
+
     await linkProjectExecutables(projects, projectGraph);
 
     /**
@@ -63,7 +69,7 @@ export const BootstrapCommand: ICommand = {
      * have to, as it will slow down the bootstrapping process.
      */
 
-    const checksums = await getAllChecksums(kbn, log);
+    const checksums = await getAllChecksums(kbn, log, yarnLock);
     const caches = new Map<Project, { file: BootstrapCacheFile; valid: boolean }>();
     let cachedProjectCount = 0;
 

packages/kbn-pm/src/utils/fs.ts

@@ -25,6 +25,7 @@ import { promisify } from 'util';
 
 const lstat = promisify(fs.lstat);
 export const readFile = promisify(fs.readFile);
+export const writeFile = promisify(fs.writeFile);
 const symlink = promisify(fs.symlink);
 export const chmod = promisify(fs.chmod);
 const cmdShim = promisify<string, string>(cmdShimCb);

packages/kbn-pm/src/utils/kibana.ts

@@ -22,6 +22,8 @@ import Path from 'path';
 import multimatch from 'multimatch';
 import isPathInside from 'is-path-inside';
 
+import { resolveDepsForProject, YarnLock } from './yarn_lock';
+import { Log } from './log';
 import { ProjectMap, getProjects, includeTransitiveProjects } from './projects';
 import { Project } from './project';
 import { getProjectPaths } from '../config';
@@ -133,4 +135,26 @@ export class Kibana {
   isOutsideRepo(project: Project) {
     return !this.isPartOfRepo(project);
   }
+
+  resolveAllProductionDependencies(yarnLock: YarnLock, log: Log) {
+    const kibanaDeps = resolveDepsForProject({
+      project: this.kibanaProject,
+      yarnLock,
+      kbn: this,
+      includeDependentProject: true,
+      productionDepsOnly: true,
+      log,
+    })!;
+
+    const xpackDeps = resolveDepsForProject({
+      project: this.getProject('x-pack')!,
+      yarnLock,
+      kbn: this,
+      includeDependentProject: true,
+      productionDepsOnly: true,
+      log,
+    })!;
+
+    return new Map([...kibanaDeps.entries(), ...xpackDeps.entries()]);
+  }
 }

packages/kbn-pm/src/utils/project_checksums.ts

@@ -24,7 +24,7 @@ import { promisify } from 'util';
 
 import execa from 'execa';
 
-import { readYarnLock, YarnLock } from './yarn_lock';
+import { YarnLock, resolveDepsForProject } from './yarn_lock';
 import { ProjectMap } from '../utils/projects';
 import { Project } from '../utils/project';
 import { Kibana } from '../utils/kibana';
@@ -145,51 +145,6 @@ async function getLatestSha(project: Project, kbn: Kibana) {
   return stdout.trim() || undefined;
 }
 
-/**
- * Get a list of the absolute dependencies of this project, as resolved
- * in the yarn.lock file, does not include other projects in the workspace
- * or their dependencies
- */
-function resolveDepsForProject(project: Project, yarnLock: YarnLock, kbn: Kibana, log: Log) {
-  /** map of [name@range, name@resolved] */
-  const resolved = new Map<string, string>();
-
-  const queue: Array<[string, string]> = Object.entries(project.allDependencies);
-
-  while (queue.length) {
-    const [name, versionRange] = queue.shift()!;
-    const req = `${name}@${versionRange}`;
-
-    if (resolved.has(req)) {
-      continue;
-    }
-
-    if (!kbn.hasProject(name)) {
-      const pkg = yarnLock[req];
-      if (!pkg) {
-        log.warning(
-          'yarn.lock file is out of date, please run `yarn kbn bootstrap` to re-enable caching'
-        );
-        return;
-      }
-
-      const res = `${name}@${pkg.version}`;
-      resolved.set(req, res);
-
-      const allDepsEntries = [
-        ...Object.entries(pkg.dependencies || {}),
-        ...Object.entries(pkg.optionalDependencies || {}),
-      ];
-
-      for (const [childName, childVersionRange] of allDepsEntries) {
-        queue.push([childName, childVersionRange]);
-      }
-    }
-  }
-
-  return Array.from(resolved.values()).sort((a, b) => a.localeCompare(b));
-}
-
 /**
  * Get the checksum for a specific project in the workspace
  */
@@ -224,11 +179,22 @@ async function getChecksum(
       })
   );
 
-  const deps = await resolveDepsForProject(project, yarnLock, kbn, log);
-  if (!deps) {
+  const depMap = resolveDepsForProject({
+    project,
+    yarnLock,
+    kbn,
+    log,
+    includeDependentProject: false,
+    productionDepsOnly: false,
+  });
+  if (!depMap) {
     return;
   }
 
+  const deps = Array.from(depMap.values())
+    .map(({ name, version }) => `${name}@${version}`)
+    .sort((a, b) => a.localeCompare(b));
+
   log.verbose(`[${project.name}] resolved %d deps`, deps.length);
 
   const checksum = JSON.stringify(
@@ -256,10 +222,9 @@ async function getChecksum(
  *  - un-committed changes
  *  - resolved dependencies from yarn.lock referenced by project package.json
  */
-export async function getAllChecksums(kbn: Kibana, log: Log) {
+export async function getAllChecksums(kbn: Kibana, log: Log, yarnLock: YarnLock) {
   const projects = kbn.getAllProjects();
   const changesByProject = await getChangesForProjects(projects, kbn, log);
-  const yarnLock = await readYarnLock(kbn);
 
   /** map of [project.name, cacheKey] */
   const cacheKeys: ChecksumMap = new Map();

packages/kbn-pm/src/utils/validate_yarn_lock.ts

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// @ts-expect-error published types are useless
+import { stringify as stringifyLockfile } from '@yarnpkg/lockfile';
+import dedent from 'dedent';
+
+import { writeFile } from './fs';
+import { Kibana } from './kibana';
+import { YarnLock } from './yarn_lock';
+import { log } from './log';
+
+export async function validateYarnLock(kbn: Kibana, yarnLock: YarnLock) {
+  // look through all of the packages in the yarn.lock file to see if
+  // we have accidentally installed multiple lodash v4 versions
+  const lodash4Versions = new Set<string>();
+  const lodash4Reqs = new Set<string>();
+  for (const [req, dep] of Object.entries(yarnLock)) {
+    if (req.startsWith('lodash@') && dep.version.startsWith('4.')) {
+      lodash4Reqs.add(req);
+      lodash4Versions.add(dep.version);
+    }
+  }
+
+  // if we find more than one lodash v4 version installed then delete
+  // lodash v4 requests from the yarn.lock file and prompt the user to
+  // retry bootstrap so that a single v4 version will be installed
+  if (lodash4Versions.size > 1) {
+    for (const req of lodash4Reqs) {
+      delete yarnLock[req];
+    }
+
+    await writeFile(kbn.getAbsolute('yarn.lock'), stringifyLockfile(yarnLock), 'utf8');
+
+    log.error(dedent`
+
+      Multiple version of lodash v4 were detected, so they have been removed
+      from the yarn.lock file. Please rerun yarn kbn bootstrap to coalese the
+      lodash versions installed.
+
+      If you still see this error when you re-bootstrap then you might need
+      to force a new dependency to use the latest version of lodash via the
+      "resolutions" field in package.json.
+
+      If you have questions about this please reach out to the operations team.
+
+    `);
+
+    process.exit(1);
+  }
+
+  // look through all the dependencies of production packages and production
+  // dependencies of those packages to determine if we're shipping any versions
+  // of lodash v3 in the distributable
+  const prodDependencies = kbn.resolveAllProductionDependencies(yarnLock, log);
+  const lodash3Versions = new Set<string>();
+  for (const dep of prodDependencies.values()) {
+    if (dep.name === 'lodash' && dep.version.startsWith('3.')) {
+      lodash3Versions.add(dep.version);
+    }
+  }
+
+  // if any lodash v3 packages were found we abort and tell the user to fix things
+  if (lodash3Versions.size) {
+    log.error(dedent`
+
+      Due to changes in the yarn.lock file and/or package.json files a version of
+      lodash 3 is now included in the production dependencies. To reduce the size of
+      our distributable and especially our front-end bundles we have decided to
+      prevent adding any new instances of lodash 3.
+
+      Please inspect the changes to yarn.lock or package.json files to identify where
+      the lodash 3 version is coming from and remove it.
+
+      If you have questions about this please reack out to the operations team.
+
+    `);
+
+    process.exit(1);
+  }
+
+  log.success('yarn.lock analysis completed without any issues');
+}