Skip to content

Commit

Permalink
chore(NA): support bazel and kbn packages in parallel on kbn pm and o…
Browse files Browse the repository at this point in the history
…n distributable build scripts (elastic#89961)

* chore(NA): bazel projects support initial flag on kbn pm

* chore(NA): every needed step to build production bazel packages except the final function

* chore(NA): include build bazel production projects on kibana distributable build

* chore(NA): support bazel packages when creating the package.json

* chore(NA): including last changes on kbn pm and build distributable to support both bazel and kbn packages

* chore(NA): missing annotation on build bazel packages

* chore(NA): changed values on bazelrc common

* chore(NA): fix bazel common rc

* chore(NA): auto discovery if a kbn package is a Bazel package

* chore(NA): last details to make bazel packages to built on distributable scripts

* chore(NA): removed wrongly added line to x-pack package.json

* chore(NA): apply correct formating

* chore(NA): move into bazel bin

* chore(NA): merge chnages on kbn pm

* chore(NA): correctly setup ignore files for new bazel aggregated folder

* chore(NA): merge with last master

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
mistic and kibanamachine committed Feb 4, 2021
1 parent 3c95438 commit 7b55c99
Show file tree
Hide file tree
Showing 13 changed files with 1,413 additions and 150 deletions.
7 changes: 7 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
exports_files(
[
"tsconfig.json",
"package.json"
],
visibility = ["//visibility:public"]
)
5 changes: 5 additions & 0 deletions packages/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Call each package final target
filegroup(
name = "build",
srcs = [],
)
1,308 changes: 1,187 additions & 121 deletions packages/kbn-pm/dist/index.js

Large diffs are not rendered by default.

26 changes: 18 additions & 8 deletions packages/kbn-pm/src/commands/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,29 @@ import { sep } from 'path';
import { linkProjectExecutables } from '../utils/link_project_executables';
import { log } from '../utils/log';
import { parallelizeBatches } from '../utils/parallelize';
import { topologicallyBatchProjects } from '../utils/projects';
import { getNonBazelProjectsOnly, topologicallyBatchProjects } from '../utils/projects';
import { Project } from '../utils/project';
import { ICommand } from './';
import { getAllChecksums } from '../utils/project_checksums';
import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
import { readYarnLock } from '../utils/yarn_lock';
import { validateDependencies } from '../utils/validate_dependencies';
import { installBazelTools } from '../utils/bazel';
import { installBazelTools, runBazel } from '../utils/bazel';

export const BootstrapCommand: ICommand = {
description: 'Install dependencies and crosslink projects',
name: 'bootstrap',

async run(projects, projectGraph, { options, kbn, rootPath }) {
const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
const nonBazelProjectsOnly = await getNonBazelProjectsOnly(projects);
const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph);
const kibanaProjectPath = projects.get('kibana')?.path;

// Install bazel machinery tools if needed
await installBazelTools(rootPath);

// Install monorepo npm dependencies
for (const batch of batchedProjects) {
for (const batch of batchedNonBazelProjects) {
for (const project of batch) {
const isExternalPlugin = project.path.includes(`${kibanaProjectPath}${sep}plugins`);

Expand Down Expand Up @@ -62,9 +63,18 @@ export const BootstrapCommand: ICommand = {
// copy those scripts into the top level node_modules folder
await linkProjectExecutables(projects, projectGraph);

// Bootstrap process for Bazel packages
//
// NOTE: Bazel projects will be introduced incrementally
// And should begin from the ones with none dependencies forward.
// That way non bazel projects could depend on bazel projects but not the other way around
// That is only intended during the migration process while non Bazel projects are not removed at all.
await runBazel(['build', '//packages:build']);

// Bootstrap process for non Bazel packages
/**
* At the end of the bootstrapping process we call all `kbn:bootstrap` scripts
* in the list of projects. We do this because some projects need to be
* in the list of non Bazel projects. We do this because some projects need to be
* transpiled before they can be used. Ideally we shouldn't do this unless we
* have to, as it will slow down the bootstrapping process.
*/
Expand All @@ -73,8 +83,8 @@ export const BootstrapCommand: ICommand = {
const caches = new Map<Project, { file: BootstrapCacheFile; valid: boolean }>();
let cachedProjectCount = 0;

for (const project of projects.values()) {
if (project.hasScript('kbn:bootstrap')) {
for (const project of nonBazelProjectsOnly.values()) {
if (project.hasScript('kbn:bootstrap') && !project.isBazelPackage()) {
const file = new BootstrapCacheFile(kbn, project, checksums);
const valid = options.cache && file.isValid();

Expand All @@ -91,7 +101,7 @@ export const BootstrapCommand: ICommand = {
log.success(`${cachedProjectCount} bootstrap builds are cached`);
}

await parallelizeBatches(batchedProjects, async (project) => {
await parallelizeBatches(batchedNonBazelProjects, async (project) => {
const cache = caches.get(project);
if (cache && !cache.valid) {
log.info(`[${project.name}] running [kbn:bootstrap] script`);
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-pm/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
*/

export { run } from './cli';
export { buildProductionProjects } from './production';
export { buildBazelProductionProjects, buildNonBazelProductionProjects } from './production';
export { getProjects } from './utils/projects';
export { Project } from './utils/project';
export { transformDependencies } from './utils/package_json';
Expand Down
103 changes: 103 additions & 0 deletions packages/kbn-pm/src/production/build_bazel_production_projects.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import copy from 'cpy';
import globby from 'globby';
import { basename, join, relative, resolve } from 'path';

import { buildProject, getProductionProjects } from './build_non_bazel_production_projects';
import { chmod, isFile, isDirectory } from '../utils/fs';
import { log } from '../utils/log';
import {
createProductionPackageJson,
readPackageJson,
writePackageJson,
} from '../utils/package_json';
import { getBazelProjectsOnly } from '../utils/projects';
import { Project } from '..';

export async function buildBazelProductionProjects({
kibanaRoot,
buildRoot,
onlyOSS,
}: {
kibanaRoot: string;
buildRoot: string;
onlyOSS?: boolean;
}) {
const projects = await getBazelProjectsOnly(await getProductionProjects(kibanaRoot, onlyOSS));

const projectNames = [...projects.values()].map((project) => project.name);
log.info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`);

for (const project of projects.values()) {
await buildProject(project);
await copyToBuild(project, kibanaRoot, buildRoot);
await applyCorrectPermissions(project, kibanaRoot, buildRoot);
}
}

/**
* Copy all the project's files from its Bazel dist directory into the
* project build folder.
*
* When copying all the files into the build, we exclude `node_modules` because
* we want the Kibana build to be responsible for actually installing all
* dependencies. The primary reason for allowing the Kibana build process to
* manage dependencies is that it will "dedupe" them, so we don't include
* unnecessary copies of dependencies. We also exclude every related Bazel build
* files in order to get the most cleaner package module we can in the final distributable.
*/
async function copyToBuild(project: Project, kibanaRoot: string, buildRoot: string) {
// We want the package to have the same relative location within the build
const relativeProjectPath = relative(kibanaRoot, project.path);
const buildProjectPath = resolve(buildRoot, relativeProjectPath);

const bazelFilesToExclude = ['!*.params', '!*_mappings.json', '!*_options.optionsvalid.d.ts'];
await copy(['**/*', '!node_modules/**', ...bazelFilesToExclude], buildProjectPath, {
cwd: join(kibanaRoot, 'bazel', 'bin', 'packages', basename(buildProjectPath)),
dot: true,
onlyFiles: true,
parents: true,
} as copy.Options);

// If a project is using an intermediate build directory, we special-case our
// handling of `package.json`, as the project build process might have copied
// (a potentially modified) `package.json` into the intermediate build
// directory already. If so, we want to use that `package.json` as the basis
// for creating the production-ready `package.json`. If it's not present in
// the intermediate build, we fall back to using the project's already defined
// `package.json`.
const packageJson = (await isFile(join(buildProjectPath, 'package.json')))
? await readPackageJson(buildProjectPath)
: project.json;

const preparedPackageJson = createProductionPackageJson(packageJson);
await writePackageJson(buildProjectPath, preparedPackageJson);
}

async function applyCorrectPermissions(project: Project, kibanaRoot: string, buildRoot: string) {
const relativeProjectPath = relative(kibanaRoot, project.path);
const buildProjectPath = resolve(buildRoot, relativeProjectPath);
const allPluginPaths = await globby([`**/*`], {
onlyFiles: false,
cwd: join(kibanaRoot, 'bazel', 'bin', 'packages', basename(buildProjectPath)),
dot: true,
});

for (const pluginPath of allPluginPaths) {
const resolvedPluginPath = resolve(buildRoot, pluginPath);
if (await isFile(resolvedPluginPath)) {
await chmod(resolvedPluginPath, 0o644);
}

if (await isDirectory(resolvedPluginPath)) {
await chmod(resolvedPluginPath, 0o755);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@ import {
} from '../utils/package_json';
import {
buildProjectGraph,
getNonBazelProjectsOnly,
getProjects,
includeTransitiveProjects,
topologicallyBatchProjects,
} from '../utils/projects';
import { Project } from '..';

export async function buildProductionProjects({
export async function buildNonBazelProductionProjects({
kibanaRoot,
buildRoot,
onlyOSS,
Expand All @@ -35,12 +36,12 @@ export async function buildProductionProjects({
buildRoot: string;
onlyOSS?: boolean;
}) {
const projects = await getProductionProjects(kibanaRoot, onlyOSS);
const projects = await getNonBazelProjectsOnly(await getProductionProjects(kibanaRoot, onlyOSS));
const projectGraph = buildProjectGraph(projects);
const batchedProjects = topologicallyBatchProjects(projects, projectGraph);

const projectNames = [...projects.values()].map((project) => project.name);
log.info(`Preparing production build for [${projectNames.join(', ')}]`);
log.info(`Preparing non Bazel production build for [${projectNames.join(', ')}]`);

for (const batch of batchedProjects) {
for (const project of batch) {
Expand All @@ -58,7 +59,7 @@ export async function buildProductionProjects({
* we only include Kibana's transitive _production_ dependencies. If onlyOSS
* is supplied, we omit projects with build.oss in their package.json set to false.
*/
async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
export async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
const projectPaths = getProjectPaths({ rootPath });
const projects = await getProjects(rootPath, projectPaths);
const projectsSubset = [projects.get('kibana')!];
Expand Down Expand Up @@ -93,7 +94,7 @@ async function deleteTarget(project: Project) {
}
}

async function buildProject(project: Project) {
export async function buildProject(project: Project) {
if (project.hasScript('build')) {
await project.runScript('build');
}
Expand Down
3 changes: 2 additions & 1 deletion packages/kbn-pm/src/production/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@
* Side Public License, v 1.
*/

export { buildProductionProjects } from './build_production_projects';
export { buildBazelProductionProjects } from './build_bazel_production_projects';
export { buildNonBazelProductionProjects } from './build_non_bazel_production_projects';
20 changes: 17 additions & 3 deletions packages/kbn-pm/src/utils/package_json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ export const createProductionPackageJson = (pkgJson: IPackageJson) => ({

export const isLinkDependency = (depVersion: string) => depVersion.startsWith('link:');

export const isBazelPackageDependency = (depVersion: string) =>
depVersion.startsWith('link:bazel/bin/');

/**
* Replaces `link:` dependencies with `file:` dependencies. When installing
* dependencies, these `file:` dependencies will be copied into `node_modules`
Expand All @@ -42,16 +45,27 @@ export const isLinkDependency = (depVersion: string) => depVersion.startsWith('l
* This will allow us to copy packages into the build and run `yarn`, which
* will then _copy_ the `file:` dependencies into `node_modules` instead of
* symlinking like we do in development.
*
* Additionally it also taken care of replacing `link:bazel/bin/` with
* `file:` so we can also support the copy of the Bazel packages dist already into
* build/packages to be copied into the node_modules
*/
export function transformDependencies(dependencies: IPackageDependencies = {}) {
const newDeps: IPackageDependencies = {};
for (const name of Object.keys(dependencies)) {
const depVersion = dependencies[name];
if (isLinkDependency(depVersion)) {
newDeps[name] = depVersion.replace('link:', 'file:');
} else {

if (!isLinkDependency(depVersion)) {
newDeps[name] = depVersion;
continue;
}

if (isBazelPackageDependency(depVersion)) {
newDeps[name] = depVersion.replace('link:bazel/bin/', 'file:');
continue;
}

newDeps[name] = depVersion.replace('link:', 'file:');
}
return newDeps;
}
28 changes: 22 additions & 6 deletions packages/kbn-pm/src/utils/project.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* Side Public License, v 1.
*/

import Fs from 'fs';
import Path from 'path';
import { inspect } from 'util';

Expand Down Expand Up @@ -56,6 +57,8 @@ export class Project {
public readonly devDependencies: IPackageDependencies;
/** scripts defined in the package.json file for the project [name => body] */
public readonly scripts: IPackageScripts;
/** states if this project is a Bazel package */
public readonly bazelPackage: boolean;

public isSinglePackageJsonProject = false;

Expand All @@ -77,6 +80,9 @@ export class Project {
this.isSinglePackageJsonProject = this.json.name === 'kibana';

this.scripts = this.json.scripts || {};

this.bazelPackage =
!this.isSinglePackageJsonProject && Fs.existsSync(Path.resolve(this.path, 'BUILD.bazel'));
}

public get name(): string {
Expand All @@ -85,21 +91,27 @@ export class Project {

public ensureValidProjectDependency(project: Project) {
const relativePathToProject = normalizePath(Path.relative(this.path, project.path));
const relativePathToProjectIfBazelPkg = normalizePath(
Path.relative(this.path, `bazel/bin/packages/${Path.basename(project.path)}`)
);

const versionInPackageJson = this.allDependencies[project.name];
const expectedVersionInPackageJson = `link:${relativePathToProject}`;

// TODO: after introduce bazel to build packages do not allow child projects
// to hold dependencies

if (versionInPackageJson === expectedVersionInPackageJson) {
const expectedVersionInPackageJsonIfBazelPkg = `link:${relativePathToProjectIfBazelPkg}`;

// TODO: after introduce bazel to build all the packages and completely remove the support for kbn packages
// do not allow child projects to hold dependencies
if (
versionInPackageJson === expectedVersionInPackageJson ||
versionInPackageJson === expectedVersionInPackageJsonIfBazelPkg
) {
return;
}

const updateMsg = 'Update its package.json to the expected value below.';
const meta = {
actual: `"${project.name}": "${versionInPackageJson}"`,
expected: `"${project.name}": "${expectedVersionInPackageJson}"`,
expected: `"${project.name}": "${expectedVersionInPackageJson}" or "${project.name}": "${expectedVersionInPackageJsonIfBazelPkg}"`,
package: `${this.name} (${this.packageJsonLocation})`,
};

Expand Down Expand Up @@ -133,6 +145,10 @@ export class Project {
return (this.json.kibana && this.json.kibana.clean) || {};
}

public isBazelPackage() {
return this.bazelPackage;
}

public isFlaggedAsDevOnly() {
return !!(this.json.kibana && this.json.kibana.devOnly);
}
Expand Down
Loading

0 comments on commit 7b55c99

Please sign in to comment.