diff --git a/.buildkite/scripts/steps/checks.sh b/.buildkite/scripts/steps/checks.sh index 9e335fc3cdea3..bde9e82a4d1e6 100755 --- a/.buildkite/scripts/steps/checks.sh +++ b/.buildkite/scripts/steps/checks.sh @@ -6,6 +6,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false .buildkite/scripts/bootstrap.sh .buildkite/scripts/steps/checks/commit/commit.sh +.buildkite/scripts/steps/checks/bazel_packages.sh .buildkite/scripts/steps/checks/telemetry.sh .buildkite/scripts/steps/checks/ts_projects.sh .buildkite/scripts/steps/checks/jest_configs.sh diff --git a/.buildkite/scripts/steps/checks/bazel_packages.sh b/.buildkite/scripts/steps/checks/bazel_packages.sh new file mode 100755 index 0000000000000..85268bdcb0f06 --- /dev/null +++ b/.buildkite/scripts/steps/checks/bazel_packages.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +echo --- Check Bazel Packages Manifest +node scripts/generate packages_build_manifest --validate diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index 66361060f1ee6..f01042cecfae9 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -1,6 +1,7 @@ ################ ################ -## This file is automatically generated, to create a new package use `node scripts/generate package --help` +## This file is automatically generated, to create a new package use `node scripts/generate package --help` or run +## `node scripts/generate packages_build_manifest` to regenerate it from the current state of the repo ################ ################ diff --git a/packages/kbn-bazel-packages/BUILD.bazel b/packages/kbn-bazel-packages/BUILD.bazel index 08ffbe24ba2aa..9adfe80060889 100644 --- a/packages/kbn-bazel-packages/BUILD.bazel +++ b/packages/kbn-bazel-packages/BUILD.bazel @@ -40,6 +40,7 @@ RUNTIME_DEPS = [ "//packages/kbn-utils", "//packages/kbn-std", "@npm//globby", + "@npm//normalize-path", ] # In this array place dependencies necessary to build the types, which will include the @@ -55,7 +56,9 @@ RUNTIME_DEPS = [ TYPES_DEPS = [ "//packages/kbn-utils:npm_module_types", "//packages/kbn-std:npm_module_types", + "@npm//@types/normalize-path", "@npm//globby", + "@npm//normalize-path", ] jsts_transpiler( diff --git a/packages/kbn-bazel-packages/src/bazel_package.test.ts b/packages/kbn-bazel-packages/src/bazel_package.test.ts index 884cf0e646ba0..70d540e43f06a 100644 --- a/packages/kbn-bazel-packages/src/bazel_package.test.ts +++ b/packages/kbn-bazel-packages/src/bazel_package.test.ts @@ -15,24 +15,34 @@ const OWN_BAZEL_BUILD_FILE = Fs.readFileSync(Path.resolve(__dirname, '../BUILD.b describe('hasBuildRule()', () => { it('returns true if there is a rule with the name "build"', () => { - const pkg = new BazelPackage('foo', {}, OWN_BAZEL_BUILD_FILE); + const pkg = new BazelPackage('foo', { name: 'foo' }, OWN_BAZEL_BUILD_FILE); expect(pkg.hasBuildRule()).toBe(true); }); it('returns false if there is no rule with name "build"', () => { - const pkg = new BazelPackage('foo', {}, ``); + const pkg = new BazelPackage('foo', { name: 'foo' }, ``); + expect(pkg.hasBuildRule()).toBe(false); + }); + + it('returns false if there is no BUILD.bazel file', () => { + const pkg = new BazelPackage('foo', { name: 'foo' }); expect(pkg.hasBuildRule()).toBe(false); }); }); describe('hasBuildTypesRule()', () => { it('returns true if there is a rule with the name "build_types"', () => { - const pkg = new BazelPackage('foo', {}, OWN_BAZEL_BUILD_FILE); + const pkg = new BazelPackage('foo', { name: 'foo' }, OWN_BAZEL_BUILD_FILE); expect(pkg.hasBuildTypesRule()).toBe(true); }); it('returns false if there is no rule with name "build_types"', () => { - const pkg = new BazelPackage('foo', {}, ``); + const pkg = new BazelPackage('foo', { name: 'foo' }, ``); + expect(pkg.hasBuildTypesRule()).toBe(false); + }); + + it('returns false if there is no BUILD.bazel file', () => { + const pkg = new BazelPackage('foo', { name: 'foo' }); expect(pkg.hasBuildTypesRule()).toBe(false); }); }); diff --git a/packages/kbn-bazel-packages/src/bazel_package.ts b/packages/kbn-bazel-packages/src/bazel_package.ts index 28170cb68a5d2..35950c9896faf 100644 --- a/packages/kbn-bazel-packages/src/bazel_package.ts +++ b/packages/kbn-bazel-packages/src/bazel_package.ts @@ -6,14 +6,49 @@ * Side Public License, v 1. */ +import { inspect } from 'util'; import Path from 'path'; import Fsp from 'fs/promises'; + +import normalizePath from 'normalize-path'; import { REPO_ROOT } from '@kbn/utils'; const BUILD_RULE_NAME = /(^|\s)name\s*=\s*"build"/; const BUILD_TYPES_RULE_NAME = /(^|\s)name\s*=\s*"build_types"/; +/** + * Simple parsed representation of a package.json file, validated + * by `assertParsedPackageJson()` and extensible as needed in the future + */ +export interface ParsedPackageJson { + /** + * The name of the package, usually `@kbn/`+something + */ + name: string; + /** + * All other fields in the package.json are typed as unknown as all we need at this time is "name" + */ + [key: string]: unknown; +} + +function isObj(v: unknown): v is Record { + return !!(typeof v === 'object' && v); +} + +function assertParsedPackageJson(v: unknown): asserts v is ParsedPackageJson { + if (!isObj(v) || typeof v.name !== 'string') { + throw new Error('Expected parsed package.json to be an object with at least a "name" property'); + } +} + +/** + * Representation of a Bazel Package in the Kibana repository + */ export class BazelPackage { + /** + * Create a BazelPackage object from a package directory. Reads some files from the package and returns + * a Promise for a BazelPackage instance + */ static async fromDir(dir: string) { let pkg; try { @@ -22,6 +57,8 @@ export class BazelPackage { throw new Error(`unable to parse package.json in [${dir}]: ${error.message}`); } + assertParsedPackageJson(pkg); + let buildBazelContent; if (pkg.name !== '@kbn/pm') { try { @@ -31,20 +68,43 @@ export class BazelPackage { } } - return new BazelPackage(Path.relative(REPO_ROOT, dir), pkg, buildBazelContent); + return new BazelPackage(normalizePath(Path.relative(REPO_ROOT, dir)), pkg, buildBazelContent); } constructor( - public readonly repoRelativeDir: string, - public readonly pkg: any, - public readonly buildBazelContent?: string + /** + * Relative path from the root of the repository to the package directory + */ + public readonly normalizedRepoRelativeDir: string, + /** + * Parsed package.json file from the package + */ + public readonly pkg: ParsedPackageJson, + /** + * Content of the BUILD.bazel file + */ + private readonly buildBazelContent?: string ) {} + /** + * Returns true if the package includes a `:build` bazel rule + */ hasBuildRule() { return !!(this.buildBazelContent && BUILD_RULE_NAME.test(this.buildBazelContent)); } + /** + * Returns true if the package includes a `:build_types` bazel rule + */ hasBuildTypesRule() { return !!(this.buildBazelContent && BUILD_TYPES_RULE_NAME.test(this.buildBazelContent)); } + + /** + * Custom inspect handler so that logging variables in scripts/generate doesn't + * print all the BUILD.bazel files + */ + [inspect.custom]() { + return `BazelPackage<${this.normalizedRepoRelativeDir}>`; + } } diff --git a/packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.test.ts b/packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.test.ts deleted file mode 100644 index 2d9fd2ed48adc..0000000000000 --- a/packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { generatePackagesBuildBazelFile } from './generate_packages_build_bazel_file'; - -import { BazelPackage } from './bazel_package'; - -it('produces a valid BUILD.bazel file', () => { - const packages = [ - new BazelPackage( - 'foo', - {}, - ` - rule( - name = "build" - ) - rule( - name = "build_types" - ) - ` - ), - new BazelPackage( - 'bar', - {}, - ` - rule( - name= "build_types" - ) - ` - ), - new BazelPackage( - 'bar', - {}, - ` - rule( - name ="build" - ) - ` - ), - new BazelPackage('bar', {}), - ]; - - expect(generatePackagesBuildBazelFile(packages)).toMatchInlineSnapshot(` - "################ - ################ - ## This file is automatically generated, to create a new package use \`node scripts/generate package --help\` - ################ - ################ - - # It will build all declared code packages - filegroup( - name = \\"build_pkg_code\\", - srcs = [ - \\"//foo:build\\", - \\"//bar:build\\", - ], - ) - - # It will build all declared package types - filegroup( - name = \\"build_pkg_types\\", - srcs = [ - \\"//foo:build_types\\", - \\"//bar:build_types\\", - ], - ) - - # Grouping target to call all underlying packages build - # targets so we can build them all at once - # It will auto build all declared code packages and types packages - filegroup( - name = \\"build\\", - srcs = [ - \\":build_pkg_code\\", - \\":build_pkg_types\\" - ], - ) - " - `); -}); diff --git a/packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.ts b/packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.ts deleted file mode 100644 index d1dd3561ed39d..0000000000000 --- a/packages/kbn-bazel-packages/src/generate_packages_build_bazel_file.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { BazelPackage } from './bazel_package'; - -export function generatePackagesBuildBazelFile(packages: BazelPackage[]) { - return `################ -################ -## This file is automatically generated, to create a new package use \`node scripts/generate package --help\` -################ -################ - -# It will build all declared code packages -filegroup( - name = "build_pkg_code", - srcs = [ -${packages - .flatMap((p) => (p.hasBuildRule() ? ` "//${p.repoRelativeDir}:build",` : [])) - .join('\n')} - ], -) - -# It will build all declared package types -filegroup( - name = "build_pkg_types", - srcs = [ -${packages - .flatMap((p) => (p.hasBuildTypesRule() ? ` "//${p.repoRelativeDir}:build_types",` : [])) - .join('\n')} - ], -) - -# Grouping target to call all underlying packages build -# targets so we can build them all at once -# It will auto build all declared code packages and types packages -filegroup( - name = "build", - srcs = [ - ":build_pkg_code", - ":build_pkg_types" - ], -) -`; -} diff --git a/packages/kbn-bazel-packages/src/index.ts b/packages/kbn-bazel-packages/src/index.ts index 7e73fcd0a63ee..c200882df8ab9 100644 --- a/packages/kbn-bazel-packages/src/index.ts +++ b/packages/kbn-bazel-packages/src/index.ts @@ -8,4 +8,3 @@ export * from './discover_packages'; export type { BazelPackage } from './bazel_package'; -export * from './generate_packages_build_bazel_file'; diff --git a/packages/kbn-generate/src/cli.ts b/packages/kbn-generate/src/cli.ts index 9ade4f68366f5..0b52f5bb4da72 100644 --- a/packages/kbn-generate/src/cli.ts +++ b/packages/kbn-generate/src/cli.ts @@ -12,6 +12,7 @@ import { Render } from './lib/render'; import { ContextExtensions } from './generate_command'; import { PackageCommand } from './commands/package_command'; +import { PackagesBuildManifestCommand } from './commands/packages_build_manifest_command'; /** * Runs the generate CLI. Called by `node scripts/generate` and not intended for use outside of that script @@ -26,6 +27,6 @@ export function runGenerateCli() { }; }, }, - [PackageCommand] + [PackageCommand, PackagesBuildManifestCommand] ).execute(); } diff --git a/packages/kbn-generate/src/commands/package_command.ts b/packages/kbn-generate/src/commands/package_command.ts index 284b3b96a0308..6ce14571e64bb 100644 --- a/packages/kbn-generate/src/commands/package_command.ts +++ b/packages/kbn-generate/src/commands/package_command.ts @@ -13,10 +13,10 @@ import normalizePath from 'normalize-path'; import globby from 'globby'; import { REPO_ROOT } from '@kbn/utils'; -import { discoverBazelPackages, generatePackagesBuildBazelFile } from '@kbn/bazel-packages'; +import { discoverBazelPackages } from '@kbn/bazel-packages'; import { createFailError, createFlagError, isFailError, sortPackageJson } from '@kbn/dev-utils'; -import { ROOT_PKG_DIR, PKG_TEMPLATE_DIR } from '../paths'; +import { TEMPLATE_DIR, ROOT_PKG_DIR, PKG_TEMPLATE_DIR } from '../paths'; import type { GenerateCommand } from '../generate_command'; export const PackageCommand: GenerateCommand = { @@ -49,7 +49,7 @@ export const PackageCommand: GenerateCommand = { const containingDir = flags.dir ? Path.resolve(`${flags.dir}`) : ROOT_PKG_DIR; const packageDir = Path.resolve(containingDir, name.slice(1).replace('/', '-')); - const repoRelativeDir = normalizePath(Path.relative(REPO_ROOT, packageDir)); + const normalizedRepoRelativeDir = normalizePath(Path.relative(REPO_ROOT, packageDir)); try { await Fsp.readdir(packageDir); @@ -107,8 +107,8 @@ export const PackageCommand: GenerateCommand = { name, web, dev, - directoryName: Path.basename(repoRelativeDir), - repoRelativeDir, + directoryName: Path.basename(normalizedRepoRelativeDir), + normalizedRepoRelativeDir, }, }); } @@ -122,17 +122,20 @@ export const PackageCommand: GenerateCommand = { ? [packageJson.devDependencies, packageJson.dependencies] : [packageJson.dependencies, packageJson.devDependencies]; - addDeps[name] = `link:bazel-bin/${repoRelativeDir}`; - addDeps[typePkgName] = `link:bazel-bin/${repoRelativeDir}/npm_module_types`; + addDeps[name] = `link:bazel-bin/${normalizedRepoRelativeDir}`; + addDeps[typePkgName] = `link:bazel-bin/${normalizedRepoRelativeDir}/npm_module_types`; delete removeDeps[name]; delete removeDeps[typePkgName]; await Fsp.writeFile(packageJsonPath, sortPackageJson(JSON.stringify(packageJson))); log.info('Updated package.json file'); - await Fsp.writeFile( + await render.toFile( + Path.resolve(TEMPLATE_DIR, 'packages_BUILD.bazel.ejs'), Path.resolve(REPO_ROOT, 'packages/BUILD.bazel'), - generatePackagesBuildBazelFile(await discoverBazelPackages()) + { + packages: await discoverBazelPackages(), + } ); log.info('Updated packages/BUILD.bazel'); diff --git a/packages/kbn-generate/src/commands/packages_build_manifest_command.ts b/packages/kbn-generate/src/commands/packages_build_manifest_command.ts new file mode 100644 index 0000000000000..9103d56d42a1d --- /dev/null +++ b/packages/kbn-generate/src/commands/packages_build_manifest_command.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fsp from 'fs/promises'; + +import { REPO_ROOT } from '@kbn/utils'; +import { discoverBazelPackages } from '@kbn/bazel-packages'; + +import { TEMPLATE_DIR } from '../paths'; +import { GenerateCommand } from '../generate_command'; +import { validateFile } from '../lib/validate_file'; + +const USAGE = `node scripts/generate packages_build_manifest`; + +export const PackagesBuildManifestCommand: GenerateCommand = { + name: 'packages_build_manifest', + usage: USAGE, + description: 'Generate the packages/BUILD.bazel file', + flags: { + boolean: ['validate'], + help: ` + --validate Rather than writing the generated output to disk, validate that the content on disk is in sync with the + `, + }, + async run({ log, render, flags }) { + const validate = !!flags.validate; + + const packages = await discoverBazelPackages(); + const dest = Path.resolve(REPO_ROOT, 'packages/BUILD.bazel'); + const relDest = Path.relative(process.cwd(), dest); + + const content = await render.toString( + Path.join(TEMPLATE_DIR, 'packages_BUILD.bazel.ejs'), + dest, + { + packages, + } + ); + + if (validate) { + await validateFile(log, USAGE, dest, content); + return; + } + + await Fsp.writeFile(dest, content); + log.success('Wrote', relDest); + }, +}; diff --git a/packages/kbn-generate/src/lib/validate_file.ts b/packages/kbn-generate/src/lib/validate_file.ts new file mode 100644 index 0000000000000..d4f3640a45471 --- /dev/null +++ b/packages/kbn-generate/src/lib/validate_file.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Fsp from 'fs/promises'; +import Path from 'path'; + +import { ToolingLog, createFailError, diffStrings } from '@kbn/dev-utils'; + +export async function validateFile(log: ToolingLog, usage: string, path: string, expected: string) { + const relPath = Path.relative(process.cwd(), path); + + let current; + try { + current = await Fsp.readFile(path, 'utf8'); + } catch (error) { + if (error && error.code === 'ENOENT') { + throw createFailError(`${relPath} is missing, please run "${usage}" and commit the result`); + } + + throw error; + } + + if (current !== expected) { + log.error(`${relPath} is outdated:\n${diffStrings(expected, current)}`); + throw createFailError(`${relPath} is outdated, please run "${usage}" and commit the result`); + } + + log.success(`${relPath} is valid`); +} diff --git a/packages/kbn-generate/templates/package/jest.config.js.ejs b/packages/kbn-generate/templates/package/jest.config.js.ejs index 1846d6a8f96f5..6a65cc6b304c5 100644 --- a/packages/kbn-generate/templates/package/jest.config.js.ejs +++ b/packages/kbn-generate/templates/package/jest.config.js.ejs @@ -1,5 +1,5 @@ module.exports = { preset: <%- js(pkg.web ? '@kbn/test' : '@kbn/test/jest_node') %>, rootDir: '../..', - roots: [<%- js(`/${pkg.repoRelativeDir}`) %>], + roots: [<%- js(`/${pkg.normalizedRepoRelativeDir}`) %>], }; diff --git a/packages/kbn-generate/templates/packages_BUILD.bazel.ejs b/packages/kbn-generate/templates/packages_BUILD.bazel.ejs new file mode 100644 index 0000000000000..43dd306d3cbb7 --- /dev/null +++ b/packages/kbn-generate/templates/packages_BUILD.bazel.ejs @@ -0,0 +1,37 @@ +################ +################ +## This file is automatically generated, to create a new package use `node scripts/generate package --help` or run +## `node scripts/generate packages_build_manifest` to regenerate it from the current state of the repo +################ +################ + +# It will build all declared code packages +filegroup( + name = "build_pkg_code", + srcs = [ +<% for (const p of packages.filter(p => p.hasBuildRule())) { _%> + "//<%- p.normalizedRepoRelativeDir %>:build", +<% } _%> + ], +) + +# It will build all declared package types +filegroup( + name = "build_pkg_types", + srcs = [ +<% for (const p of packages.filter(p => p.hasBuildTypesRule())) { _%> + "//<%- p.normalizedRepoRelativeDir %>:build_types", +<% } _%> + ], +) + +# Grouping target to call all underlying packages build +# targets so we can build them all at once +# It will auto build all declared code packages and types packages +filegroup( + name = "build", + srcs = [ + ":build_pkg_code", + ":build_pkg_types" + ], +) diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 86448be7c3e53..eb65e0b752174 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -67,6 +67,9 @@ export const IGNORE_FILE_GLOBS = [ // Buildkite '.buildkite/**/*', + + // generator templates use weird filenames based on the requirements for the files they're generating + 'packages/kbn-generate/templates/**/*', ]; /** @@ -107,10 +110,7 @@ export const IGNORE_DIRECTORY_GLOBS = [ * * @type {Array} */ -export const REMOVE_EXTENSION = [ - 'packages/kbn-plugin-generator/template/**/*.ejs', - 'packages/kbn-generate/templates/**/*.ejs', -]; +export const REMOVE_EXTENSION = ['packages/kbn-plugin-generator/template/**/*.ejs']; /** * DO NOT ADD FILES TO THIS LIST!!