From 646a212b7bf550d2a557dc7c3f74e6cc0c46d559 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 15 Jun 2020 18:04:33 -0400 Subject: [PATCH 1/5] [Endpoint] [ES Archiver] Allowing create option to be passed through the cli for es archiver (#69191) * Allowing create option to be passed through the cli * Using kebab casing --- src/es_archiver/cli.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/es_archiver/cli.ts b/src/es_archiver/cli.ts index 98888b81d9a3..85e10b31a87e 100644 --- a/src/es_archiver/cli.ts +++ b/src/es_archiver/cli.ts @@ -67,9 +67,10 @@ cmd .action((name, indices) => execute((archiver, { raw }) => archiver.save(name, indices, { raw }))); cmd + .option('--use-create', 'use create instead of index for loading documents') .command('load ') .description('load the archive in --dir with ') - .action((name) => execute((archiver) => archiver.load(name))); + .action((name) => execute((archiver, { useCreate }) => archiver.load(name, { useCreate }))); cmd .command('unload ') From f7398f086b354e509998eabdb5468f15ec2b67ab Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 15 Jun 2020 16:18:03 -0600 Subject: [PATCH 2/5] [Security] siem to securitySolution SO migration (#68776) --- .../saved_objects/migrations.test.ts | 57 +++++++++++++++++++ .../ui_settings/saved_objects/migrations.ts | 42 ++++++++++++++ .../ui_settings/saved_objects/ui_settings.ts | 2 + .../apis/saved_objects/bulk_get.js | 1 + 4 files changed, 102 insertions(+) create mode 100644 src/core/server/ui_settings/saved_objects/migrations.test.ts create mode 100644 src/core/server/ui_settings/saved_objects/migrations.ts diff --git a/src/core/server/ui_settings/saved_objects/migrations.test.ts b/src/core/server/ui_settings/saved_objects/migrations.test.ts new file mode 100644 index 000000000000..1620995c932c --- /dev/null +++ b/src/core/server/ui_settings/saved_objects/migrations.test.ts @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SavedObjectUnsanitizedDoc } from 'kibana/server'; +import { migrations } from './migrations'; + +describe('ui_settings 7.9.0 migrations', () => { + const migration = migrations['7.9.0']; + + test('returns doc on empty object', () => { + expect(migration({} as SavedObjectUnsanitizedDoc)).toEqual({ + references: [], + }); + }); + test('properly renames siem attributes to securitySolution', () => { + const doc = { + type: 'config', + id: '8.0.0', + attributes: { + buildNum: 9007199254740991, + 'siem:defaultAnomalyScore': 59, + 'siem:enableNewsFeed': false, + }, + references: [], + updated_at: '2020-06-09T20:18:20.349Z', + migrationVersion: {}, + }; + expect(migration(doc)).toEqual({ + type: 'config', + id: '8.0.0', + attributes: { + buildNum: 9007199254740991, + 'securitySolution:defaultAnomalyScore': 59, + 'securitySolution:enableNewsFeed': false, + }, + references: [], + updated_at: '2020-06-09T20:18:20.349Z', + migrationVersion: {}, + }); + }); +}); diff --git a/src/core/server/ui_settings/saved_objects/migrations.ts b/src/core/server/ui_settings/saved_objects/migrations.ts new file mode 100644 index 000000000000..750d4e6bc1ea --- /dev/null +++ b/src/core/server/ui_settings/saved_objects/migrations.ts @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SavedObjectUnsanitizedDoc, SavedObjectSanitizedDoc } from 'kibana/server'; + +export const migrations = { + '7.9.0': (doc: SavedObjectUnsanitizedDoc): SavedObjectSanitizedDoc => ({ + ...doc, + ...(doc.attributes && { + attributes: Object.keys(doc.attributes).reduce( + (acc, key) => + key.startsWith('siem:') + ? { + ...acc, + [key.replace('siem', 'securitySolution')]: doc.attributes[key], + } + : { + ...acc, + [key]: doc.attributes[key], + }, + {} + ), + }), + references: doc.references || [], + }), +}; diff --git a/src/core/server/ui_settings/saved_objects/ui_settings.ts b/src/core/server/ui_settings/saved_objects/ui_settings.ts index 0eab40a7b3a5..26704f46a509 100644 --- a/src/core/server/ui_settings/saved_objects/ui_settings.ts +++ b/src/core/server/ui_settings/saved_objects/ui_settings.ts @@ -18,6 +18,7 @@ */ import { SavedObjectsType } from '../../saved_objects'; +import { migrations } from './migrations'; export const uiSettingsType: SavedObjectsType = { name: 'config', @@ -46,4 +47,5 @@ export const uiSettingsType: SavedObjectsType = { return `Advanced Settings [${obj.id}]`; }, }, + migrations, }; diff --git a/test/api_integration/apis/saved_objects/bulk_get.js b/test/api_integration/apis/saved_objects/bulk_get.js index 23aa175740b6..c802d5291306 100644 --- a/test/api_integration/apis/saved_objects/bulk_get.js +++ b/test/api_integration/apis/saved_objects/bulk_get.js @@ -94,6 +94,7 @@ export default function ({ getService }) { buildNum: 8467, defaultIndex: '91200a00-9efd-11e7-acb3-3dab96693fab', }, + migrationVersion: resp.body.saved_objects[2].migrationVersion, references: [], }, ], From 41e801f830f4ad7776d9e42187e514194c381ae5 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 15 Jun 2020 23:48:32 +0100 Subject: [PATCH 3/5] chore(NA): include hidden files when creating package with plugin helpers (#68247) Co-authored-by: Elastic Machine --- packages/kbn-plugin-helpers/src/tasks/build/create_package.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts b/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts index 9fa2305a94ea..02c200490034 100644 --- a/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts +++ b/packages/kbn-plugin-helpers/src/tasks/build/create_package.ts @@ -36,7 +36,7 @@ export async function createPackage( // zip up the package await pipeline( - vfs.src(buildFiles, { cwd: buildTarget, base: buildTarget }), + vfs.src(buildFiles, { cwd: buildTarget, base: buildTarget, dot: true }), zip(`${buildId}.zip`), vfs.dest(buildTarget) ); From 7e2ab7fb823577fa05e6ae27fda7a2c4cb00c26f Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 15 Jun 2020 16:19:25 -0700 Subject: [PATCH 4/5] [Reporting/Server] register plugin routes synchronously (#68976) * register routes synchronously * back out some refactoring * comment fix * fix tests * register route handler context provider * Add function level comments in core methods * fix tests * revert editor help * route context is the ReportingStart contract * Fix reporting job route tests * Fix generation tests Co-authored-by: Joel Griffith Co-authored-by: Elastic Machine --- x-pack/plugins/reporting/server/core.ts | 113 ++++++++++++------ .../csv/server/execute_job.test.ts | 25 ++-- .../export_types/csv/server/execute_job.ts | 2 +- .../server/lib/generate_csv_search.ts | 2 +- .../server/execute_job/index.test.ts | 36 +----- .../reporting/server/lib/jobs_query.ts | 12 +- .../plugins/reporting/server/plugin.test.ts | 3 +- x-pack/plugins/reporting/server/plugin.ts | 74 ++++++------ .../server/routes/generation.test.ts | 4 +- .../reporting/server/routes/generation.ts | 21 ++-- .../reporting/server/routes/jobs.test.ts | 4 +- .../plugins/reporting/server/routes/jobs.ts | 55 +++++++-- .../routes/lib/authorized_user_pre_routing.ts | 2 +- .../server/routes/lib/job_response_handler.ts | 21 ++-- .../create_mock_reportingplugin.ts | 81 +++++-------- .../usage/reporting_usage_collector.test.ts | 27 +++-- .../server/usage/reporting_usage_collector.ts | 14 +-- 17 files changed, 262 insertions(+), 234 deletions(-) diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index 94b138ffcae0..9acd359fa0db 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -5,7 +5,7 @@ */ import * as Rx from 'rxjs'; -import { first, map, mapTo } from 'rxjs/operators'; +import { first, map, take } from 'rxjs/operators'; import { BasePath, ElasticsearchServiceSetup, @@ -33,7 +33,8 @@ export interface ReportingInternalSetup { security?: SecurityPluginSetup; } -interface ReportingInternalStart { +export interface ReportingInternalStart { + browserDriverFactory: HeadlessChromiumDriverFactory; enqueueJob: EnqueueJobFn; esqueue: ESQueueInstance; savedObjects: SavedObjectsServiceStart; @@ -43,33 +44,83 @@ interface ReportingInternalStart { export class ReportingCore { private pluginSetupDeps?: ReportingInternalSetup; private pluginStartDeps?: ReportingInternalStart; - private browserDriverFactory?: HeadlessChromiumDriverFactory; - private readonly pluginSetup$ = new Rx.ReplaySubject(); - private readonly pluginStart$ = new Rx.ReplaySubject(); + private readonly pluginSetup$ = new Rx.ReplaySubject(); // observe async background setupDeps and config each are done + private readonly pluginStart$ = new Rx.ReplaySubject(); // observe async background startDeps private exportTypesRegistry = getExportTypesRegistry(); + private config?: ReportingConfig; - constructor(private config: ReportingConfig) {} + constructor() {} - public pluginSetup(reportingSetupDeps: ReportingInternalSetup) { - this.pluginSetupDeps = reportingSetupDeps; - this.pluginSetup$.next(reportingSetupDeps); + /* + * Register setupDeps + */ + public pluginSetup(setupDeps: ReportingInternalSetup) { + this.pluginSetup$.next(true); // trigger the observer + this.pluginSetupDeps = setupDeps; // cache } - public pluginStart(reportingStartDeps: ReportingInternalStart) { - this.pluginStart$.next(reportingStartDeps); + /* + * Register startDeps + */ + public pluginStart(startDeps: ReportingInternalStart) { + this.pluginStart$.next(startDeps); // trigger the observer + this.pluginStartDeps = startDeps; // cache } - public pluginHasStarted(): Promise { - return this.pluginStart$.pipe(first(), mapTo(true)).toPromise(); + /* + * Blocks the caller until setup is done + */ + public async pluginSetsUp(): Promise { + // use deps and config as a cached resolver + if (this.pluginSetupDeps && this.config) { + return true; + } + return await this.pluginSetup$.pipe(take(2)).toPromise(); // once for pluginSetupDeps (sync) and twice for config (async) } - public setBrowserDriverFactory(browserDriverFactory: HeadlessChromiumDriverFactory) { - this.browserDriverFactory = browserDriverFactory; + /* + * Blocks the caller until start is done + */ + public async pluginStartsUp(): Promise { + return await this.getPluginStartDeps().then(() => true); + } + + /* + * Synchronously checks if all async background setup and startup is completed + */ + public pluginIsStarted() { + return this.pluginSetupDeps != null && this.config != null && this.pluginStartDeps != null; } /* - * Internal module dependencies + * Allows config to be set in the background */ + public setConfig(config: ReportingConfig) { + this.config = config; + this.pluginSetup$.next(true); + } + + /* + * Gives synchronous access to the config + */ + public getConfig(): ReportingConfig { + if (!this.config) { + throw new Error('Config is not yet initialized'); + } + return this.config; + } + + /* + * Gives async access to the startDeps + */ + private async getPluginStartDeps() { + if (this.pluginStartDeps) { + return this.pluginStartDeps; + } + + return await this.pluginStart$.pipe(first()).toPromise(); + } + public getExportTypesRegistry() { return this.exportTypesRegistry; } @@ -92,18 +143,15 @@ export class ReportingCore { .toPromise(); } - public getConfig(): ReportingConfig { - return this.config; - } - - public getScreenshotsObservable(): ScreenshotsObservableFn { - const { browserDriverFactory } = this; - if (!browserDriverFactory) { - throw new Error(`"browserDriverFactory" dependency hasn't initialized yet`); - } - return screenshotsObservableFactory(this.config.get('capture'), browserDriverFactory); + public async getScreenshotsObservable(): Promise { + const config = this.getConfig(); + const { browserDriverFactory } = await this.getPluginStartDeps(); + return screenshotsObservableFactory(config.get('capture'), browserDriverFactory); } + /* + * Gives synchronous access to the setupDeps + */ public getPluginSetupDeps() { if (!this.pluginSetupDeps) { throw new Error(`"pluginSetupDeps" dependencies haven't initialized yet`); @@ -111,18 +159,7 @@ export class ReportingCore { return this.pluginSetupDeps; } - /* - * Outside dependencies - */ - - private async getPluginStartDeps() { - if (this.pluginStartDeps) { - return this.pluginStartDeps; - } - return await this.pluginStart$.pipe(first()).toPromise(); - } - - public async getElasticsearchService() { + public getElasticsearchService() { return this.getPluginSetupDeps().elasticsearch; } diff --git a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts index ddcf94079ade..4ce448e953bd 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.test.ts @@ -5,10 +5,16 @@ */ import nodeCrypto from '@elastic/node-crypto'; +import { IUiSettingsClient, ElasticsearchServiceSetup } from 'kibana/server'; // @ts-ignore import Puid from 'puid'; import sinon from 'sinon'; +import { ReportingConfig, ReportingCore } from '../../../'; import { fieldFormats, UI_SETTINGS } from '../../../../../../../src/plugins/data/server'; +import { + CSV_QUOTE_VALUES_SETTING, + CSV_SEPARATOR_SETTING, +} from '../../../../../../../src/plugins/share/server'; import { CancellationToken } from '../../../../common'; import { CSV_BOM_CHARS } from '../../../../common/constants'; import { LevelLogger } from '../../../lib'; @@ -16,10 +22,6 @@ import { setFieldFormats } from '../../../services'; import { createMockReportingCore } from '../../../test_helpers'; import { JobDocPayloadDiscoverCsv } from '../types'; import { executeJobFactory } from './execute_job'; -import { - CSV_SEPARATOR_SETTING, - CSV_QUOTE_VALUES_SETTING, -} from '../../../../../../../src/plugins/share/server'; const delay = (ms: number) => new Promise((resolve) => setTimeout(() => resolve(), ms)); @@ -48,8 +50,8 @@ describe('CSV Execute Job', function () { let clusterStub: any; let configGetStub: any; - let mockReportingConfig: any; - let mockReportingCore: any; + let mockReportingConfig: ReportingConfig; + let mockReportingCore: ReportingCore; let callAsCurrentUserStub: any; let cancellationToken: any; @@ -78,9 +80,11 @@ describe('CSV Execute Job', function () { mockReportingConfig = { get: configGetStub, kbnConfig: { get: configGetStub } }; mockReportingCore = await createMockReportingCore(mockReportingConfig); - mockReportingCore.getUiSettingsServiceFactory = () => Promise.resolve(mockUiSettingsClient); - mockReportingCore.getElasticsearchService = () => Promise.resolve(mockElasticsearch); - mockReportingCore.config = mockReportingConfig; + mockReportingCore.getUiSettingsServiceFactory = () => + Promise.resolve((mockUiSettingsClient as unknown) as IUiSettingsClient); + mockReportingCore.getElasticsearchService = () => + mockElasticsearch as ElasticsearchServiceSetup; + mockReportingCore.setConfig(mockReportingConfig); cancellationToken = new CancellationToken(); @@ -995,7 +999,8 @@ describe('CSV Execute Job', function () { let maxSizeReached: boolean; beforeEach(async function () { - mockReportingCore.getUiSettingsServiceFactory = () => mockUiSettingsClient; + mockReportingCore.getUiSettingsServiceFactory = () => + Promise.resolve((mockUiSettingsClient as unknown) as IUiSettingsClient); configGetStub.withArgs('csv', 'maxSizeBytes').returns(18); callAsCurrentUserStub.onFirstCall().returns({ diff --git a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts index 4b17cc669efe..91a4db0469fb 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/server/execute_job.ts @@ -33,7 +33,7 @@ export const executeJobFactory: ExecuteJobFactory callAsCurrentUser(...params); const uiSettings = await getUiSettings(uiConfig); diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts index b8e1e5eebd9e..2f4ca47cf739 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf/server/execute_job/index.test.ts @@ -84,6 +84,7 @@ test(`passes browserTimezone to generatePdf`, async () => { await executeJob( 'pdfJobId', getJobDocPayload({ + title: 'PDF Params Timezone Test', relativeUrl: '/app/kibana#/something', browserTimezone, headers: encryptedHeaders, @@ -91,39 +92,8 @@ test(`passes browserTimezone to generatePdf`, async () => { cancellationToken ); - expect(generatePdfObservable.mock.calls).toMatchInlineSnapshot(` - Array [ - Array [ - LevelLogger { - "_logger": Object { - "get": [MockFunction], - }, - "_tags": Array [ - "printable_pdf", - "execute", - "pdfJobId", - ], - "warning": [Function], - }, - undefined, - Array [ - "http://localhost:5601/sbp/app/kibana#/something", - ], - "UTC", - Object { - "conditions": Object { - "basePath": "/sbp", - "hostname": "localhost", - "port": 5601, - "protocol": "http", - }, - "headers": Object {}, - }, - undefined, - false, - ], - ] - `); + const tzParam = generatePdfObservable.mock.calls[0][3]; + expect(tzParam).toBe('UTC'); }); test(`returns content_type of application/pdf`, async () => { diff --git a/x-pack/plugins/reporting/server/lib/jobs_query.ts b/x-pack/plugins/reporting/server/lib/jobs_query.ts index 8784d8ff35d2..f4670847260e 100644 --- a/x-pack/plugins/reporting/server/lib/jobs_query.ts +++ b/x-pack/plugins/reporting/server/lib/jobs_query.ts @@ -6,10 +6,9 @@ import { i18n } from '@kbn/i18n'; import { errors as elasticsearchErrors } from 'elasticsearch'; -import { ElasticsearchServiceSetup } from 'kibana/server'; import { get } from 'lodash'; +import { ReportingCore } from '../'; import { AuthenticatedUser } from '../../../security/server'; -import { ReportingConfig } from '../'; import { JobSource } from '../types'; const esErrors = elasticsearchErrors as Record; @@ -42,11 +41,8 @@ interface CountAggResult { const getUsername = (user: AuthenticatedUser | null) => (user ? user.username : false); -export function jobsQueryFactory( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup -) { - const index = config.get('index'); +export function jobsQueryFactory(reportingCore: ReportingCore) { + const { elasticsearch } = reportingCore.getPluginSetupDeps(); const { callAsInternalUser } = elasticsearch.legacy.client; function execQuery(queryType: string, body: QueryBody) { @@ -60,6 +56,8 @@ export function jobsQueryFactory( }, }; + const config = reportingCore.getConfig(); + const index = config.get('index'); const query = { index: `${index}-*`, body: Object.assign(defaultBody[queryType] || {}, body), diff --git a/x-pack/plugins/reporting/server/plugin.test.ts b/x-pack/plugins/reporting/server/plugin.test.ts index b2bcd6b9c97c..420fa8347cde 100644 --- a/x-pack/plugins/reporting/server/plugin.test.ts +++ b/x-pack/plugins/reporting/server/plugin.test.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + jest.mock('./browsers/install', () => ({ installBrowser: jest.fn().mockImplementation(() => ({ binaryPath$: { @@ -62,10 +63,10 @@ describe('Reporting Plugin', () => { }); it('logs setup issues', async () => { + initContext.config = null; const plugin = new ReportingPlugin(initContext); // @ts-ignore overloading error logger plugin.logger.error = jest.fn(); - coreSetup.elasticsearch = null; plugin.setup(coreSetup, pluginSetup); await sleep(5); diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index a3c89c7b8a8c..693b0917603f 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import * as Rx from 'rxjs'; import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from 'src/core/server'; import { ReportingCore } from './'; import { initializeBrowserDriverFactory } from './browsers'; @@ -15,47 +14,57 @@ import { setFieldFormats } from './services'; import { ReportingSetup, ReportingSetupDeps, ReportingStart, ReportingStartDeps } from './types'; import { registerReportingUsageCollector } from './usage'; +declare module 'src/core/server' { + interface RequestHandlerContext { + reporting?: ReportingStart | null; + } +} + export class ReportingPlugin implements Plugin { private readonly initializerContext: PluginInitializerContext; private logger: LevelLogger; - private reportingCore?: ReportingCore; - - // Setup some observables for modules that need to await setup/start - public readonly setup$ = new Rx.Subject(); - public readonly start$ = new Rx.Subject(); + private reportingCore: ReportingCore; constructor(context: PluginInitializerContext) { this.logger = new LevelLogger(context.logger.get()); this.initializerContext = context; + this.reportingCore = new ReportingCore(); } public setup(core: CoreSetup, plugins: ReportingSetupDeps) { + // prevent throwing errors in route handlers about async deps not being initialized + core.http.registerRouteHandlerContext('reporting', () => { + if (this.reportingCore.pluginIsStarted()) { + return {}; // ReportingStart contract + } else { + return null; + } + }); + const { elasticsearch, http } = core; const { licensing, security } = plugins; - const { initializerContext: initContext } = this; + const { initializerContext: initContext, reportingCore } = this; + const router = http.createRouter(); const basePath = http.basePath.get; + reportingCore.pluginSetup({ + elasticsearch, + licensing, + basePath, + router, + security, + }); + + registerReportingUsageCollector(reportingCore, plugins); + registerRoutes(reportingCore, this.logger); + // async background setup (async () => { const config = await buildConfig(initContext, core, this.logger); - const reportingCore = new ReportingCore(config); - - reportingCore.pluginSetup({ - elasticsearch, - licensing, - basePath, - router, - security, - }); - - registerReportingUsageCollector(reportingCore, plugins); - registerRoutes(reportingCore, this.logger); - this.reportingCore = reportingCore; - + reportingCore.setConfig(config); this.logger.debug('Setup complete'); - this.setup$.next(true); })().catch((e) => { this.logger.error(`Error in Reporting setup, reporting may not function properly`); this.logger.error(e); @@ -68,20 +77,21 @@ export class ReportingPlugin // use data plugin for csv formats setFieldFormats(plugins.data.fieldFormats); - const { logger } = this; - const reportingCore = this.getReportingCore(); - const config = reportingCore.getConfig(); + const { logger, reportingCore } = this; const { elasticsearch } = reportingCore.getPluginSetupDeps(); // async background start (async () => { + await this.reportingCore.pluginSetsUp(); + const config = reportingCore.getConfig(); + const browserDriverFactory = await initializeBrowserDriverFactory(config, logger); - reportingCore.setBrowserDriverFactory(browserDriverFactory); - const esqueue = await createQueueFactory(reportingCore, logger); - const enqueueJob = enqueueJobFactory(reportingCore, logger); + const esqueue = await createQueueFactory(reportingCore, logger); // starts polling for pending jobs + const enqueueJob = enqueueJobFactory(reportingCore, logger); // called from generation routes reportingCore.pluginStart({ + browserDriverFactory, savedObjects: core.savedObjects, uiSettings: core.uiSettings, esqueue, @@ -92,7 +102,6 @@ export class ReportingPlugin runValidations(config, elasticsearch, browserDriverFactory, this.logger); this.logger.debug('Start complete'); - this.start$.next(true); })().catch((e) => { this.logger.error(`Error in Reporting start, reporting may not function properly`); this.logger.error(e); @@ -100,11 +109,4 @@ export class ReportingPlugin return {}; } - - public getReportingCore() { - if (!this.reportingCore) { - throw new Error('Setup is not ready'); - } - return this.reportingCore; - } } diff --git a/x-pack/plugins/reporting/server/routes/generation.test.ts b/x-pack/plugins/reporting/server/routes/generation.test.ts index f9b3e5446cfc..4474f2c95e1c 100644 --- a/x-pack/plugins/reporting/server/routes/generation.test.ts +++ b/x-pack/plugins/reporting/server/routes/generation.test.ts @@ -18,6 +18,7 @@ import { of } from 'rxjs'; type setupServerReturn = UnwrapPromise>; describe('POST /api/reporting/generate', () => { + const reportingSymbol = Symbol('reporting'); let server: setupServerReturn['server']; let httpSetup: setupServerReturn['httpSetup']; let exportTypesRegistry: ExportTypesRegistry; @@ -47,7 +48,8 @@ describe('POST /api/reporting/generate', () => { } as unknown) as jest.Mocked; beforeEach(async () => { - ({ server, httpSetup } = await setupServer()); + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); const mockDeps = ({ elasticsearch: { legacy: { diff --git a/x-pack/plugins/reporting/server/routes/generation.ts b/x-pack/plugins/reporting/server/routes/generation.ts index f2e616c0803a..b4c81e698ce7 100644 --- a/x-pack/plugins/reporting/server/routes/generation.ts +++ b/x-pack/plugins/reporting/server/routes/generation.ts @@ -17,15 +17,21 @@ import { HandlerFunction } from './types'; const esErrors = elasticsearchErrors as Record; -export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Logger) { +const getDownloadBaseUrl = (reporting: ReportingCore) => { const config = reporting.getConfig(); - const downloadBaseUrl = - config.kbnConfig.get('server', 'basePath') + `${API_BASE_URL}/jobs/download`; + return config.kbnConfig.get('server', 'basePath') + `${API_BASE_URL}/jobs/download`; +}; +export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Logger) { /* * Generates enqueued job details to use in responses */ const handler: HandlerFunction = async (user, exportTypeId, jobParams, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return res.custom({ statusCode: 503, body: 'Not Available' }); + } + const licenseInfo = await reporting.getLicenseInfo(); const licenseResults = licenseInfo[exportTypeId]; @@ -42,6 +48,7 @@ export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Lo // return the queue's job information const jobJson = job.toJSON(); + const downloadBaseUrl = getDownloadBaseUrl(reporting); return res.ok({ headers: { @@ -86,10 +93,6 @@ export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Lo } registerGenerateFromJobParams(reporting, handler, handleError); - - // Register beta panel-action download-related API's - if (config.get('csv', 'enablePanelActionDownload')) { - registerGenerateCsvFromSavedObject(reporting, handler, handleError); - registerGenerateCsvFromSavedObjectImmediate(reporting, handleError, logger); - } + registerGenerateCsvFromSavedObject(reporting, handler, handleError); // FIXME: remove this https://github.com/elastic/kibana/issues/62986 + registerGenerateCsvFromSavedObjectImmediate(reporting, handleError, logger); } diff --git a/x-pack/plugins/reporting/server/routes/jobs.test.ts b/x-pack/plugins/reporting/server/routes/jobs.test.ts index 22d60d62d5fd..35594474685b 100644 --- a/x-pack/plugins/reporting/server/routes/jobs.test.ts +++ b/x-pack/plugins/reporting/server/routes/jobs.test.ts @@ -19,6 +19,7 @@ import { registerJobInfoRoutes } from './jobs'; type setupServerReturn = UnwrapPromise>; describe('GET /api/reporting/jobs/download', () => { + const reportingSymbol = Symbol('reporting'); let server: setupServerReturn['server']; let httpSetup: setupServerReturn['httpSetup']; let exportTypesRegistry: ExportTypesRegistry; @@ -39,7 +40,8 @@ describe('GET /api/reporting/jobs/download', () => { }; beforeEach(async () => { - ({ server, httpSetup } = await setupServer()); + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext(reportingSymbol, 'reporting', () => ({})); core = await createMockReportingCore(config, ({ elasticsearch: { legacy: { client: { callAsInternalUser: jest.fn() } }, diff --git a/x-pack/plugins/reporting/server/routes/jobs.ts b/x-pack/plugins/reporting/server/routes/jobs.ts index 29cf55bc5c72..90185f0736ed 100644 --- a/x-pack/plugins/reporting/server/routes/jobs.ts +++ b/x-pack/plugins/reporting/server/routes/jobs.ts @@ -4,16 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import Boom from 'boom'; import { schema } from '@kbn/config-schema'; +import Boom from 'boom'; import { ReportingCore } from '../'; import { API_BASE_URL } from '../../common/constants'; import { jobsQueryFactory } from '../lib/jobs_query'; +import { authorizedUserPreRoutingFactory } from './lib/authorized_user_pre_routing'; import { deleteJobResponseHandlerFactory, downloadJobResponseHandlerFactory, } from './lib/job_response_handler'; -import { authorizedUserPreRoutingFactory } from './lib/authorized_user_pre_routing'; interface ListQuery { page: string; @@ -22,12 +22,14 @@ interface ListQuery { } const MAIN_ENTRY = `${API_BASE_URL}/jobs`; +const handleUnavailable = (res: any) => { + return res.custom({ statusCode: 503, body: 'Not Available' }); +}; + export function registerJobInfoRoutes(reporting: ReportingCore) { - const config = reporting.getConfig(); const setupDeps = reporting.getPluginSetupDeps(); const userHandler = authorizedUserPreRoutingFactory(reporting); - const { elasticsearch, router } = setupDeps; - const jobsQuery = jobsQueryFactory(config, elasticsearch); + const { router } = setupDeps; // list jobs in the queue, paginated router.get( @@ -36,6 +38,11 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { validate: false, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); @@ -47,6 +54,7 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { const page = parseInt(queryPage, 10) || 0; const size = Math.min(100, parseInt(querySize, 10) || 10); const jobIds = queryIds ? queryIds.split(',') : null; + const jobsQuery = jobsQueryFactory(reporting); const results = await jobsQuery.list(jobTypes, user, page, size, jobIds); return res.ok({ @@ -65,10 +73,16 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { validate: false, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); + const jobsQuery = jobsQueryFactory(reporting); const count = await jobsQuery.count(jobTypes, user); return res.ok({ @@ -91,11 +105,17 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); + const jobsQuery = jobsQueryFactory(reporting); const result = await jobsQuery.get(user, docId, { includeContent: true }); if (!result) { @@ -130,11 +150,17 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return res.custom({ statusCode: 503 }); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, } = await reporting.getLicenseInfo(); + const jobsQuery = jobsQueryFactory(reporting); const result = await jobsQuery.get(user, docId); if (!result) { @@ -164,12 +190,7 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { ); // trigger a download of the output from a job - const exportTypesRegistry = reporting.getExportTypesRegistry(); - const downloadResponseHandler = downloadJobResponseHandlerFactory( - config, - elasticsearch, - exportTypesRegistry - ); + const downloadResponseHandler = downloadJobResponseHandlerFactory(reporting); router.get( { @@ -181,6 +202,11 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, @@ -191,7 +217,7 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { ); // allow a report to be deleted - const deleteResponseHandler = deleteJobResponseHandlerFactory(config, elasticsearch); + const deleteResponseHandler = deleteJobResponseHandlerFactory(reporting); router.delete( { path: `${MAIN_ENTRY}/delete/{docId}`, @@ -202,6 +228,11 @@ export function registerJobInfoRoutes(reporting: ReportingCore) { }, }, userHandler(async (user, context, req, res) => { + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + const { docId } = req.params as { docId: string }; const { management: { jobTypes = [] }, diff --git a/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts b/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts index 2ad974c9dd8e..2f5d4ebe1419 100644 --- a/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts +++ b/x-pack/plugins/reporting/server/routes/lib/authorized_user_pre_routing.ts @@ -19,7 +19,6 @@ export type RequestHandlerUser = RequestHandler extends (...a: infer U) => infer export const authorizedUserPreRoutingFactory = function authorizedUserPreRoutingFn( reporting: ReportingCore ) { - const config = reporting.getConfig(); const setupDeps = reporting.getPluginSetupDeps(); const getUser = getUserFactory(setupDeps.security); return (handler: RequestHandlerUser): RequestHandler => { @@ -36,6 +35,7 @@ export const authorizedUserPreRoutingFactory = function authorizedUserPreRouting if (user) { // check allowance with the configured set of roleas + "superuser" + const config = reporting.getConfig(); const allowedRoles = config.get('roles', 'allow') || []; const authorizedRoles = [superuserRole, ...allowedRoles]; diff --git a/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts b/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts index 1a2e10cf355a..a8492481e6b1 100644 --- a/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts +++ b/x-pack/plugins/reporting/server/routes/lib/job_response_handler.ts @@ -4,11 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ElasticsearchServiceSetup, kibanaResponseFactory } from 'kibana/server'; +import { kibanaResponseFactory } from 'kibana/server'; +import { ReportingCore } from '../../'; import { AuthenticatedUser } from '../../../../security/server'; -import { ReportingConfig } from '../../'; import { WHITELISTED_JOB_CONTENT_TYPES } from '../../../common/constants'; -import { ExportTypesRegistry } from '../../lib/export_types_registry'; import { jobsQueryFactory } from '../../lib/jobs_query'; import { getDocumentPayloadFactory } from './get_document_payload'; @@ -20,12 +19,9 @@ interface JobResponseHandlerOpts { excludeContent?: boolean; } -export function downloadJobResponseHandlerFactory( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup, - exportTypesRegistry: ExportTypesRegistry -) { - const jobsQuery = jobsQueryFactory(config, elasticsearch); +export function downloadJobResponseHandlerFactory(reporting: ReportingCore) { + const jobsQuery = jobsQueryFactory(reporting); + const exportTypesRegistry = reporting.getExportTypesRegistry(); const getDocumentPayload = getDocumentPayloadFactory(exportTypesRegistry); return async function jobResponseHandler( @@ -69,11 +65,8 @@ export function downloadJobResponseHandlerFactory( }; } -export function deleteJobResponseHandlerFactory( - config: ReportingConfig, - elasticsearch: ElasticsearchServiceSetup -) { - const jobsQuery = jobsQueryFactory(config, elasticsearch); +export function deleteJobResponseHandlerFactory(reporting: ReportingCore) { + const jobsQuery = jobsQueryFactory(reporting); return async function deleteJobResponseHander( res: typeof kibanaResponseFactory, diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index 669381a92c52..579035a46f61 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -11,18 +11,15 @@ jest.mock('../lib/create_queue'); jest.mock('../lib/enqueue_job'); jest.mock('../lib/validate'); -import { of } from 'rxjs'; -import { first } from 'rxjs/operators'; -import { coreMock } from 'src/core/server/mocks'; +import * as Rx from 'rxjs'; import { ReportingConfig, ReportingCore } from '../'; import { chromium, HeadlessChromiumDriverFactory, initializeBrowserDriverFactory, } from '../browsers'; -import { ReportingInternalSetup } from '../core'; -import { ReportingPlugin } from '../plugin'; -import { ReportingSetupDeps, ReportingStartDeps } from '../types'; +import { ReportingInternalSetup, ReportingInternalStart } from '../core'; +import { ReportingStartDeps } from '../types'; (initializeBrowserDriverFactory as jest.Mock< Promise @@ -30,32 +27,30 @@ import { ReportingSetupDeps, ReportingStartDeps } from '../types'; (chromium as any).createDriverFactory.mockImplementation(() => ({})); -const createMockSetupDeps = (setupMock?: any): ReportingSetupDeps => { +const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup => { return { + elasticsearch: setupMock.elasticsearch || { legacy: { client: {} } }, + basePath: setupMock.basePath, + router: setupMock.router, security: setupMock.security, - licensing: { - license$: of({ isAvailable: true, isActive: true, type: 'basic' }), - } as any, - usageCollection: { - makeUsageCollector: jest.fn(), - registerCollector: jest.fn(), - } as any, + licensing: { license$: Rx.of({ isAvailable: true, isActive: true, type: 'basic' }) } as any, + }; +}; + +const createMockPluginStart = (startMock?: any): ReportingInternalStart => { + return { + browserDriverFactory: startMock.browserDriverFactory, + enqueueJob: startMock.enqueueJob, + esqueue: startMock.esqueue, + savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() }, + uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) }, }; }; export const createMockConfigSchema = (overrides?: any) => ({ index: '.reporting', - kibanaServer: { - hostname: 'localhost', - port: '80', - }, - capture: { - browser: { - chromium: { - disableSandbox: true, - }, - }, - }, + kibanaServer: { hostname: 'localhost', port: '80' }, + capture: { browser: { chromium: { disableSandbox: true } } }, ...overrides, }); @@ -63,36 +58,20 @@ export const createMockStartDeps = (startMock?: any): ReportingStartDeps => ({ data: startMock.data, }); -const createMockReportingPlugin = async (config: ReportingConfig): Promise => { - const mockConfigSchema = createMockConfigSchema(config); - const plugin = new ReportingPlugin(coreMock.createPluginInitializerContext(mockConfigSchema)); - const setupMock = coreMock.createSetup(); - const coreStartMock = coreMock.createStart(); - const startMock = { - ...coreStartMock, - data: { fieldFormats: {} }, - }; - - plugin.setup(setupMock, createMockSetupDeps(setupMock)); - await plugin.setup$.pipe(first()).toPromise(); - plugin.start(startMock, createMockStartDeps(startMock)); - await plugin.start$.pipe(first()).toPromise(); - - return plugin; -}; - export const createMockReportingCore = async ( config: ReportingConfig, - setupDepsMock?: ReportingInternalSetup -): Promise => { + setupDepsMock: ReportingInternalSetup | undefined = createMockPluginSetup({}), + startDepsMock: ReportingInternalStart | undefined = createMockPluginStart({}) +) => { config = config || {}; - const plugin = await createMockReportingPlugin(config); - const core = plugin.getReportingCore(); + const core = new ReportingCore(); + + core.pluginSetup(setupDepsMock); + core.setConfig(config); + await core.pluginSetsUp(); - if (setupDepsMock) { - // @ts-ignore overwriting private properties - core.pluginSetupDeps = setupDepsMock; - } + core.pluginStart(startDepsMock); + await core.pluginStartsUp(); return core; }; diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts index d5dccaca3042..ed2abef2542d 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts @@ -7,7 +7,7 @@ import * as Rx from 'rxjs'; import sinon from 'sinon'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { ReportingConfig } from '../'; +import { ReportingConfig, ReportingCore } from '../'; import { createMockReportingCore } from '../test_helpers'; import { getExportTypesRegistry } from '../lib/export_types_registry'; import { ReportingSetupDeps } from '../types'; @@ -62,8 +62,10 @@ const getResponseMock = (base = {}) => base; describe('license checks', () => { let mockConfig: ReportingConfig; + let mockCore: ReportingCore; beforeAll(async () => { mockConfig = getMockReportingConfig(); + mockCore = await createMockReportingCore(mockConfig); }); describe('with a basic license', () => { @@ -72,7 +74,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'basic' }); const callClusterMock = jest.fn(() => Promise.resolve(getResponseMock())); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('basic'), exportTypesRegistry, @@ -102,7 +104,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'none' }); const callClusterMock = jest.fn(() => Promise.resolve(getResponseMock())); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('none'), exportTypesRegistry, @@ -132,7 +134,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'platinum' }); const callClusterMock = jest.fn(() => Promise.resolve(getResponseMock())); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('platinum'), exportTypesRegistry, @@ -162,7 +164,7 @@ describe('license checks', () => { const plugins = getPluginsMock({ license: 'basic' }); const callClusterMock = jest.fn(() => Promise.resolve({})); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock('basic'), exportTypesRegistry, @@ -184,11 +186,16 @@ describe('license checks', () => { }); describe('data modeling', () => { + let mockConfig: ReportingConfig; + let mockCore: ReportingCore; + beforeAll(async () => { + mockConfig = getMockReportingConfig(); + mockCore = await createMockReportingCore(mockConfig); + }); test('with normal looking usage data', async () => { - const mockConfig = getMockReportingConfig(); const plugins = getPluginsMock(); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock(), exportTypesRegistry, @@ -238,10 +245,9 @@ describe('data modeling', () => { }); test('with sparse data', async () => { - const mockConfig = getMockReportingConfig(); const plugins = getPluginsMock(); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock(), exportTypesRegistry, @@ -291,10 +297,9 @@ describe('data modeling', () => { }); test('with empty data', async () => { - const mockConfig = getMockReportingConfig(); const plugins = getPluginsMock(); const { fetch } = getReportingUsageCollector( - mockConfig, + mockCore, plugins.usageCollection, getLicenseMock(), exportTypesRegistry, diff --git a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts index d77d1b539684..364f5187f056 100644 --- a/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts +++ b/x-pack/plugins/reporting/server/usage/reporting_usage_collector.ts @@ -9,7 +9,6 @@ import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { ReportingCore } from '../'; import { KIBANA_REPORTING_TYPE } from '../../common/constants'; -import { ReportingConfig } from '../../server'; import { ExportTypesRegistry } from '../lib/export_types_registry'; import { ReportingSetupDeps } from '../types'; import { GetLicense } from './'; @@ -23,7 +22,7 @@ const METATYPE = 'kibana_stats'; * @return {Object} kibana usage stats type collection object */ export function getReportingUsageCollector( - config: ReportingConfig, + reporting: ReportingCore, usageCollection: UsageCollectionSetup, getLicense: GetLicense, exportTypesRegistry: ExportTypesRegistry, @@ -31,8 +30,10 @@ export function getReportingUsageCollector( ) { return usageCollection.makeUsageCollector({ type: KIBANA_REPORTING_TYPE, - fetch: (callCluster: CallCluster) => - getReportingUsage(config, getLicense, callCluster, exportTypesRegistry), + fetch: (callCluster: CallCluster) => { + const config = reporting.getConfig(); + return getReportingUsage(config, getLicense, callCluster, exportTypesRegistry); + }, isReady, /* @@ -63,7 +64,6 @@ export function registerReportingUsageCollector( return; } - const config = reporting.getConfig(); const exportTypesRegistry = reporting.getExportTypesRegistry(); const getLicense = async () => { return await licensing.license$ @@ -78,10 +78,10 @@ export function registerReportingUsageCollector( ) .toPromise(); }; - const collectionIsReady = reporting.pluginHasStarted.bind(reporting); + const collectionIsReady = reporting.pluginStartsUp.bind(reporting); const collector = getReportingUsageCollector( - config, + reporting, usageCollection, getLicense, exportTypesRegistry, From cc1758dd96c0152ecaf7bb2a8cd690536e34f6cf Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 15 Jun 2020 16:27:56 -0700 Subject: [PATCH 5/5] [release-notes] add script to generate release notes from PRs (#68816) Co-authored-by: spalger Co-authored-by: Elastic Machine --- .gitignore | 4 + package.json | 1 + packages/kbn-release-notes/package.json | 23 ++ packages/kbn-release-notes/src/cli.ts | 162 ++++++++++ .../kbn-release-notes/src/formats/asciidoc.ts | 84 +++++ packages/kbn-release-notes/src/formats/csv.ts | 74 +++++ .../kbn-release-notes/src/formats/format.ts | 34 ++ .../kbn-release-notes/src/formats/index.ts | 25 ++ packages/kbn-release-notes/src/index.ts | 20 ++ .../kbn-release-notes/src/lib/classify_pr.ts | 66 ++++ .../src/lib/get_fix_references.test.ts | 68 ++++ .../src/lib/get_fix_references.ts | 29 ++ .../src/lib/get_note_from_description.test.ts | 79 +++++ .../src/lib/get_note_from_description.ts | 35 +++ packages/kbn-release-notes/src/lib/index.ts | 26 ++ .../src/lib/irrelevant_pr_summary.ts | 61 ++++ .../src/lib/is_pr_relevant.ts | 61 ++++ .../kbn-release-notes/src/lib/pull_request.ts | 206 ++++++++++++ packages/kbn-release-notes/src/lib/streams.ts | 34 ++ .../kbn-release-notes/src/lib/type_helpers.ts | 20 ++ .../kbn-release-notes/src/lib/version.test.ts | 146 +++++++++ packages/kbn-release-notes/src/lib/version.ts | 123 ++++++++ .../src/release_notes_config.ts | 294 ++++++++++++++++++ packages/kbn-release-notes/tsconfig.json | 12 + packages/kbn-release-notes/yarn.lock | 1 + scripts/release_notes.js | 21 ++ yarn.lock | 39 ++- 27 files changed, 1745 insertions(+), 3 deletions(-) create mode 100644 packages/kbn-release-notes/package.json create mode 100644 packages/kbn-release-notes/src/cli.ts create mode 100644 packages/kbn-release-notes/src/formats/asciidoc.ts create mode 100644 packages/kbn-release-notes/src/formats/csv.ts create mode 100644 packages/kbn-release-notes/src/formats/format.ts create mode 100644 packages/kbn-release-notes/src/formats/index.ts create mode 100644 packages/kbn-release-notes/src/index.ts create mode 100644 packages/kbn-release-notes/src/lib/classify_pr.ts create mode 100644 packages/kbn-release-notes/src/lib/get_fix_references.test.ts create mode 100644 packages/kbn-release-notes/src/lib/get_fix_references.ts create mode 100644 packages/kbn-release-notes/src/lib/get_note_from_description.test.ts create mode 100644 packages/kbn-release-notes/src/lib/get_note_from_description.ts create mode 100644 packages/kbn-release-notes/src/lib/index.ts create mode 100644 packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts create mode 100644 packages/kbn-release-notes/src/lib/is_pr_relevant.ts create mode 100644 packages/kbn-release-notes/src/lib/pull_request.ts create mode 100644 packages/kbn-release-notes/src/lib/streams.ts create mode 100644 packages/kbn-release-notes/src/lib/type_helpers.ts create mode 100644 packages/kbn-release-notes/src/lib/version.test.ts create mode 100644 packages/kbn-release-notes/src/lib/version.ts create mode 100644 packages/kbn-release-notes/src/release_notes_config.ts create mode 100644 packages/kbn-release-notes/tsconfig.json create mode 120000 packages/kbn-release-notes/yarn.lock create mode 100644 scripts/release_notes.js diff --git a/.gitignore b/.gitignore index c7c80fc48264..32377ec0f1ff 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,7 @@ npm-debug.log* # apm plugin /x-pack/plugins/apm/tsconfig.json apm.tsconfig.json + +# release notes script output +report.csv +report.asciidoc diff --git a/package.json b/package.json index 887ffae755e3..e8b07ae4abba 100644 --- a/package.json +++ b/package.json @@ -306,6 +306,7 @@ "@kbn/expect": "1.0.0", "@kbn/optimizer": "1.0.0", "@kbn/plugin-generator": "1.0.0", + "@kbn/release-notes": "1.0.0", "@kbn/test": "1.0.0", "@kbn/utility-types": "1.0.0", "@microsoft/api-documenter": "7.7.2", diff --git a/packages/kbn-release-notes/package.json b/packages/kbn-release-notes/package.json new file mode 100644 index 000000000000..25e1816b6cc1 --- /dev/null +++ b/packages/kbn-release-notes/package.json @@ -0,0 +1,23 @@ +{ + "name": "@kbn/release-notes", + "version": "1.0.0", + "license": "Apache-2.0", + "main": "target/index.js", + "scripts": { + "kbn:bootstrap": "tsc", + "kbn:watch": "tsc --watch" + }, + "dependencies": { + "@kbn/dev-utils": "1.0.0", + "axios": "^0.19.2", + "cheerio": "0.22.0", + "dedent": "^0.7.0", + "graphql": "^14.0.0", + "graphql-tag": "^2.10.3", + "terminal-link": "^2.1.1" + }, + "devDependencies": { + "markdown-it": "^10.0.0", + "typescript": "3.9.5" + } +} \ No newline at end of file diff --git a/packages/kbn-release-notes/src/cli.ts b/packages/kbn-release-notes/src/cli.ts new file mode 100644 index 000000000000..44b4a7a0282d --- /dev/null +++ b/packages/kbn-release-notes/src/cli.ts @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; +import Path from 'path'; +import { inspect } from 'util'; + +import { run, createFlagError, createFailError, REPO_ROOT } from '@kbn/dev-utils'; + +import { FORMATS, SomeFormat } from './formats'; +import { + iterRelevantPullRequests, + getPr, + Version, + ClassifiedPr, + streamFromIterable, + asyncPipeline, + IrrelevantPrSummary, + isPrRelevant, + classifyPr, +} from './lib'; + +const rootPackageJson = JSON.parse( + Fs.readFileSync(Path.resolve(REPO_ROOT, 'package.json'), 'utf8') +); +const extensions = FORMATS.map((f) => f.extension); + +export function runReleaseNotesCli() { + run( + async ({ flags, log }) => { + const token = flags.token; + if (!token || typeof token !== 'string') { + throw createFlagError('--token must be defined'); + } + + const version = Version.fromFlag(flags.version); + if (!version) { + throw createFlagError('unable to parse --version, use format "v{major}.{minor}.{patch}"'); + } + + const includeVersions = Version.fromFlags(flags.include || []); + if (!includeVersions) { + throw createFlagError('unable to parse --include, use format "v{major}.{minor}.{patch}"'); + } + + const Formats: SomeFormat[] = []; + for (const flag of Array.isArray(flags.format) ? flags.format : [flags.format]) { + const Format = FORMATS.find((F) => F.extension === flag); + if (!Format) { + throw createFlagError(`--format must be one of "${extensions.join('", "')}"`); + } + Formats.push(Format); + } + + const filename = flags.filename; + if (!filename || typeof filename !== 'string') { + throw createFlagError('--filename must be a string'); + } + + if (flags['debug-pr']) { + const number = parseInt(String(flags['debug-pr']), 10); + if (Number.isNaN(number)) { + throw createFlagError('--debug-pr must be a pr number when specified'); + } + + const summary = new IrrelevantPrSummary(log); + const pr = await getPr(token, number); + log.success( + inspect( + { + version: version.label, + includeVersions: includeVersions.map((v) => v.label), + isPrRelevant: isPrRelevant(pr, version, includeVersions, summary), + ...classifyPr(pr, log), + pr, + }, + { depth: 100 } + ) + ); + summary.logStats(); + return; + } + + log.info(`Loading all PRs with label [${version.label}] to build release notes...`); + + const summary = new IrrelevantPrSummary(log); + const prsToReport: ClassifiedPr[] = []; + const prIterable = iterRelevantPullRequests(token, version, log); + for await (const pr of prIterable) { + if (!isPrRelevant(pr, version, includeVersions, summary)) { + continue; + } + prsToReport.push(classifyPr(pr, log)); + } + summary.logStats(); + + if (!prsToReport.length) { + throw createFailError( + `All PRs with label [${version.label}] were filtered out by the config. Run again with --debug for more info.` + ); + } + + log.info(`Found ${prsToReport.length} prs to report on`); + + for (const Format of Formats) { + const format = new Format(version, prsToReport, log); + const outputPath = Path.resolve(`${filename}.${Format.extension}`); + await asyncPipeline(streamFromIterable(format.print()), Fs.createWriteStream(outputPath)); + log.success(`[${Format.extension}] report written to ${outputPath}`); + } + }, + { + usage: `node scripts/release_notes --token {token} --version {version}`, + flags: { + alias: { + version: 'v', + include: 'i', + }, + string: ['token', 'version', 'format', 'filename', 'include', 'debug-pr'], + default: { + filename: 'report', + version: rootPackageJson.version, + format: extensions, + }, + help: ` + --token (required) The Github access token to use for requests + --version, -v The version to fetch PRs by, PRs with version labels prior to + this one will be ignored (see --include-version) (default ${ + rootPackageJson.version + }) + --include, -i A version that is before --version but shouldn't be considered + "released" and cause PRs with a matching label to be excluded from + release notes. Use this when PRs are labeled with a version that + is less that --version and is expected to be released after + --version, can be specified multiple times. + --format Only produce a certain format, options: "${extensions.join('", "')}" + --filename Output filename, defaults to "report" + --debug-pr Fetch and print the details for a single PR, disabling reporting + `, + }, + description: ` + Fetch details from Github PRs for generating release notes + `, + } + ); +} diff --git a/packages/kbn-release-notes/src/formats/asciidoc.ts b/packages/kbn-release-notes/src/formats/asciidoc.ts new file mode 100644 index 000000000000..d6c707f009f3 --- /dev/null +++ b/packages/kbn-release-notes/src/formats/asciidoc.ts @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import dedent from 'dedent'; + +import { Format } from './format'; +import { + ASCIIDOC_SECTIONS, + UNKNOWN_ASCIIDOC_SECTION, + AREAS, + UNKNOWN_AREA, +} from '../release_notes_config'; + +function* lines(body: string) { + for (const line of dedent(body).split('\n')) { + yield `${line}\n`; + } +} + +export class AsciidocFormat extends Format { + static extension = 'asciidoc'; + + *print() { + const sortedAreas = [ + ...AREAS.slice().sort((a, b) => a.title.localeCompare(b.title)), + UNKNOWN_AREA, + ]; + + yield* lines(` + [[release-notes-${this.version.label}]] + == ${this.version.label} Release Notes + + Also see <>. + `); + + for (const section of [...ASCIIDOC_SECTIONS, UNKNOWN_ASCIIDOC_SECTION]) { + const prsInSection = this.prs.filter((pr) => pr.asciidocSection === section); + if (!prsInSection.length) { + continue; + } + + yield '\n'; + yield* lines(` + [float] + [[${section.id}-${this.version.label}]] + === ${section.title} + `); + + for (const area of sortedAreas) { + const prsInArea = prsInSection.filter((pr) => pr.area === area); + + if (!prsInArea.length) { + continue; + } + + yield `${area.title}::\n`; + for (const pr of prsInArea) { + const fixes = pr.fixes.length ? `[Fixes ${pr.fixes.join(', ')}] ` : ''; + const strippedTitle = pr.title.replace(/^\s*\[[^\]]+\]\s*/, ''); + yield `* ${fixes}${strippedTitle} {pull}${pr.number}[#${pr.number}]\n`; + if (pr.note) { + yield ` - ${pr.note}\n`; + } + } + } + } + } +} diff --git a/packages/kbn-release-notes/src/formats/csv.ts b/packages/kbn-release-notes/src/formats/csv.ts new file mode 100644 index 000000000000..0cf99edada69 --- /dev/null +++ b/packages/kbn-release-notes/src/formats/csv.ts @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Format } from './format'; + +/** + * Escape a value to conform to field and header encoding defined at https://tools.ietf.org/html/rfc4180 + */ +function esc(value: string | number) { + if (typeof value === 'number') { + return String(value); + } + + if (!value.includes(',') && !value.includes('\n') && !value.includes('"')) { + return value; + } + + return `"${value.split('"').join('""')}"`; +} + +function row(...fields: Array) { + return fields.map(esc).join(',') + '\r\n'; +} + +export class CsvFormat extends Format { + static extension = 'csv'; + + *print() { + // columns + yield row( + 'areas', + 'versions', + 'user', + 'title', + 'number', + 'url', + 'date', + 'fixes', + 'labels', + 'state' + ); + + for (const pr of this.prs) { + yield row( + pr.area.title, + pr.versions.map((v) => v.label).join(', '), + pr.user.name || pr.user.login, + pr.title, + pr.number, + pr.url, + pr.mergedAt, + pr.fixes.join(', '), + pr.labels.join(', '), + pr.state + ); + } + } +} diff --git a/packages/kbn-release-notes/src/formats/format.ts b/packages/kbn-release-notes/src/formats/format.ts new file mode 100644 index 000000000000..41b769ab05de --- /dev/null +++ b/packages/kbn-release-notes/src/formats/format.ts @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; + +import { Version, ClassifiedPr } from '../lib'; + +export abstract class Format { + static extension: string; + + constructor( + protected readonly version: Version, + protected readonly prs: ClassifiedPr[], + protected readonly log: ToolingLog + ) {} + + abstract print(): Iterator; +} diff --git a/packages/kbn-release-notes/src/formats/index.ts b/packages/kbn-release-notes/src/formats/index.ts new file mode 100644 index 000000000000..3403e445a84a --- /dev/null +++ b/packages/kbn-release-notes/src/formats/index.ts @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ArrayItem } from '../lib'; +import { AsciidocFormat } from './asciidoc'; +import { CsvFormat } from './csv'; + +export const FORMATS = [CsvFormat, AsciidocFormat] as const; +export type SomeFormat = ArrayItem; diff --git a/packages/kbn-release-notes/src/index.ts b/packages/kbn-release-notes/src/index.ts new file mode 100644 index 000000000000..a05bc698bde1 --- /dev/null +++ b/packages/kbn-release-notes/src/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from './cli'; diff --git a/packages/kbn-release-notes/src/lib/classify_pr.ts b/packages/kbn-release-notes/src/lib/classify_pr.ts new file mode 100644 index 000000000000..c567935ab7e4 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/classify_pr.ts @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; + +import { + Area, + AREAS, + UNKNOWN_AREA, + AsciidocSection, + ASCIIDOC_SECTIONS, + UNKNOWN_ASCIIDOC_SECTION, +} from '../release_notes_config'; +import { PullRequest } from './pull_request'; + +export interface ClassifiedPr extends PullRequest { + area: Area; + asciidocSection: AsciidocSection; +} + +export function classifyPr(pr: PullRequest, log: ToolingLog): ClassifiedPr { + const filter = (a: Area | AsciidocSection) => + a.labels.some((test) => + typeof test === 'string' ? pr.labels.includes(test) : pr.labels.some((l) => l.match(test)) + ); + + const areas = AREAS.filter(filter); + const asciidocSections = ASCIIDOC_SECTIONS.filter(filter); + + const pickOne = (name: string, options: T[]) => { + if (options.length > 1) { + const matches = options.map((o) => o.title).join(', '); + log.warning(`[${pr.terminalLink}] ambiguous ${name}, mulitple match [${matches}]`); + return options[0]; + } + + if (options.length === 0) { + log.error(`[${pr.terminalLink}] unable to determine ${name} because none match`); + return; + } + + return options[0]; + }; + + return { + ...pr, + area: pickOne('area', areas) || UNKNOWN_AREA, + asciidocSection: pickOne('asciidoc section', asciidocSections) || UNKNOWN_ASCIIDOC_SECTION, + }; +} diff --git a/packages/kbn-release-notes/src/lib/get_fix_references.test.ts b/packages/kbn-release-notes/src/lib/get_fix_references.test.ts new file mode 100644 index 000000000000..bdac66f6cc02 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_fix_references.test.ts @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getFixReferences } from './get_fix_references'; + +it('returns all fixed issue mentions in the PR text', () => { + expect( + getFixReferences(` + clOses #1 + closes: #2 + clOse #3 + close: #4 + clOsed #5 + closed: #6 + fiX #7 + fix: #8 + fiXes #9 + fixes: #10 + fiXed #11 + fixed: #12 + reSolve #13 + resolve: #14 + reSolves #15 + resolves: #16 + reSolved #17 + resolved: #18 + fixed + #19 + `) + ).toMatchInlineSnapshot(` + Array [ + "#1", + "#2", + "#3", + "#4", + "#5", + "#6", + "#7", + "#8", + "#9", + "#10", + "#11", + "#12", + "#13", + "#14", + "#15", + "#16", + "#17", + "#18", + ] + `); +}); diff --git a/packages/kbn-release-notes/src/lib/get_fix_references.ts b/packages/kbn-release-notes/src/lib/get_fix_references.ts new file mode 100644 index 000000000000..f45994e90ae8 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_fix_references.ts @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const FIXES_RE = /(?:closes|close|closed|fix|fixes|fixed|resolve|resolves|resolved)[ :]*(#\d*)/gi; + +export function getFixReferences(prText: string) { + const fixes: string[] = []; + let match; + while ((match = FIXES_RE.exec(prText))) { + fixes.push(match[1]); + } + return fixes; +} diff --git a/packages/kbn-release-notes/src/lib/get_note_from_description.test.ts b/packages/kbn-release-notes/src/lib/get_note_from_description.test.ts new file mode 100644 index 000000000000..23dcb302f090 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_note_from_description.test.ts @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import MarkdownIt from 'markdown-it'; +import dedent from 'dedent'; + +import { getNoteFromDescription } from './get_note_from_description'; + +it('extracts expected components from html', () => { + const mk = new MarkdownIt(); + + expect( + getNoteFromDescription( + mk.render(dedent` + My PR description + + Fixes: #1234 + + ## Release Note: + + Checkout this feature + `) + ) + ).toMatchInlineSnapshot(`"Checkout this feature"`); + + expect( + getNoteFromDescription( + mk.render(dedent` + My PR description + + Fixes: #1234 + + #### Release Note: + + We fixed an issue + `) + ) + ).toMatchInlineSnapshot(`"We fixed an issue"`); + + expect( + getNoteFromDescription( + mk.render(dedent` + My PR description + + Fixes: #1234 + + Release note: Checkout feature foo + `) + ) + ).toMatchInlineSnapshot(`"Checkout feature foo"`); + + expect( + getNoteFromDescription( + mk.render(dedent` + # Summary + + My PR description + + release note : bar + `) + ) + ).toMatchInlineSnapshot(`"bar"`); +}); diff --git a/packages/kbn-release-notes/src/lib/get_note_from_description.ts b/packages/kbn-release-notes/src/lib/get_note_from_description.ts new file mode 100644 index 000000000000..57df203470a5 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/get_note_from_description.ts @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import cheerio from 'cheerio'; + +export function getNoteFromDescription(descriptionHtml: string) { + const $ = cheerio.load(descriptionHtml); + for (const el of $('p,h1,h2,h3,h4,h5').toArray()) { + const text = $(el).text(); + const match = text.match(/^(\s*release note(?:s)?\s*:?\s*)/i); + + if (!match) { + continue; + } + + const note = text.replace(match[1], '').trim(); + return note || $(el).next().text().trim(); + } +} diff --git a/packages/kbn-release-notes/src/lib/index.ts b/packages/kbn-release-notes/src/lib/index.ts new file mode 100644 index 000000000000..00d8f49cf763 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/index.ts @@ -0,0 +1,26 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from './pull_request'; +export * from './version'; +export * from './is_pr_relevant'; +export * from './streams'; +export * from './type_helpers'; +export * from './irrelevant_pr_summary'; +export * from './classify_pr'; diff --git a/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts b/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts new file mode 100644 index 000000000000..1a458a04c774 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ToolingLog } from '@kbn/dev-utils'; + +import { PullRequest } from './pull_request'; +import { Version } from './version'; + +export class IrrelevantPrSummary { + private readonly stats = { + 'skipped by label': new Map(), + 'skipped by label regexp': new Map(), + 'skipped by version': new Map(), + }; + + constructor(private readonly log: ToolingLog) {} + + skippedByLabel(pr: PullRequest, label: string) { + this.log.debug(`${pr.terminalLink} skipped, label [${label}] is ignored`); + this.increment('skipped by label', label); + } + + skippedByLabelRegExp(pr: PullRequest, regexp: RegExp, label: string) { + this.log.debug(`${pr.terminalLink} skipped, label [${label}] matches regexp [${regexp}]`); + this.increment('skipped by label regexp', `${regexp}`); + } + + skippedByVersion(pr: PullRequest, earliestVersion: Version) { + this.log.debug(`${pr.terminalLink} skipped, earliest version is [${earliestVersion.label}]`); + this.increment('skipped by version', earliestVersion.label); + } + + private increment(stat: keyof IrrelevantPrSummary['stats'], key: string) { + const n = this.stats[stat].get(key) || 0; + this.stats[stat].set(key, n + 1); + } + + logStats() { + for (const [description, stats] of Object.entries(this.stats)) { + for (const [key, count] of stats) { + this.log.warning(`${count} ${count === 1 ? 'pr was' : 'prs were'} ${description} [${key}]`); + } + } + } +} diff --git a/packages/kbn-release-notes/src/lib/is_pr_relevant.ts b/packages/kbn-release-notes/src/lib/is_pr_relevant.ts new file mode 100644 index 000000000000..af2ef9440ded --- /dev/null +++ b/packages/kbn-release-notes/src/lib/is_pr_relevant.ts @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Version } from './version'; +import { PullRequest } from './pull_request'; +import { IGNORE_LABELS } from '../release_notes_config'; +import { IrrelevantPrSummary } from './irrelevant_pr_summary'; + +export function isPrRelevant( + pr: PullRequest, + version: Version, + includeVersions: Version[], + summary: IrrelevantPrSummary +) { + for (const label of IGNORE_LABELS) { + if (typeof label === 'string') { + if (pr.labels.includes(label)) { + summary.skippedByLabel(pr, label); + return false; + } + } + + if (label instanceof RegExp) { + const matching = pr.labels.find((l) => label.test(l)); + if (matching) { + summary.skippedByLabelRegExp(pr, label, matching); + return false; + } + } + } + + const [earliestVersion] = Version.sort( + // filter out `includeVersions` so that they won't be considered the "earliest version", only + // versions which are actually before the current `version` or the `version` itself are eligible + pr.versions.filter((v) => !includeVersions.includes(v)), + 'asc' + ); + + if (version !== earliestVersion) { + summary.skippedByVersion(pr, earliestVersion); + return false; + } + + return true; +} diff --git a/packages/kbn-release-notes/src/lib/pull_request.ts b/packages/kbn-release-notes/src/lib/pull_request.ts new file mode 100644 index 000000000000..e61e49664206 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/pull_request.ts @@ -0,0 +1,206 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { inspect } from 'util'; + +import Axios from 'axios'; +import gql from 'graphql-tag'; +import * as GraphqlPrinter from 'graphql/language/printer'; +import { DocumentNode } from 'graphql/language/ast'; +import makeTerminalLink from 'terminal-link'; +import { ToolingLog } from '@kbn/dev-utils'; + +import { Version } from './version'; +import { getFixReferences } from './get_fix_references'; +import { getNoteFromDescription } from './get_note_from_description'; + +const PrNodeFragment = gql` + fragment PrNode on PullRequest { + number + url + title + bodyText + bodyHTML + mergedAt + baseRefName + state + author { + login + ... on User { + name + } + } + labels(first: 100) { + nodes { + name + } + } + } +`; + +export interface PullRequest { + number: number; + url: string; + title: string; + targetBranch: string; + mergedAt: string; + state: string; + labels: string[]; + fixes: string[]; + user: { + name: string; + login: string; + }; + versions: Version[]; + terminalLink: string; + note?: string; +} + +/** + * Send a single request to the Github v4 GraphQL API + */ +async function gqlRequest( + token: string, + query: DocumentNode, + variables: Record = {} +) { + const resp = await Axios.request({ + url: 'https://api.github.com/graphql', + method: 'POST', + headers: { + 'user-agent': '@kbn/release-notes', + authorization: `bearer ${token}`, + }, + data: { + query: GraphqlPrinter.print(query), + variables, + }, + }); + + return resp.data; +} + +/** + * Convert the Github API response into the structure used by this tool + * + * @param node A GraphQL response from Github using the PrNode fragment + */ +function parsePullRequestNode(node: any): PullRequest { + const terminalLink = makeTerminalLink(`#${node.number}`, node.url); + + const labels: string[] = node.labels.nodes.map((l: { name: string }) => l.name); + + return { + number: node.number, + url: node.url, + terminalLink, + title: node.title, + targetBranch: node.baseRefName, + state: node.state, + mergedAt: node.mergedAt, + labels, + fixes: getFixReferences(node.bodyText), + user: { + login: node.author?.login || 'deleted user', + name: node.author?.name, + }, + versions: labels + .map((l) => Version.fromLabel(l)) + .filter((v): v is Version => v instanceof Version), + note: getNoteFromDescription(node.bodyHTML), + }; +} + +/** + * Iterate all of the PRs which have the `version` label + */ +export async function* iterRelevantPullRequests(token: string, version: Version, log: ToolingLog) { + let nextCursor: string | undefined; + let hasNextPage = true; + + while (hasNextPage) { + const resp = await gqlRequest( + token, + gql` + query($cursor: String, $labels: [String!]) { + repository(owner: "elastic", name: "kibana") { + pullRequests(first: 100, after: $cursor, labels: $labels, states: MERGED) { + pageInfo { + hasNextPage + endCursor + } + nodes { + ...PrNode + } + } + } + } + ${PrNodeFragment} + `, + { + cursor: nextCursor, + labels: [version.label], + } + ); + + const pullRequests = resp.data?.repository?.pullRequests; + if (!pullRequests) { + throw new Error(`unexpected github response, unable to fetch PRs: ${inspect(resp)}`); + } + + hasNextPage = pullRequests.pageInfo?.hasNextPage; + nextCursor = pullRequests.pageInfo?.endCursor; + + if (hasNextPage === undefined || (hasNextPage && !nextCursor)) { + throw new Error( + `github response does not include valid pagination information: ${inspect(resp)}` + ); + } + + for (const node of pullRequests.nodes) { + yield parsePullRequestNode(node); + } + } +} + +export async function getPr(token: string, number: number) { + const resp = await gqlRequest( + token, + gql` + query($number: Int!) { + repository(owner: "elastic", name: "kibana") { + pullRequest(number: $number) { + ...PrNode + } + } + } + ${PrNodeFragment} + `, + { + number, + } + ); + + const node = resp.data?.repository?.pullRequest; + if (!node) { + throw new Error(`unexpected github response, unable to fetch PR: ${inspect(resp)}`); + } + + return parsePullRequestNode(node); +} diff --git a/packages/kbn-release-notes/src/lib/streams.ts b/packages/kbn-release-notes/src/lib/streams.ts new file mode 100644 index 000000000000..f8cb9ec39186 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/streams.ts @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { promisify } from 'util'; +import { Readable, pipeline } from 'stream'; + +/** + * @types/node still doesn't have this method that was added + * in 10.17.0 https://nodejs.org/api/stream.html#stream_stream_readable_from_iterable_options + */ +export function streamFromIterable( + iter: Iterable | AsyncIterable +): Readable { + // @ts-ignore + return Readable.from(iter); +} + +export const asyncPipeline = promisify(pipeline); diff --git a/packages/kbn-release-notes/src/lib/type_helpers.ts b/packages/kbn-release-notes/src/lib/type_helpers.ts new file mode 100644 index 000000000000..c9402b358495 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/type_helpers.ts @@ -0,0 +1,20 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export type ArrayItem = T extends ReadonlyArray ? X : never; diff --git a/packages/kbn-release-notes/src/lib/version.test.ts b/packages/kbn-release-notes/src/lib/version.test.ts new file mode 100644 index 000000000000..afef26186569 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/version.test.ts @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Version } from './version'; + +it('parses version labels, returns null on failure', () => { + expect(Version.fromLabel('v1.0.2')).toMatchInlineSnapshot(` + Version { + "label": "v1.0.2", + "major": 1, + "minor": 0, + "patch": 2, + "tag": undefined, + "tagNum": undefined, + "tagOrder": Infinity, + } + `); + expect(Version.fromLabel('v1.0.0')).toMatchInlineSnapshot(` + Version { + "label": "v1.0.0", + "major": 1, + "minor": 0, + "patch": 0, + "tag": undefined, + "tagNum": undefined, + "tagOrder": Infinity, + } + `); + expect(Version.fromLabel('v9.0.2')).toMatchInlineSnapshot(` + Version { + "label": "v9.0.2", + "major": 9, + "minor": 0, + "patch": 2, + "tag": undefined, + "tagNum": undefined, + "tagOrder": Infinity, + } + `); + expect(Version.fromLabel('v9.0.2-alpha0')).toMatchInlineSnapshot(` + Version { + "label": "v9.0.2-alpha0", + "major": 9, + "minor": 0, + "patch": 2, + "tag": "alpha", + "tagNum": 0, + "tagOrder": 1, + } + `); + expect(Version.fromLabel('v9.0.2-beta1')).toMatchInlineSnapshot(` + Version { + "label": "v9.0.2-beta1", + "major": 9, + "minor": 0, + "patch": 2, + "tag": "beta", + "tagNum": 1, + "tagOrder": 2, + } + `); + expect(Version.fromLabel('v9.0')).toMatchInlineSnapshot(`undefined`); + expect(Version.fromLabel('some:area')).toMatchInlineSnapshot(`undefined`); +}); + +it('sorts versions in ascending order', () => { + const versions = [ + 'v1.7.3', + 'v1.7.0', + 'v1.5.0', + 'v2.7.0', + 'v7.0.0-beta2', + 'v7.0.0-alpha1', + 'v2.0.0', + 'v0.0.0', + 'v7.0.0-beta1', + 'v7.0.0', + ].map((l) => Version.fromLabel(l)!); + + const sorted = Version.sort(versions); + + expect(sorted.map((v) => v.label)).toMatchInlineSnapshot(` + Array [ + "v0.0.0", + "v1.5.0", + "v1.7.0", + "v1.7.3", + "v2.0.0", + "v2.7.0", + "v7.0.0-alpha1", + "v7.0.0-beta1", + "v7.0.0-beta2", + "v7.0.0", + ] + `); + + // ensure versions was not mutated + expect(sorted).not.toEqual(versions); +}); + +it('sorts versions in decending order', () => { + const versions = [ + 'v1.7.3', + 'v1.7.0', + 'v1.5.0', + 'v7.0.0-beta1', + 'v2.7.0', + 'v2.0.0', + 'v0.0.0', + 'v7.0.0', + ].map((l) => Version.fromLabel(l)!); + + const sorted = Version.sort(versions, 'desc'); + + expect(sorted.map((v) => v.label)).toMatchInlineSnapshot(` + Array [ + "v7.0.0", + "v7.0.0-beta1", + "v2.7.0", + "v2.0.0", + "v1.7.3", + "v1.7.0", + "v1.5.0", + "v0.0.0", + ] + `); + + // ensure versions was not mutated + expect(sorted).not.toEqual(versions); +}); diff --git a/packages/kbn-release-notes/src/lib/version.ts b/packages/kbn-release-notes/src/lib/version.ts new file mode 100644 index 000000000000..e0a5c5e177c8 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/version.ts @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const LABEL_RE = /^v(\d+)\.(\d+)\.(\d+)(?:-(alpha|beta)(\d+))?$/; + +const versionCache = new Map(); + +const multiCompare = (...diffs: number[]) => { + for (const diff of diffs) { + if (diff !== 0) { + return diff; + } + } + return 0; +}; + +export class Version { + static fromFlag(flag: string | string[] | boolean | undefined) { + if (typeof flag !== 'string') { + return; + } + + return Version.fromLabel(flag) || Version.fromLabel(`v${flag}`); + } + + static fromFlags(flag: string | string[] | boolean | undefined) { + const flags = Array.isArray(flag) ? flag : [flag]; + const versions: Version[] = []; + + for (const f of flags) { + const version = Version.fromFlag(f); + if (!version) { + return; + } + versions.push(version); + } + + return versions; + } + + static fromLabel(label: string) { + const match = label.match(LABEL_RE); + if (!match) { + return; + } + + const cached = versionCache.get(label); + if (cached) { + return cached; + } + + const [, major, minor, patch, tag, tagNum] = match; + const version = new Version( + parseInt(major, 10), + parseInt(minor, 10), + parseInt(patch, 10), + tag as 'alpha' | 'beta' | undefined, + tagNum ? parseInt(tagNum, 10) : undefined + ); + + versionCache.set(label, version); + return version; + } + + static sort(versions: Version[], dir: 'asc' | 'desc' = 'asc') { + const order = dir === 'asc' ? 1 : -1; + + return versions.slice().sort((a, b) => a.compare(b) * order); + } + + public readonly label = `v${this.major}.${this.minor}.${this.patch}${ + this.tag ? `-${this.tag}${this.tagNum}` : '' + }`; + private readonly tagOrder: number; + + constructor( + public readonly major: number, + public readonly minor: number, + public readonly patch: number, + public readonly tag: 'alpha' | 'beta' | undefined, + public readonly tagNum: number | undefined + ) { + switch (tag) { + case undefined: + this.tagOrder = Infinity; + break; + case 'alpha': + this.tagOrder = 1; + break; + case 'beta': + this.tagOrder = 2; + break; + default: + throw new Error('unexpected tag'); + } + } + + compare(other: Version) { + return multiCompare( + this.major - other.major, + this.minor - other.minor, + this.patch - other.patch, + this.tagOrder - other.tagOrder, + (this.tagNum ?? 0) - (other.tagNum ?? 0) + ); + } +} diff --git a/packages/kbn-release-notes/src/release_notes_config.ts b/packages/kbn-release-notes/src/release_notes_config.ts new file mode 100644 index 000000000000..88ab5dfa2fda --- /dev/null +++ b/packages/kbn-release-notes/src/release_notes_config.ts @@ -0,0 +1,294 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Exclude any PR from release notes that has a matching label. String + * labels must match exactly, for more complicated use a RegExp + */ +export const IGNORE_LABELS: Array = [ + 'Team:Docs', + ':KibanaApp/fix-it-week', + 'reverted', + /^test/, + 'non-issue', + 'jenkins', + 'build', + 'chore', + 'backport', + 'release_note:skip', + 'release_note:dev_docs', +]; + +/** + * Define areas that are used to categorize changes in the release notes + * based on the labels a PR has. the `labels` array can contain strings, which + * are matched exactly, or regular expressions. The first area, in definition + * order, which has a `label` which matches and label on a PR is the area + * assigned to that PR. + */ + +export interface Area { + title: string; + labels: Array; +} + +export const AREAS: Area[] = [ + { + title: 'Design', + labels: ['Team:Design', 'Project:Accessibility'], + }, + { + title: 'Logstash', + labels: ['App:Logstash', 'Feature:Logstash Pipelines'], + }, + { + title: 'Management', + labels: [ + 'Feature:license', + 'Feature:Console', + 'Feature:Search Profiler', + 'Feature:watcher', + 'Feature:Index Patterns', + 'Feature:Kibana Management', + 'Feature:Dev Tools', + 'Feature:Inspector', + 'Feature:Index Management', + 'Feature:Snapshot and Restore', + 'Team:Elasticsearch UI', + 'Feature:FieldFormatters', + 'Feature:CCR', + 'Feature:ILM', + 'Feature:Transforms', + ], + }, + { + title: 'Monitoring', + labels: ['Team:Monitoring', 'Feature:Telemetry', 'Feature:Stack Monitoring'], + }, + { + title: 'Operations', + labels: ['Team:Operations', 'Feature:License'], + }, + { + title: 'Kibana UI', + labels: ['Kibana UI', 'Team:Core UI', 'Feature:Header'], + }, + { + title: 'Platform', + labels: [ + 'Team:Platform', + 'Feature:Plugins', + 'Feature:New Platform', + 'Project:i18n', + 'Feature:ExpressionLanguage', + 'Feature:Saved Objects', + 'Team:Stack Services', + 'Feature:NP Migration', + 'Feature:Task Manager', + 'Team:Pulse', + ], + }, + { + title: 'Machine Learning', + labels: [ + ':ml', + 'Feature:Anomaly Detection', + 'Feature:Data Frames', + 'Feature:File Data Viz', + 'Feature:ml-results', + 'Feature:Data Frame Analytics', + ], + }, + { + title: 'Maps', + labels: ['Team:Geo'], + }, + { + title: 'Canvas', + labels: ['Team:Canvas'], + }, + { + title: 'QA', + labels: ['Team:QA'], + }, + { + title: 'Security', + labels: [ + 'Team:Security', + 'Feature:Security/Spaces', + 'Feature:users and roles', + 'Feature:Security/Authentication', + 'Feature:Security/Authorization', + 'Feature:Security/Feature Controls', + ], + }, + { + title: 'Dashboard', + labels: ['Feature:Dashboard', 'Feature:Drilldowns'], + }, + { + title: 'Discover', + labels: ['Feature:Discover'], + }, + { + title: 'Kibana Home & Add Data', + labels: ['Feature:Add Data', 'Feature:Home'], + }, + { + title: 'Querying & Filtering', + labels: [ + 'Feature:Query Bar', + 'Feature:Courier', + 'Feature:Filters', + 'Feature:Timepicker', + 'Feature:Highlight', + 'Feature:KQL', + 'Feature:Rollups', + ], + }, + { + title: 'Reporting', + labels: ['Feature:Reporting', 'Team:Reporting Services'], + }, + { + title: 'Sharing', + labels: ['Feature:Embedding', 'Feature:SharingURLs'], + }, + { + title: 'Visualizations', + labels: [ + 'Feature:Timelion', + 'Feature:TSVB', + 'Feature:Coordinate Map', + 'Feature:Region Map', + 'Feature:Vega', + 'Feature:Gauge Vis', + 'Feature:Tagcloud', + 'Feature:Vis Loader', + 'Feature:Vislib', + 'Feature:Vis Editor', + 'Feature:Aggregations', + 'Feature:Input Control', + 'Feature:Visualizations', + 'Feature:Markdown', + 'Feature:Data Table', + 'Feature:Heatmap', + 'Feature:Pie Chart', + 'Feature:XYAxis', + 'Feature:Graph', + 'Feature:New Feature', + 'Feature:MetricVis', + ], + }, + { + title: 'SIEM', + labels: ['Team:SIEM'], + }, + { + title: 'Code', + labels: ['Team:Code'], + }, + { + title: 'Infrastructure', + labels: ['App:Infrastructure', 'Feature:Infra UI', 'Feature:Service Maps'], + }, + { + title: 'Logs', + labels: ['App:Logs', 'Feature:Logs UI'], + }, + { + title: 'Uptime', + labels: ['App:Uptime', 'Feature:Uptime', 'Team:uptime'], + }, + { + title: 'Beats Management', + labels: ['App:Beats', 'Feature:beats-cm', 'Team:Beats'], + }, + { + title: 'APM', + labels: ['Team:apm', /^apm[:\-]/], + }, + { + title: 'Lens', + labels: ['App:Lens', 'Feature:Lens'], + }, + { + title: 'Alerting', + labels: ['App:Alerting', 'Feature:Alerting', 'Team:Alerting Services', 'Feature:Actions'], + }, + { + title: 'Metrics', + labels: ['App:Metrics', 'Feature:Metrics UI', 'Team:logs-metrics-ui'], + }, + { + title: 'Data ingest', + labels: ['Ingest', 'Feature:Ingest Node Pipelines'], + }, +]; + +export const UNKNOWN_AREA: Area = { + title: 'Unknown', + labels: [], +}; + +/** + * Define the sections that will be assigned to PRs when generating the + * asciidoc formatted report. The order of the sections determines the + * order they will be rendered in the report + */ + +export interface AsciidocSection { + title: string; + labels: Array; + id: string; +} + +export const ASCIIDOC_SECTIONS: AsciidocSection[] = [ + { + id: 'enhancement', + title: 'Enhancements', + labels: ['release_note:enhancement'], + }, + { + id: 'bug', + title: 'Bug fixes', + labels: ['release_note:fix'], + }, + { + id: 'roadmap', + title: 'Roadmap', + labels: ['release_note:roadmap'], + }, + { + id: 'deprecation', + title: 'Deprecations', + labels: ['release_note:deprecation'], + }, + { + id: 'breaking', + title: 'Breaking Changes', + labels: ['release_note:breaking'], + }, +]; + +export const UNKNOWN_ASCIIDOC_SECTION: AsciidocSection = { + id: 'unknown', + title: 'Unknown', + labels: [], +}; diff --git a/packages/kbn-release-notes/tsconfig.json b/packages/kbn-release-notes/tsconfig.json new file mode 100644 index 000000000000..6ffa64d91fba --- /dev/null +++ b/packages/kbn-release-notes/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./target", + "declaration": true, + "sourceMap": true, + "target": "ES2019" + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-release-notes/yarn.lock b/packages/kbn-release-notes/yarn.lock new file mode 120000 index 000000000000..3f82ebc9cdba --- /dev/null +++ b/packages/kbn-release-notes/yarn.lock @@ -0,0 +1 @@ +../../yarn.lock \ No newline at end of file diff --git a/scripts/release_notes.js b/scripts/release_notes.js new file mode 100644 index 000000000000..f46ee5823d70 --- /dev/null +++ b/scripts/release_notes.js @@ -0,0 +1,21 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +require('../src/setup_node_env/prebuilt_dev_only_entry'); +require('@kbn/release-notes').runReleaseNotesCli(); diff --git a/yarn.lock b/yarn.lock index 8c795a27e10a..20089fdb8309 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5355,9 +5355,9 @@ "@types/node" "*" "@types/node@*", "@types/node@8.10.54", "@types/node@>=10.17.17 <10.20.0", "@types/node@>=8.9.0", "@types/node@^12.0.2": - version "10.17.17" - resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.17.tgz#7a183163a9e6ff720d86502db23ba4aade5999b8" - integrity sha512-gpNnRnZP3VWzzj5k3qrpRC6Rk3H/uclhAVo1aIvwzK5p5cOrs9yEyQ8H/HBsBY0u5rrWxXEiVPQ0dEB6pkjE8Q== + version "10.17.26" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.26.tgz#a8a119960bff16b823be4c617da028570779bcfd" + integrity sha512-myMwkO2Cr82kirHY8uknNRHEVtn0wV3DTQfkrjx17jmkstDRZ24gNUdl8AHXVyVclTYI/bNjgTPTAWvWLqXqkw== "@types/nodemailer@^6.2.1": version "6.2.1" @@ -16148,6 +16148,11 @@ graphql-tag@2.10.1, graphql-tag@^2.9.2: resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.10.1.tgz#10aa41f1cd8fae5373eaf11f1f67260a3cad5e02" integrity sha512-jApXqWBzNXQ8jYa/HLkZJaVw9jgwNqZkywa2zfFn16Iv1Zb7ELNHkJaXHR7Quvd5SIGsy6Ny7SUKATgnu05uEg== +graphql-tag@^2.10.3: + version "2.10.3" + resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.10.3.tgz#ea1baba5eb8fc6339e4c4cf049dabe522b0edf03" + integrity sha512-4FOv3ZKfA4WdOKJeHdz6B3F/vxBLSgmBcGeAFPf4n1F64ltJUvOOerNj0rsJxONQGdhUMynQIvd6LzB+1J5oKA== + graphql-toolkit@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/graphql-toolkit/-/graphql-toolkit-0.2.0.tgz#91364b69911d51bc915269a37963f4ea2d5f335c" @@ -16194,6 +16199,13 @@ graphql@^0.13.2: dependencies: iterall "^1.2.1" +graphql@^14.0.0: + version "14.6.0" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.6.0.tgz#57822297111e874ea12f5cd4419616930cd83e49" + integrity sha512-VKzfvHEKybTKjQVpTFrA5yUq2S9ihcZvfJAtsDBBCuV6wauPu1xl/f9ehgVf0FcEJJs4vz6ysb/ZMkGigQZseg== + dependencies: + iterall "^1.2.2" + graphviz@^0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/graphviz/-/graphviz-0.0.8.tgz#e599e40733ef80e1653bfe89a5f031ecf2aa4aaa" @@ -18731,6 +18743,11 @@ iterall@^1.1.3, iterall@^1.2.1: resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.2.2.tgz#92d70deb8028e0c39ff3164fdbf4d8b088130cd7" integrity sha512-yynBb1g+RFUPY64fTrFv7nsjRrENBQJaX2UL+2Szc9REFrSNm1rpSXHGzhmAy7a9uv3vlvgBlXnf9RqmPH1/DA== +iterall@^1.2.2: + version "1.3.0" + resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea" + integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg== + jest-changed-files@^24.9.0: version "24.9.0" resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-24.9.0.tgz#08d8c15eb79a7fa3fc98269bc14b451ee82f8039" @@ -29202,6 +29219,14 @@ supports-hyperlinks@^1.0.1: has-flag "^2.0.0" supports-color "^5.0.0" +supports-hyperlinks@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.1.0.tgz#f663df252af5f37c5d49bbd7eeefa9e0b9e59e47" + integrity sha512-zoE5/e+dnEijk6ASB6/qrK+oYdm2do1hjoLWrqUC/8WEIW1gbxFcKuBof7sW8ArN6e+AYvsE8HBGiVRWL/F5CA== + dependencies: + has-flag "^4.0.0" + supports-color "^7.0.0" + suricata-sid-db@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/suricata-sid-db/-/suricata-sid-db-1.0.2.tgz#96ceda4db117a9f1282c8f9d785285e5ccf342b1" @@ -29554,6 +29579,14 @@ term-size@^2.1.0: resolved "https://registry.yarnpkg.com/term-size/-/term-size-2.2.0.tgz#1f16adedfe9bdc18800e1776821734086fcc6753" integrity sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw== +terminal-link@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/terminal-link/-/terminal-link-2.1.1.tgz#14a64a27ab3c0df933ea546fba55f2d078edc994" + integrity sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ== + dependencies: + ansi-escapes "^4.2.1" + supports-hyperlinks "^2.0.0" + terser-webpack-plugin@^1.2.4, terser-webpack-plugin@^1.4.3: version "1.4.4" resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.4.4.tgz#2c63544347324baafa9a56baaddf1634c8abfc2f"