Merge branch 'master' into hosted-policy-detail-changes

docs/developer/plugin-list.asciidoc

@@ -485,6 +485,10 @@ Elastic.
 |Welcome to the Kibana rollup plugin! This plugin provides Kibana support for Elasticsearch's rollup feature. Please refer to the Elasticsearch documentation to understand rollup indices and how to create rollup jobs.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/rule_registry/README.md[ruleRegistry]
+|The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/runtime_fields/README.md[runtimeFields]
 |Welcome to the home of the runtime field editor and everything related to runtime fields!
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternselectprops.md

@@ -12,6 +12,5 @@ export declare type IndexPatternSelectProps = Required<Omit<EuiComboBoxProps<any
     indexPatternId: string;
     fieldTypes?: string[];
     onNoIndexPatterns?: () => void;
-    maxIndexPatterns?: number;
 };
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md

@@ -19,6 +19,7 @@ export interface ISearchOptions
 |  [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
+|  [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | <code>RequestResponder</code> |  |
 |  [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md)
+
+## ISearchOptions.requestResponder property
+
+<b>Signature:</b>
+
+```typescript
+requestResponder?: RequestResponder;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md

@@ -53,7 +53,6 @@ search: {
             timeRange: import("../common").TimeRange | undefined;
         } | undefined;
     };
-    getRequestInspectorStats: typeof getRequestInspectorStats;
     getResponseInspectorStats: typeof getResponseInspectorStats;
     tabifyAggResponse: typeof tabifyAggResponse;
     tabifyGetColumns: typeof tabifyGetColumns;

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchbar.md

@@ -7,7 +7,7 @@
 <b>Signature:</b>
 
 ```typescript
-SearchBar: React.ComponentClass<Pick<Pick<SearchBarProps, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "isInvalid" | "storageKey" | "intl" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "disableLanguageSwitcher" | "autoSubmit" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated">, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "isInvalid" | "storageKey" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "disableLanguageSwitcher" | "autoSubmit" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated">, any> & {
-    WrappedComponent: React.ComponentType<Pick<SearchBarProps, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "isInvalid" | "storageKey" | "intl" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "disableLanguageSwitcher" | "autoSubmit" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated"> & ReactIntl.InjectedIntlProps>;
+SearchBar: React.ComponentClass<Pick<Pick<SearchBarProps, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "intl" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated">, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated">, any> & {
+    WrappedComponent: React.ComponentType<Pick<SearchBarProps, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "intl" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated"> & ReactIntl.InjectedIntlProps>;
 }
 ```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md

@@ -9,9 +9,9 @@ Returns body contents of the search request, often referred as query DSL.
 <b>Signature:</b>
 
 ```typescript
-getSearchRequestBody(): Promise<any>;
+getSearchRequestBody(): any;
 ```
 <b>Returns:</b>
 
-`Promise<any>`
+`any`
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md

@@ -19,6 +19,7 @@ export interface ISearchOptions
 |  [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
 |  [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
+|  [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | <code>RequestResponder</code> |  |
 |  [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md)
+
+## ISearchOptions.requestResponder property
+
+<b>Signature:</b>
+
+```typescript
+requestResponder?: RequestResponder;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md

@@ -36,8 +36,6 @@ search: {
         toAbsoluteDates: typeof toAbsoluteDates;
         calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan;
     };
-    getRequestInspectorStats: typeof getRequestInspectorStats;
-    getResponseInspectorStats: typeof getResponseInspectorStats;
     tabifyAggResponse: typeof tabifyAggResponse;
     tabifyGetColumns: typeof tabifyGetColumns;
 }

examples/search_examples/public/search/app.tsx

@@ -204,8 +204,8 @@ export const SearchExamplesApp = ({
         });
       }
 
-      setRequest(await searchSource.getSearchRequestBody());
-      const res = await searchSource.fetch();
+      setRequest(searchSource.getSearchRequestBody());
+      const res = await searchSource.fetch$().toPromise();
       setResponse(res);
 
       const message = <EuiText>Searched {res.hits.total} documents.</EuiText>;

src/core/server/config/ensure_valid_configuration.test.ts

@@ -16,14 +16,40 @@ describe('ensureValidConfiguration', () => {
   beforeEach(() => {
     jest.clearAllMocks();
     configService = configServiceMock.create();
-    configService.getUsedPaths.mockReturnValue(Promise.resolve(['core', 'elastic']));
+
+    configService.validate.mockResolvedValue();
+    configService.getUsedPaths.mockReturnValue(Promise.resolve([]));
   });
 
-  it('returns normally when there is no unused keys', async () => {
-    configService.getUnusedPaths.mockResolvedValue([]);
+  it('returns normally when there is no unused keys and when the config validates', async () => {
     await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined();
   });
 
+  it('throws when config validation fails', async () => {
+    configService.validate.mockImplementation(() => {
+      throw new Error('some message');
+    });
+
+    await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot(
+      `[Error: some message]`
+    );
+  });
+
+  it('throws a `CriticalError` with the correct processExitCode value when config validation fails', async () => {
+    expect.assertions(2);
+
+    configService.validate.mockImplementation(() => {
+      throw new Error('some message');
+    });
+
+    try {
+      await ensureValidConfiguration(configService as any);
+    } catch (e) {
+      expect(e).toBeInstanceOf(CriticalError);
+      expect(e.processExitCode).toEqual(78);
+    }
+  });
+
   it('throws when there are some unused keys', async () => {
     configService.getUnusedPaths.mockResolvedValue(['some.key', 'some.other.key']);
 
@@ -44,4 +70,18 @@ describe('ensureValidConfiguration', () => {
       expect(e.processExitCode).toEqual(64);
     }
   });
+
+  it('does not throw when all unused keys are included in the ignored paths', async () => {
+    configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'elastic.apm.enabled']);
+
+    await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined();
+  });
+
+  it('throws when only some keys are included in the ignored paths', async () => {
+    configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'some.key']);
+
+    await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot(
+      `[Error: Unknown configuration key(s): "some.key". Check for spelling errors and ensure that expected plugins are installed.]`
+    );
+  });
 });

src/core/server/config/ensure_valid_configuration.ts

@@ -9,22 +9,27 @@
 import { ConfigService } from '@kbn/config';
 import { CriticalError } from '../errors';
 
+const ignoredPaths = ['dev.', 'elastic.apm.'];
+
+const invalidConfigExitCode = 78;
+const legacyInvalidConfigExitCode = 64;
+
 export async function ensureValidConfiguration(configService: ConfigService) {
-  await configService.validate();
+  try {
+    await configService.validate();
+  } catch (e) {
+    throw new CriticalError(e.message, 'InvalidConfig', invalidConfigExitCode, e);
+  }
 
-  const unusedConfigKeys = await configService.getUnusedPaths();
+  const unusedPaths = await configService.getUnusedPaths();
+  const unusedConfigKeys = unusedPaths.filter((unusedPath) => {
+    return !ignoredPaths.some((ignoredPath) => unusedPath.startsWith(ignoredPath));
+  });
 
   if (unusedConfigKeys.length > 0) {
     const message = `Unknown configuration key(s): ${unusedConfigKeys
       .map((key) => `"${key}"`)
       .join(', ')}. Check for spelling errors and ensure that expected plugins are installed.`;
-    throw new InvalidConfigurationError(message);
-  }
-}
-
-class InvalidConfigurationError extends CriticalError {
-  constructor(message: string) {
-    super(message, 'InvalidConfig', 64);
-    Object.setPrototypeOf(this, InvalidConfigurationError.prototype);
+    throw new CriticalError(message, 'InvalidConfig', legacyInvalidConfigExitCode);
   }
 }

src/core/server/server.ts

@@ -36,7 +36,6 @@ import { config as cspConfig } from './csp';
 import { config as elasticsearchConfig } from './elasticsearch';
 import { config as httpConfig } from './http';
 import { config as loggingConfig } from './logging';
-import { config as devConfig } from './dev';
 import { config as kibanaConfig } from './kibana_config';
 import { savedObjectsConfig, savedObjectsMigrationConfig } from './saved_objects';
 import { config as uiSettingsConfig } from './ui_settings';
@@ -303,7 +302,6 @@ export class Server {
       loggingConfig,
       httpConfig,
       pluginsConfig,
-      devConfig,
       kibanaConfig,
       savedObjectsConfig,
       savedObjectsMigrationConfig,

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -535,6 +535,9 @@ export class IndexPatternsService {
     });
     indexPattern.id = response.id;
     this.indexPatternCache.set(indexPattern.id, Promise.resolve(indexPattern));
+    if (this.savedObjectsCache) {
+      this.savedObjectsCache.push(response as SavedObject<IndexPatternSavedObjectAttrs>);
+    }
     return indexPattern;
   }
 

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -8,7 +8,6 @@
 
 import { noop } from 'lodash';
 import { i18n } from '@kbn/i18n';
-import type { RequestAdapter } from 'src/plugins/inspector/common';
 
 import { BucketAggType, IBucketAggConfig } from './bucket_agg_type';
 import { BUCKET_TYPES } from './bucket_agg_types';
@@ -21,7 +20,6 @@ import { aggTermsFnName } from './terms_fn';
 import { AggConfigSerialized, BaseAggParams } from '../types';
 
 import { KBN_FIELD_TYPES } from '../../../../common';
-import { getRequestInspectorStats, getResponseInspectorStats } from '../../expressions';
 
 import {
   buildOtherBucketAgg,
@@ -103,36 +101,28 @@ export const getTermsBucketAgg = () =>
 
         nestedSearchSource.setField('aggs', filterAgg);
 
-        let request: ReturnType<RequestAdapter['start']> | undefined;
-        if (inspectorRequestAdapter) {
-          request = inspectorRequestAdapter.start(
-            i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', {
-              defaultMessage: 'Other bucket',
+        const requestResponder = inspectorRequestAdapter?.start(
+          i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', {
+            defaultMessage: 'Other bucket',
+          }),
+          {
+            description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', {
+              defaultMessage:
+                'This request counts the number of documents that fall ' +
+                'outside the criterion of the data buckets.',
             }),
-            {
-              description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', {
-                defaultMessage:
-                  'This request counts the number of documents that fall ' +
-                  'outside the criterion of the data buckets.',
-              }),
-              searchSessionId,
-            }
-          );
-          nestedSearchSource.getSearchRequestBody().then((body) => {
-            request!.json(body);
-          });
-          request.stats(getRequestInspectorStats(nestedSearchSource));
-        }
+            searchSessionId,
+          }
+        );
+
+        const response = await nestedSearchSource
+          .fetch$({
+            abortSignal,
+            sessionId: searchSessionId,
+            requestResponder,
+          })
+          .toPromise();
 
-        const response = await nestedSearchSource.fetch({
-          abortSignal,
-          sessionId: searchSessionId,
-        });
-        if (request) {
-          request
-            .stats(getResponseInspectorStats(response, nestedSearchSource))
-            .ok({ json: response });
-        }
         resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg());
       }
       if (aggConfig.params.missingBucket) {

src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts

@@ -133,7 +133,7 @@ describe('esaggs expression function - public', () => {
   test('calls searchSource.fetch', async () => {
     await handleRequest(mockParams);
     const searchSource = await mockParams.searchSourceService.create();
-    expect(searchSource.fetch).toHaveBeenCalledWith({
+    expect(searchSource.fetch$).toHaveBeenCalledWith({
       abortSignal: mockParams.abortSignal,
       sessionId: mockParams.searchSessionId,
     });