Merge branch 'master' into monitoring/bulk_uploader/use-default-client

docs/developer/telemetry.asciidoc

@@ -5,8 +5,9 @@ To help us provide a good developer experience, we track some straightforward me
 
 The operations we current report timing data for:
 
-* Total execution time of `yarn kbn bootstrap`
+* Total execution time of `yarn kbn bootstrap`.
 * Total execution time of `@kbn/optimizer` runs as well as the following metadata about the runs: The number of bundles created, the number of bundles which were cached, usage of `--watch`, `--dist`, `--workers` and `--no-cache` flags, and the count of themes being built.
+* The time from when you run `yarn start` until both the Kibana server and `@kbn/optimizer` are ready for use.
 
 Along with the execution time of each execution, we ship the following information about your machine to the service:
 

docs/discover/search-sessions.asciidoc

@@ -0,0 +1,72 @@
+[[search-sessions]]
+=== Run a search session in the background
+
+Sometimes you might need to search through large amounts of data no matter
+how long the search takes. While this might not happen often,
+there are times that long-running queries are required.
+Consider a threat hunting scenario, where you need to search through years of data.
+
+If your query is running long, you can save your search session, which
+allows {kib} to continue processing your request in the
+background.  Save your search session from *Discover* or *Dashboard*,
+and when your session is complete, view and manage it in *Stack Management*.
+
+[role="screenshot"]
+image::images/search-session.png[Search Session indicator displaying the current state of the search, which you can click to stop or save a running Search Session ]
+
+Search sessions are <<search-session-settings-kb,enabled by default>>. Saving
+a search session is only available when
+<<set-time-filter,auto refresh>> is off.
+
+
+[float]
+==== Requirements
+
+
+* To save a session, you must have permissions for *Discover* and *Dashboard*,
+and the <<kibana-feature-privileges, search sessions subfeature>>.
+
+* To view and restore a saved session, you must have access to *Stack Management*.
+
+[float]
+==== Example: Save a search session
+
+You’re trying to understand a trend you see on a dashboard. You
+need to look at several years of data, currently in
+{ref}/data-tiers.html#cold-tier[cold storage],
+but you don’t have time to wait. You want {kib} to
+continue working in the background, so tomorrow you can
+open your browser and pick up where you left off.
+
+. Load your dashboard.
++
+Your search session begins automatically. The icon after the dashboard title
+displays the current state of the search session. A clock indicates the search session is in progress.
+A checkmark indicates that the search session is complete.
+
+. To instruct {kib} to continue a search in the background, click the clock icon,
+and then click *Save session*. Once you save a search session, you can start a new search,
+navigate to a different application, or close the browser.
++
+[role="screenshot"]
+image::images/search-session-awhile.png[Search Session indicator displaying the current state of the search, which you can click to stop or save a running Search Session ]
+
+. To view your saved searches, open the main menu, and then click
+*Stack Management > Search Sessions*.  You can also open this view from the search sessions popup for a saved or completed session.
++
+[role="screenshot"]
+image::images/search-sessions-menu.png[Search Sessions management view with actions for inspecting, extending, and deleting a session. ]
+
+. Use the edit menu in *Search Sessions* to:
+* *Inspect* the queries and filters that makeup the session.
+* *Extend* the expiration of a completed session.
+* *Delete* a session.
+
+. To restore a search session, click its name in the *Search Sessions* view.
++
+You're returned to the place from where you started the search session. The data is the same, but
+behaves differently:
++
+* Relative dates are converted to absolute dates.
+* Panning and zooming is disabled for maps.
+* Changing a filter, query, or drilldown starts a new search session, which can be slow.

docs/discover/search.asciidoc

@@ -1,13 +1,15 @@
 [[search]]
-== Search data
-Many Kibana apps embed a query bar for real-time search, including
-*Discover* and *Dashboard*.
+== Search your data
+
+You can search your data in any app that has a query bar, or by clicking on
+elements in a visualization.  A search matches indices in the current
+<<index-patterns, index pattern>> and in the current <<set-time-filter,time frame>>.
+
 
 [float]
-=== Search your data
+=== Search with KQL
 
-To search the indices that match the current <<index-patterns, index pattern>>,
-enter your search criteria in the query bar. By default, you'll use
+By default, you search using
 {kib}'s <<kuery-query, standard query language>> (KQL), which
 features autocomplete and a simple, easy-to-use syntax. If you prefer to use
 {kib}'s legacy query
@@ -21,32 +23,17 @@ JSON-based {ref}/query-dsl.html[Elasticsearch Query DSL].
 [float]
 [[autorefresh]]
 === Refresh search results
-As more documents are added to the indices you're searching, the search results
-shown in *Discover*, and used to display visualizations, get stale. Using the
-time filter, you can
+As more documents are added to the indices you're searching, the search results get stale.
+Using the time filter, you can
 configure a refresh interval to periodically resubmit your searches to
 retrieve the latest results.
 
 [role="screenshot"]
-image::images/autorefresh-interval.png[Image showing what refresh interval option looks like. The configurable time interval is located in the dropdown]
+image::images/autorefresh-interval.png[Refresh interval option in time filter. The configurable time interval is located in the dropdown.]
 
 You can also manually refresh the search results by
 clicking the *Refresh* button.
 
-[float]
-=== Searching large amounts of data
-
-Sometimes you want to search through large amounts of data no matter how long
-the search takes.  While this might not happen often, there are times
-that long-running queries are required.  Consider a threat hunting scenario
-where you need to search through years of data.
-
-If you run a query, and the run time gets close to the
-timeout, you're presented the option to ignore the timeout.  This enables you to
-run queries with large amounts of data to completion.
-
-By default, a query times out after 30 seconds.
-The timeout is in place to avoid unintentional load on the cluster.
 
 
 include::kuery.asciidoc[]
@@ -211,3 +198,5 @@ To completely delete a query:
 image::discover/images/saved-query-management-component-delete-query-button.png["Example of the saved query management popover when a query is hovered over and we are about to delete a query",width="80%"]
 
 You can import, export, and delete saved queries from <<managing-saved-objects, Saved Objects in Management>>.
+
+include::search-sessions.asciidoc[]

docs/settings/search-sessions-settings.asciidoc

@@ -0,0 +1,25 @@
+
+[[search-session-settings-kb]]
+=== Search sessions settings in {kib}
+++++
+<titleabbrev>Search sessions settings</titleabbrev>
+++++
+
+Configure the search session settings in your `kibana.yml` configuration file.
+
+
+[cols="2*<"]
+|===
+a| `xpack.data_enhanced.`
+`search.sessions:enabled`
+  | Set to `true` (default) to enable search sessions.
+
+a| `xpack.data.enhanced.`
+`search.sessions:trackingInterval`
+  | The frequency for updating the state of a search session. The default is 10s.
+
+a| `xpack.data.enhanced.`
+`search.sessions:defaultExpiration`
+  | How long search session results are stored before they are deleted.
+  Extending a search session resets the expiration by the same value. The default is 7d.
+|===

docs/setup/settings.asciidoc

@@ -227,7 +227,7 @@ The default application to load. *Default: `"home"`*
 for more details. {kib} uses an index in {es} to store saved searches, visualizations, and
 dashboards. {kib} creates a new index if the index doesn’t already exist.
 If you configure a custom index, the name must be lowercase, and conform to the
-{es} {ref}/indices-create-index.html[index name limitations]. 
+{es} {ref}/indices-create-index.html[index name limitations].
 *Default: `".kibana"`*
 
 | `kibana.autocompleteTimeout:` {ess-icon}
@@ -475,7 +475,7 @@ running behind a proxy. Use the <<server-rewriteBasePath, `server.rewriteBasePat
 if it should remove the basePath from requests it receives, and to prevent a
 deprecation warning at startup. This setting cannot end in a slash (`/`).
 
-|[[server-publicBaseUrl]] `server.publicBaseUrl:`
+|[[server-publicBaseUrl]] `server.publicBaseUrl:` {ess-icon}
  | The publicly available URL that end-users access Kibana at. Must include the protocol, hostname, port
  (if different than the defaults for `http` and `https`, 80 and 443 respectively), and the
  <<server-basePath, `server.basePath`>> (if configured). This setting cannot end in a slash (`/`).
@@ -696,6 +696,7 @@ include::{kib-repo-dir}/settings/ml-settings.asciidoc[]
 include::{kib-repo-dir}/settings/monitoring-settings.asciidoc[]
 include::{kib-repo-dir}/settings/reporting-settings.asciidoc[]
 include::secure-settings.asciidoc[]
+include::{kib-repo-dir}/settings/search-sessions-settings.asciidoc[]
 include::{kib-repo-dir}/settings/security-settings.asciidoc[]
 include::{kib-repo-dir}/settings/spaces-settings.asciidoc[]
 include::{kib-repo-dir}/settings/task-manager-settings.asciidoc[]

docs/user/management.asciidoc

@@ -141,6 +141,12 @@ These include dashboards, visualizations, maps, index patterns, Canvas workpads,
 | <<managing-tags, Tags>>
 |Create, manage, and assign tags to your saved objects.
 
+| <<search-sessions, Search Sessions>>
+| Manage your saved search sessions, groups of queries that run in the background.
+Search sessions are useful when your queries take longer than usual to process,
+for example, when you have a large volume of data or when the performance of your storage location is slow.
+
+
 | <<xpack-spaces, Spaces>>
 | Create spaces to organize your dashboards and other saved objects into categories.
 A space is isolated from all other spaces,

src/core/server/ui_settings/settings/index.ts

@@ -15,14 +15,20 @@ import { getNotificationsSettings } from './notifications';
 import { getThemeSettings } from './theme';
 import { getStateSettings } from './state';
 
-export const getCoreSettings = (): Record<string, UiSettingsParams> => {
+interface GetCoreSettingsOptions {
+  isDist?: boolean;
+}
+
+export const getCoreSettings = (
+  options?: GetCoreSettingsOptions
+): Record<string, UiSettingsParams> => {
   return {
     ...getAccessibilitySettings(),
     ...getDateFormatSettings(),
     ...getMiscUiSettings(),
     ...getNavigationSettings(),
     ...getNotificationsSettings(),
-    ...getThemeSettings(),
+    ...getThemeSettings(options),
     ...getStateSettings(),
   };
 };

src/core/server/ui_settings/settings/theme.test.ts

@@ -44,3 +44,57 @@ describe('theme settings', () => {
     });
   });
 });
+
+describe('process.env.KBN_OPTIMIZER_THEMES handling', () => {
+  it('provides valid options based on tags', () => {
+    process.env.KBN_OPTIMIZER_THEMES = 'v7light,v8dark';
+    let settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:version'].options).toEqual(['v7', 'v8']);
+
+    process.env.KBN_OPTIMIZER_THEMES = 'v8dark,v7light';
+    settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:version'].options).toEqual(['v7', 'v8']);
+
+    process.env.KBN_OPTIMIZER_THEMES = 'v8dark,v7light,v7dark,v8light';
+    settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:version'].options).toEqual(['v7', 'v8']);
+
+    process.env.KBN_OPTIMIZER_THEMES = '*';
+    settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:version'].options).toEqual(['v7', 'v8']);
+
+    process.env.KBN_OPTIMIZER_THEMES = 'v7light';
+    settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:version'].options).toEqual(['v7']);
+
+    process.env.KBN_OPTIMIZER_THEMES = 'v8light';
+    settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:version'].options).toEqual(['v8']);
+  });
+
+  it('defaults to properties of first tag', () => {
+    process.env.KBN_OPTIMIZER_THEMES = 'v8dark,v7light';
+    let settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:darkMode'].value).toBe(true);
+    expect(settings['theme:version'].value).toBe('v8');
+
+    process.env.KBN_OPTIMIZER_THEMES = 'v7light,v8dark';
+    settings = getThemeSettings({ isDist: false });
+    expect(settings['theme:darkMode'].value).toBe(false);
+    expect(settings['theme:version'].value).toBe('v7');
+  });
+
+  it('ignores the value when isDist is undefined', () => {
+    process.env.KBN_OPTIMIZER_THEMES = 'v7light';
+    const settings = getThemeSettings({ isDist: undefined });
+    expect(settings['theme:darkMode'].value).toBe(false);
+    expect(settings['theme:version'].options).toEqual(['v7', 'v8']);
+  });
+
+  it('ignores the value when isDist is true', () => {
+    process.env.KBN_OPTIMIZER_THEMES = 'v7light';
+    const settings = getThemeSettings({ isDist: true });
+    expect(settings['theme:darkMode'].value).toBe(false);
+    expect(settings['theme:version'].options).toEqual(['v7', 'v8']);
+  });
+});

src/core/server/ui_settings/settings/theme.ts

@@ -6,17 +6,54 @@
  * Side Public License, v 1.
  */
 
-import { schema } from '@kbn/config-schema';
+import { schema, Type } from '@kbn/config-schema';
 import { i18n } from '@kbn/i18n';
 import { UiSettingsParams } from '../../../types';
 
-export const getThemeSettings = (): Record<string, UiSettingsParams> => {
+function parseThemeTags() {
+  if (!process.env.KBN_OPTIMIZER_THEMES) {
+    return ['v8light', 'v8dark'];
+  }
+
+  if (process.env.KBN_OPTIMIZER_THEMES === '*') {
+    return ['v8light', 'v8dark', 'v7light', 'v7dark'];
+  }
+
+  return process.env.KBN_OPTIMIZER_THEMES.split(',').map((t) => t.trim());
+}
+
+function getThemeInfo(options: GetThemeSettingsOptions) {
+  if (options?.isDist ?? true) {
+    return {
+      defaultDarkMode: false,
+      defaultVersion: 'v8',
+      availableVersions: ['v7', 'v8'],
+    };
+  }
+
+  const themeTags = parseThemeTags();
+  return {
+    defaultDarkMode: themeTags[0].endsWith('dark'),
+    defaultVersion: themeTags[0].slice(0, 2),
+    availableVersions: ['v7', 'v8'].filter((v) => themeTags.some((t) => t.startsWith(v))),
+  };
+}
+
+interface GetThemeSettingsOptions {
+  isDist?: boolean;
+}
+
+export const getThemeSettings = (
+  options: GetThemeSettingsOptions = {}
+): Record<string, UiSettingsParams> => {
+  const { availableVersions, defaultDarkMode, defaultVersion } = getThemeInfo(options);
+
   return {
     'theme:darkMode': {
       name: i18n.translate('core.ui_settings.params.darkModeTitle', {
         defaultMessage: 'Dark mode',
       }),
-      value: false,
+      value: defaultDarkMode,
       description: i18n.translate('core.ui_settings.params.darkModeText', {
         defaultMessage: `Enable a dark mode for the Kibana UI. A page refresh is required for the setting to be applied.`,
       }),
@@ -27,14 +64,14 @@ export const getThemeSettings = (): Record<string, UiSettingsParams> => {
       name: i18n.translate('core.ui_settings.params.themeVersionTitle', {
         defaultMessage: 'Theme version',
       }),
-      value: 'v8',
+      value: defaultVersion,
       type: 'select',
-      options: ['v7', 'v8'],
+      options: availableVersions,
       description: i18n.translate('core.ui_settings.params.themeVersionText', {
         defaultMessage: `Switch between the theme used for the current and next version of Kibana. A page refresh is required for the setting to be applied.`,
       }),
       requiresPageReload: true,
-      schema: schema.oneOf([schema.literal('v7'), schema.literal('v8')]),
+      schema: schema.oneOf(availableVersions.map((v) => schema.literal(v)) as [Type<string>]),
     },
   };
 };

src/core/server/ui_settings/ui_settings_service.ts

@@ -37,11 +37,13 @@ export class UiSettingsService
   implements CoreService<InternalUiSettingsServiceSetup, InternalUiSettingsServiceStart> {
   private readonly log: Logger;
   private readonly config$: Observable<UiSettingsConfigType>;
+  private readonly isDist: boolean;
   private readonly uiSettingsDefaults = new Map<string, UiSettingsParams>();
   private overrides: Record<string, any> = {};
 
   constructor(private readonly coreContext: CoreContext) {
     this.log = coreContext.logger.get('ui-settings-service');
+    this.isDist = coreContext.env.packageInfo.dist;
     this.config$ = coreContext.configService.atPath<UiSettingsConfigType>(uiConfigDefinition.path);
   }
 
@@ -50,7 +52,11 @@ export class UiSettingsService
 
     savedObjects.registerType(uiSettingsType);
     registerRoutes(http.createRouter(''));
-    this.register(getCoreSettings());
+    this.register(
+      getCoreSettings({
+        isDist: this.isDist,
+      })
+    );
 
     const config = await this.config$.pipe(first()).toPromise();
     this.overrides = config.overrides;