Merge branch 'master' into fix-migration-bug

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -55,21 +55,20 @@ const uploadPipeline = (pipelineContent) => {
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
 
     if (
-      await doAnyChangesMatch([
+      (await doAnyChangesMatch([
         /^x-pack\/plugins\/security_solution/,
         /^x-pack\/test\/security_solution_cypress/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
-      ]) || process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+      ])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
-    // Disabled for now, these are failing/disabled in Jenkins currently as well
     // if (
-    //   await doAnyChangesMatch([
-    //     /^x-pack\/plugins\/apm/,
-    //   ]) || process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    //   (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
+    //   process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     // ) {
     //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     // }

packages/kbn-securitysolution-io-ts-list-types/src/typescript_types/index.ts

@@ -43,6 +43,7 @@ export interface UseExceptionListsProps {
   initialPagination?: Pagination;
   showTrustedApps: boolean;
   showEventFilters: boolean;
+  showHostIsolationExceptions: boolean;
 }
 
 export interface UseExceptionListProps {

packages/kbn-securitysolution-list-hooks/src/use_exception_lists/index.ts

@@ -41,6 +41,7 @@ const DEFAULT_PAGINATION = {
  * @param notifications kibana service for displaying toasters
  * @param showTrustedApps boolean - include/exclude trusted app lists
  * @param showEventFilters boolean - include/exclude event filters lists
+ * @param showHostIsolationExceptions boolean - include/exclude host isolation exceptions lists
  * @param initialPagination
  *
  */
@@ -53,6 +54,7 @@ export const useExceptionLists = ({
   notifications,
   showTrustedApps = false,
   showEventFilters = false,
+  showHostIsolationExceptions = false,
 }: UseExceptionListsProps): ReturnExceptionLists => {
   const [exceptionLists, setExceptionLists] = useState<ExceptionListSchema[]>([]);
   const [pagination, setPagination] = useState<Pagination>(initialPagination);
@@ -62,8 +64,14 @@ export const useExceptionLists = ({
   const namespaceTypesAsString = useMemo(() => namespaceTypes.join(','), [namespaceTypes]);
   const filters = useMemo(
     (): string =>
-      getFilters({ filters: filterOptions, namespaceTypes, showTrustedApps, showEventFilters }),
-    [namespaceTypes, filterOptions, showTrustedApps, showEventFilters]
+      getFilters({
+        filters: filterOptions,
+        namespaceTypes,
+        showTrustedApps,
+        showEventFilters,
+        showHostIsolationExceptions,
+      }),
+    [namespaceTypes, filterOptions, showTrustedApps, showEventFilters, showHostIsolationExceptions]
   );
 
   const fetchData = useCallback(async (): Promise<void> => {

packages/kbn-securitysolution-list-utils/src/get_filters/index.ts

@@ -11,25 +11,32 @@ import { getGeneralFilters } from '../get_general_filters';
 import { getSavedObjectTypes } from '../get_saved_object_types';
 import { getTrustedAppsFilter } from '../get_trusted_apps_filter';
 import { getEventFiltersFilter } from '../get_event_filters_filter';
+import { getHostIsolationExceptionsFilter } from '../get_host_isolation_exceptions_filter';
 
 export interface GetFiltersParams {
   filters: ExceptionListFilter;
   namespaceTypes: NamespaceType[];
   showTrustedApps: boolean;
   showEventFilters: boolean;
+  showHostIsolationExceptions: boolean;
 }
 
 export const getFilters = ({
   filters,
   namespaceTypes,
   showTrustedApps,
   showEventFilters,
+  showHostIsolationExceptions,
 }: GetFiltersParams): string => {
   const namespaces = getSavedObjectTypes({ namespaceType: namespaceTypes });
   const generalFilters = getGeneralFilters(filters, namespaces);
   const trustedAppsFilter = getTrustedAppsFilter(showTrustedApps, namespaces);
   const eventFiltersFilter = getEventFiltersFilter(showEventFilters, namespaces);
-  return [generalFilters, trustedAppsFilter, eventFiltersFilter]
+  const hostIsolationExceptionsFilter = getHostIsolationExceptionsFilter(
+    showHostIsolationExceptions,
+    namespaces
+  );
+  return [generalFilters, trustedAppsFilter, eventFiltersFilter, hostIsolationExceptionsFilter]
     .filter((filter) => filter.trim() !== '')
     .join(' AND ');
 };

packages/kbn-securitysolution-list-utils/src/get_host_isolation_exceptions_filter/index.test.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getHostIsolationExceptionsFilter } from '.';
+
+describe('getHostIsolationExceptionsFilter', () => {
+  test('it returns filter to search for "exception-list" namespace host isolation exceptions', () => {
+    const filter = getHostIsolationExceptionsFilter(true, ['exception-list']);
+
+    expect(filter).toEqual(
+      '(exception-list.attributes.list_id: endpoint_host_isolation_exceptions*)'
+    );
+  });
+
+  test('it returns filter to search for "exception-list" and "agnostic" namespace host isolation exceptions', () => {
+    const filter = getHostIsolationExceptionsFilter(true, [
+      'exception-list',
+      'exception-list-agnostic',
+    ]);
+
+    expect(filter).toEqual(
+      '(exception-list.attributes.list_id: endpoint_host_isolation_exceptions* OR exception-list-agnostic.attributes.list_id: endpoint_host_isolation_exceptions*)'
+    );
+  });
+
+  test('it returns filter to exclude "exception-list" namespace host isolation exceptions', () => {
+    const filter = getHostIsolationExceptionsFilter(false, ['exception-list']);
+
+    expect(filter).toEqual(
+      '(not exception-list.attributes.list_id: endpoint_host_isolation_exceptions*)'
+    );
+  });
+
+  test('it returns filter to exclude "exception-list" and "agnostic" namespace host isolation exceptions', () => {
+    const filter = getHostIsolationExceptionsFilter(false, [
+      'exception-list',
+      'exception-list-agnostic',
+    ]);
+
+    expect(filter).toEqual(
+      '(not exception-list.attributes.list_id: endpoint_host_isolation_exceptions* AND not exception-list-agnostic.attributes.list_id: endpoint_host_isolation_exceptions*)'
+    );
+  });
+});

packages/kbn-securitysolution-list-utils/src/get_host_isolation_exceptions_filter/index.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID } from '@kbn/securitysolution-list-constants';
+import { SavedObjectType } from '../types';
+
+export const getHostIsolationExceptionsFilter = (
+  showFilter: boolean,
+  namespaceTypes: SavedObjectType[]
+): string => {
+  if (showFilter) {
+    const filters = namespaceTypes.map((namespace) => {
+      return `${namespace}.attributes.list_id: ${ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID}*`;
+    });
+    return `(${filters.join(' OR ')})`;
+  } else {
+    const filters = namespaceTypes.map((namespace) => {
+      return `not ${namespace}.attributes.list_id: ${ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID}*`;
+    });
+    return `(${filters.join(' AND ')})`;
+  }
+};

packages/kbn-test/src/functional_tests/lib/babel_register_for_test_plugins.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-const Fs = require('fs');
 const Path = require('path');
 
 const { REPO_ROOT } = require('@kbn/dev-utils');
@@ -23,7 +22,7 @@ require('@babel/register')({
     // TODO: should should probably remove this link back to the source
     Path.resolve(REPO_ROOT, 'x-pack/plugins/task_manager/server/config.ts'),
     Path.resolve(REPO_ROOT, 'src/core/utils/default_app_categories.ts'),
-  ].map((path) => Fs.realpathSync(path)),
+  ],
   babelrc: false,
   presets: [require.resolve('@kbn/babel-preset/node_preset')],
   extensions: ['.js', '.ts', '.tsx'],

src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts

@@ -26,6 +26,30 @@ async function removeLogFile() {
   // ignore errors if it doesn't exist
   await fs.unlink(logFilePath).catch(() => void 0);
 }
+function sortByTypeAndId(a: { type: string; id: string }, b: { type: string; id: string }) {
+  return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
+}
+
+async function fetchDocuments(esClient: ElasticsearchClient, index: string) {
+  const { body } = await esClient.search<any>({
+    index,
+    body: {
+      query: {
+        match_all: {},
+      },
+      _source: ['type', 'id'],
+    },
+  });
+
+  return body.hits.hits
+    .map((h) => ({
+      ...h._source,
+      id: h._id,
+    }))
+    .sort(sortByTypeAndId);
+}
+
+const assertMigratedDocuments = (arr: any[], target: any[]) => target.every((v) => arr.includes(v));
 
 describe('migration v2', () => {
   let esServer: kbnTestServer.TestElasticsearchUtils;
@@ -72,16 +96,11 @@ describe('migration v2', () => {
     await new Promise((resolve) => setTimeout(resolve, 5000));
 
     const esClient: ElasticsearchClient = esServer.es.getClient();
-    const migratedIndexResponse = await esClient.count({
-      index: targetIndex,
-    });
-    const oldIndexResponse = await esClient.count({
-      index: '.kibana_7.14.0_001',
-    });
-
-    // Use a >= comparison since once Kibana has started it might create new
-    // documents like telemetry tasks
-    expect(migratedIndexResponse.body.count).toBeGreaterThanOrEqual(oldIndexResponse.body.count);
+
+    // assert that the docs from the original index have been migrated rather than comparing a doc count after startup
+    const originalDocs = await fetchDocuments(esClient, '.kibana_7.14.0_001');
+    const migratedDocs = await fetchDocuments(esClient, targetIndex);
+    expect(assertMigratedDocuments(migratedDocs, originalDocs));
   });
 
   it('fails with a descriptive message when a single document exceeds maxBatchSizeBytes', async () => {

src/core/server/saved_objects/serialization/serializer.test.ts

@@ -491,6 +491,52 @@ describe('#rawToSavedObject', () => {
       expect(actual).toHaveProperty('namespaces', ['baz']);
     });
   });
+
+  describe('throws if provided invalid type', () => {
+    expect(() =>
+      singleNamespaceSerializer.rawToSavedObject({
+        _id: 'foo:bar',
+        _source: {
+          // @ts-expect-error expects a string
+          // eslint-disable-next-line
+          type: new String('foo'),
+        },
+      })
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Expected saved object type to be a string but given [String] with [foo] value."`
+    );
+
+    expect(() =>
+      singleNamespaceSerializer.rawToSavedObject({
+        _id: 'foo:bar',
+        _source: {
+          // @ts-expect-error expects astring
+          type: {
+            toString() {
+              return 'foo';
+            },
+          },
+        },
+      })
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Expected saved object type to be a string but given [Object] with [foo] value."`
+    );
+  });
+
+  describe('throws if provided invalid id', () => {
+    expect(() =>
+      singleNamespaceSerializer.rawToSavedObject({
+        // @ts-expect-error expects a string
+        // eslint-disable-next-line
+        _id: new String('foo:bar'),
+        _source: {
+          type: 'foo',
+        },
+      })
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Expected document id to be a string but given [String] with [foo:bar] value."`
+    );
+  });
 });
 
 describe('#savedObjectToRaw', () => {

src/core/server/saved_objects/serialization/serializer.ts

@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
+import typeDetect from 'type-detect';
 import { LEGACY_URL_ALIAS_TYPE } from '../object_types';
 import { decodeVersion, encodeVersion } from '../version';
 import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry';
@@ -236,6 +236,8 @@ function checkIdMatchesPrefix(id: string, prefix: string) {
 
 function assertNonEmptyString(value: string, name: string) {
   if (!value || typeof value !== 'string') {
-    throw new TypeError(`Expected "${value}" to be a ${name}`);
+    throw new TypeError(
+      `Expected ${name} to be a string but given [${typeDetect(value)}] with [${value}] value.`
+    );
   }
 }

src/core/server/ui_settings/saved_objects/migrations.test.ts

@@ -162,4 +162,28 @@ describe('ui_settings 8.0.0 migrations', () => {
       migrationVersion: {},
     });
   });
+  test('removes telemetry:optIn and xPackMonitoring:allowReport from ui_settings', () => {
+    const doc = {
+      type: 'config',
+      id: '8.0.0',
+      attributes: {
+        buildNum: 9007199254740991,
+        'telemetry:optIn': false,
+        'xPackMonitoring:allowReport': false,
+      },
+      references: [],
+      updated_at: '2020-06-09T20:18:20.349Z',
+      migrationVersion: {},
+    };
+    expect(migration(doc)).toEqual({
+      type: 'config',
+      id: '8.0.0',
+      attributes: {
+        buildNum: 9007199254740991,
+      },
+      references: [],
+      updated_at: '2020-06-09T20:18:20.349Z',
+      migrationVersion: {},
+    });
+  });
 });

src/core/server/ui_settings/saved_objects/migrations.ts

@@ -78,13 +78,16 @@ export const migrations = {
   '8.0.0': (doc: SavedObjectUnsanitizedDoc<any>): SavedObjectSanitizedDoc<any> => ({
     ...doc,
     ...(doc.attributes && {
-      // owner: Team:Geo
       attributes: Object.keys(doc.attributes).reduce(
         (acc, key) =>
           [
+            // owner: Team:Geo
             'visualization:regionmap:showWarnings',
             'visualization:tileMap:WMSdefaults',
             'visualization:tileMap:maxPrecision',
+            // owner: Team:Core
+            'telemetry:optIn',
+            'xPackMonitoring:allowReport',
           ].includes(key)
             ? {
                 ...acc,

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -177,7 +177,6 @@ kibana_vars=(
     telemetry.allowChangingOptInStatus
     telemetry.enabled
     telemetry.optIn
-    telemetry.optInStatusUrl
     telemetry.sendUsageTo
     telemetry.sendUsageFrom
     tilemap.options.attribution