[Security Solution][Artifacts] Artifact creation for Endpoint Event F…

…iltering (elastic#96499)

* generate endpoint event filters artifacts
* Add ExperimentalFeature object to the initialization params of ManifestManager
* create event filters artifacts if feature flag is on
* change artifact migration to be less chatty in the logs (also: don't reference Fleet)

x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts

@@ -52,6 +52,7 @@ import {
 } from './find_exception_list_items';
 import { createEndpointList } from './create_endpoint_list';
 import { createEndpointTrustedAppsList } from './create_endpoint_trusted_apps_list';
+import { createEndpointEventFiltersList } from './create_endoint_event_filters_list';
 
 export class ExceptionListClient {
   private readonly user: string;
@@ -108,6 +109,18 @@ export class ExceptionListClient {
     });
   };
 
+  /**
+   * Create the Endpoint Event Filters Agnostic list if it does not yet exist (`null` is returned if it does exist)
+   */
+  public createEndpointEventFiltersList = async (): Promise<ExceptionListSchema | null> => {
+    const { savedObjectsClient, user } = this;
+    return createEndpointEventFiltersList({
+      savedObjectsClient,
+      user,
+      version: 1,
+    });
+  };
+
   /**
    * This is the same as "createListItem" except it applies specifically to the agnostic endpoint list and will
    * auto-call the "createEndpointList" for you so that you have the best chance of the agnostic endpoint

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -37,6 +37,10 @@ import { metadataTransformPrefix } from '../../common/endpoint/constants';
 import { AppClientFactory } from '../client';
 import { ConfigType } from '../config';
 import { LicenseService } from '../../common/license/license';
+import {
+  ExperimentalFeatures,
+  parseExperimentalConfigValue,
+} from '../../common/experimental_features';
 
 export interface MetadataService {
   queryStrategy(
@@ -107,6 +111,9 @@ export class EndpointAppContextService {
   private agentPolicyService: AgentPolicyServiceInterface | undefined;
   private savedObjectsStart: SavedObjectsServiceStart | undefined;
   private metadataService: MetadataService | undefined;
+  private config: ConfigType | undefined;
+
+  private experimentalFeatures: ExperimentalFeatures | undefined;
 
   public start(dependencies: EndpointAppContextServiceStartContract) {
     this.agentService = dependencies.agentService;
@@ -115,6 +122,9 @@ export class EndpointAppContextService {
     this.manifestManager = dependencies.manifestManager;
     this.savedObjectsStart = dependencies.savedObjectsStart;
     this.metadataService = createMetadataService(dependencies.packageService!);
+    this.config = dependencies.config;
+
+    this.experimentalFeatures = parseExperimentalConfigValue(this.config.enableExperimental);
 
     if (this.manifestManager && dependencies.registerIngestCallback) {
       dependencies.registerIngestCallback(
@@ -140,6 +150,10 @@ export class EndpointAppContextService {
 
   public stop() {}
 
+  public getExperimentalFeatures(): Readonly<ExperimentalFeatures> | undefined {
+    return this.experimentalFeatures;
+  }
+
   public getAgentService(): AgentService | undefined {
     return this.agentService;
   }

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts

@@ -22,6 +22,9 @@ export const ArtifactConstants = {
   SUPPORTED_OPERATING_SYSTEMS: ['macos', 'windows'],
   SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS: ['macos', 'windows', 'linux'],
   GLOBAL_TRUSTED_APPS_NAME: 'endpoint-trustlist',
+
+  SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS: ['macos', 'windows', 'linux'],
+  GLOBAL_EVENT_FILTERS_NAME: 'endpoint-eventfilterlist',
 };
 
 export const ManifestConstants = {

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -14,20 +14,21 @@ import { Entry, EntryNested } from '../../../../../lists/common/schemas/types';
 import { ExceptionListClient } from '../../../../../lists/server';
 import { ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../common/shared_imports';
 import {
+  internalArtifactCompleteSchema,
+  InternalArtifactCompleteSchema,
   InternalArtifactSchema,
   TranslatedEntry,
-  WrappedTranslatedExceptionList,
-  wrappedTranslatedExceptionList,
-  TranslatedEntryNestedEntry,
-  translatedEntryNestedEntry,
   translatedEntry as translatedEntryType,
+  translatedEntryMatchAnyMatcher,
   TranslatedEntryMatcher,
   translatedEntryMatchMatcher,
-  translatedEntryMatchAnyMatcher,
+  TranslatedEntryNestedEntry,
+  translatedEntryNestedEntry,
   TranslatedExceptionListItem,
-  internalArtifactCompleteSchema,
-  InternalArtifactCompleteSchema,
+  WrappedTranslatedExceptionList,
+  wrappedTranslatedExceptionList,
 } from '../../schemas';
+import { ENDPOINT_EVENT_FILTERS_LIST_ID } from '../../../../../lists/common/constants';
 
 export async function buildArtifact(
   exceptions: WrappedTranslatedExceptionList,
@@ -77,7 +78,10 @@ export async function getFilteredEndpointExceptionList(
   eClient: ExceptionListClient,
   schemaVersion: string,
   filter: string,
-  listId: typeof ENDPOINT_LIST_ID | typeof ENDPOINT_TRUSTED_APPS_LIST_ID
+  listId:
+    | typeof ENDPOINT_LIST_ID
+    | typeof ENDPOINT_TRUSTED_APPS_LIST_ID
+    | typeof ENDPOINT_EVENT_FILTERS_LIST_ID
 ): Promise<WrappedTranslatedExceptionList> {
   const exceptions: WrappedTranslatedExceptionList = { entries: [] };
   let page = 1;
@@ -142,6 +146,27 @@ export async function getEndpointTrustedAppsList(
   );
 }
 
+export async function getEndpointEventFiltersList(
+  eClient: ExceptionListClient,
+  schemaVersion: string,
+  os: string,
+  policyId?: string
+): Promise<WrappedTranslatedExceptionList> {
+  const osFilter = `exception-list-agnostic.attributes.os_types:\"${os}\"`;
+  const policyFilter = `(exception-list-agnostic.attributes.tags:\"policy:all\"${
+    policyId ? ` or exception-list-agnostic.attributes.tags:\"policy:${policyId}\"` : ''
+  })`;
+
+  await eClient.createEndpointEventFiltersList();
+
+  return getFilteredEndpointExceptionList(
+    eClient,
+    schemaVersion,
+    `${osFilter} and ${policyFilter}`,
+    ENDPOINT_EVENT_FILTERS_LIST_ID
+  );
+}
+
 /**
  * Translates Exception list items to Exceptions the endpoint can understand
  * @param exceptions

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts

@@ -66,8 +66,8 @@ describe('When migrating artifacts to fleet', () => {
 
   it('should do nothing if `fleetServerEnabled` flag is false', async () => {
     await migrateArtifactsToFleet(soClient, artifactClient, logger, false);
-    expect(logger.info).toHaveBeenCalledWith(
-      'Skipping Artifacts migration to fleet. [fleetServerEnabled] flag is off'
+    expect(logger.debug).toHaveBeenCalledWith(
+      'Skipping Artifacts migration. [fleetServerEnabled] flag is off'
     );
     expect(soClient.find).not.toHaveBeenCalled();
   });
@@ -94,7 +94,7 @@ describe('When migrating artifacts to fleet', () => {
     const error = new Error('test: delete failed');
     soClient.delete.mockRejectedValue(error);
     await expect(migrateArtifactsToFleet(soClient, artifactClient, logger, true)).rejects.toThrow(
-      'Artifact SO migration to fleet failed'
+      'Artifact SO migration failed'
     );
   });
 });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts

@@ -27,7 +27,7 @@ export const migrateArtifactsToFleet = async (
   isFleetServerEnabled: boolean
 ): Promise<void> => {
   if (!isFleetServerEnabled) {
-    logger.info('Skipping Artifacts migration to fleet. [fleetServerEnabled] flag is off');
+    logger.debug('Skipping Artifacts migration. [fleetServerEnabled] flag is off');
     return;
   }
 
@@ -49,14 +49,16 @@ export const migrateArtifactsToFleet = async (
       if (totalArtifactsMigrated === -1) {
         totalArtifactsMigrated = total;
         if (total > 0) {
-          logger.info(`Migrating artifacts from SavedObject to Fleet`);
+          logger.info(`Migrating artifacts from SavedObject`);
         }
       }
 
       // If nothing else to process, then exit out
       if (total === 0) {
         hasMore = false;
-        logger.info(`Total Artifacts migrated to Fleet: ${totalArtifactsMigrated}`);
+        if (totalArtifactsMigrated > 0) {
+          logger.info(`Total Artifacts migrated: ${totalArtifactsMigrated}`);
+        }
         return;
       }
 
@@ -78,7 +80,7 @@ export const migrateArtifactsToFleet = async (
       }
     }
   } catch (e) {
-    const error = new ArtifactMigrationError('Artifact SO migration to fleet failed', e);
+    const error = new ArtifactMigrationError('Artifact SO migration failed', e);
     logger.error(error);
     throw error;
   }

x-pack/plugins/security_solution/server/endpoint/mocks.ts

@@ -56,6 +56,7 @@ export const createMockEndpointAppContextService = (
   return ({
     start: jest.fn(),
     stop: jest.fn(),
+    getExperimentalFeatures: jest.fn(),
     getAgentService: jest.fn(),
     getAgentPolicyService: jest.fn(),
     getManifestManager: jest.fn().mockReturnValue(mockManifestManager ?? jest.fn()),

x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts

@@ -22,6 +22,7 @@ import {
 } from '../../../lib/artifacts/mocks';
 import { createEndpointArtifactClientMock, getManifestClientMock } from '../mocks';
 import { ManifestManager, ManifestManagerContext } from './manifest_manager';
+import { parseExperimentalConfigValue } from '../../../../../common/experimental_features';
 
 export const createExceptionListResponse = (data: ExceptionListItemSchema[], total?: number) => ({
   data,
@@ -85,6 +86,7 @@ export const buildManifestManagerContextMock = (
     ...fullOpts,
     artifactClient: createEndpointArtifactClientMock(),
     logger: loggingSystemMock.create().get() as jest.Mocked<Logger>,
+    experimentalFeatures: parseExperimentalConfigValue([]),
   };
 };
 

x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts

@@ -22,6 +22,7 @@ import {
   ArtifactConstants,
   buildArtifact,
   getArtifactId,
+  getEndpointEventFiltersList,
   getEndpointExceptionList,
   getEndpointTrustedAppsList,
   isCompressed,
@@ -34,6 +35,7 @@ import {
 } from '../../../schemas/artifacts';
 import { EndpointArtifactClientInterface } from '../artifact_client';
 import { ManifestClient } from '../manifest_client';
+import { ExperimentalFeatures } from '../../../../../common/experimental_features';
 
 interface ArtifactsBuildResult {
   defaultArtifacts: InternalArtifactCompleteSchema[];
@@ -81,6 +83,7 @@ export interface ManifestManagerContext {
   packagePolicyService: PackagePolicyServiceInterface;
   logger: Logger;
   cache: LRU<string, Buffer>;
+  experimentalFeatures: ExperimentalFeatures;
 }
 
 const getArtifactIds = (manifest: ManifestSchema) =>
@@ -99,18 +102,17 @@ export class ManifestManager {
   protected logger: Logger;
   protected cache: LRU<string, Buffer>;
   protected schemaVersion: ManifestSchemaVersion;
+  protected experimentalFeatures: ExperimentalFeatures;
 
-  constructor(
-    context: ManifestManagerContext,
-    private readonly isFleetServerEnabled: boolean = false
-  ) {
+  constructor(context: ManifestManagerContext) {
     this.artifactClient = context.artifactClient;
     this.exceptionListClient = context.exceptionListClient;
     this.packagePolicyService = context.packagePolicyService;
     this.savedObjectsClient = context.savedObjectsClient;
     this.logger = context.logger;
     this.cache = context.cache;
     this.schemaVersion = 'v1';
+    this.experimentalFeatures = context.experimentalFeatures;
   }
 
   /**
@@ -198,6 +200,41 @@ export class ManifestManager {
     return { defaultArtifacts, policySpecificArtifacts };
   }
 
+  /**
+   * Builds an array of endpoint event filters (one per supported OS) based on the current state of the
+   * Event Filters list
+   * @protected
+   */
+  protected async buildEventFiltersArtifacts(): Promise<ArtifactsBuildResult> {
+    const defaultArtifacts: InternalArtifactCompleteSchema[] = [];
+    const policySpecificArtifacts: Record<string, InternalArtifactCompleteSchema[]> = {};
+
+    for (const os of ArtifactConstants.SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS) {
+      defaultArtifacts.push(await this.buildEventFiltersForOs(os));
+    }
+
+    await iterateAllListItems(
+      (page) => this.listEndpointPolicyIds(page),
+      async (policyId) => {
+        for (const os of ArtifactConstants.SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS) {
+          policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || [];
+          policySpecificArtifacts[policyId].push(await this.buildEventFiltersForOs(os, policyId));
+        }
+      }
+    );
+
+    return { defaultArtifacts, policySpecificArtifacts };
+  }
+
+  protected async buildEventFiltersForOs(os: string, policyId?: string) {
+    return buildArtifact(
+      await getEndpointEventFiltersList(this.exceptionListClient, this.schemaVersion, os, policyId),
+      this.schemaVersion,
+      os,
+      ArtifactConstants.GLOBAL_EVENT_FILTERS_NAME
+    );
+  }
+
   /**
    * Writes new artifact SO.
    *
@@ -286,7 +323,7 @@ export class ManifestManager {
           semanticVersion: manifestSo.attributes.semanticVersion,
           soVersion: manifestSo.version,
         },
-        this.isFleetServerEnabled
+        this.experimentalFeatures.fleetServerEnabled
       );
 
       for (const entry of manifestSo.attributes.artifacts) {
@@ -327,12 +364,16 @@ export class ManifestManager {
   public async buildNewManifest(
     baselineManifest: Manifest = ManifestManager.createDefaultManifest(
       this.schemaVersion,
-      this.isFleetServerEnabled
+      this.experimentalFeatures.fleetServerEnabled
     )
   ): Promise<Manifest> {
     const results = await Promise.all([
       this.buildExceptionListArtifacts(),
       this.buildTrustedAppsArtifacts(),
+      // If Endpoint Event Filtering feature is ON, then add in the exceptions for them
+      ...(this.experimentalFeatures.eventFilteringEnabled
+        ? [this.buildEventFiltersArtifacts()]
+        : []),
     ]);
 
     const manifest = new Manifest(
@@ -341,7 +382,7 @@ export class ManifestManager {
         semanticVersion: baselineManifest.getSemanticVersion(),
         soVersion: baselineManifest.getSavedObjectVersion(),
       },
-      this.isFleetServerEnabled
+      this.experimentalFeatures.fleetServerEnabled
     );
 
     for (const result of results) {

x-pack/plugins/security_solution/server/plugin.ts

@@ -349,24 +349,22 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     if (this.lists && plugins.taskManager && plugins.fleet) {
       // Exceptions, Artifacts and Manifests start
       const taskManager = plugins.taskManager;
-      const fleetServerEnabled = parseExperimentalConfigValue(this.config.enableExperimental)
-        .fleetServerEnabled;
+      const experimentalFeatures = parseExperimentalConfigValue(this.config.enableExperimental);
+      const fleetServerEnabled = experimentalFeatures.fleetServerEnabled;
       const exceptionListClient = this.lists.getExceptionListClient(savedObjectsClient, 'kibana');
       const artifactClient = new EndpointArtifactClient(
         plugins.fleet.createArtifactsClient('endpoint')
       );
 
-      manifestManager = new ManifestManager(
-        {
-          savedObjectsClient,
-          artifactClient,
-          exceptionListClient,
-          packagePolicyService: plugins.fleet.packagePolicyService,
-          logger,
-          cache: this.artifactsCache,
-        },
-        fleetServerEnabled
-      );
+      manifestManager = new ManifestManager({
+        savedObjectsClient,
+        artifactClient,
+        exceptionListClient,
+        packagePolicyService: plugins.fleet.packagePolicyService,
+        logger,
+        cache: this.artifactsCache,
+        experimentalFeatures,
+      });
 
       // Migrate artifacts to fleet and then start the minifest task after that is done
       plugins.fleet.fleetSetupCompleted().then(() => {
@@ -376,7 +374,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
           logger,
           fleetServerEnabled
         ).finally(() => {
-          logger.info('Fleet setup complete - Starting ManifestTask');
+          logger.info('Dependent plugin setup complete - Starting ManifestTask');
 
           if (this.manifestTask) {
             this.manifestTask.start({