From 5a4856d45ec59167a7597f825af5f5197be7e017 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 2 Jul 2020 01:00:27 -0400 Subject: [PATCH] [SIEM][Security Solution][Endpoint] Endpoint Artifact Manifest Management + Artifact Download and Distribution (#67707) * stub out task for the exceptions list packager * Hits list code and pages * refactor * Begin adding saved object and type definitions * Transforms to endpoint exceptions * Get internal SO client * update messaging * cleanup * Integrating with task manager * Integrated with task manager properly * Begin adding schemas * Add multiple OS and schema version support * filter by OS * Fixing sort * Move to security_solutions * siem -> securitySolution * Progress on downloads, cleanup * Add config, update artifact creation, add TODOs * Fixing buffer serialization problem * Adding cleanup to task * Handle HEAD req * proper header * More robust task management * single -> agnostic * Fix OS filtering * Scaffolding digital signatures / tests * Adds rotue for creating endpoint user * Cleanup * persisting user * Adding route to fetch created user * Addings tests for translating exceptions * Adding test for download API * Download tweaks + artifact generation fixes * reorganize * fix imports * Fixing test * Changes id of SO * integration tests setup * Add first integration tests * Cache layer * more schema validation * Set up for manifest update * minor change * remove setup code * add manifest schema * refactoring * manifest rewrite (partial) * finish scaffolding new manifest logic * syntax errors * more refactoring * Move to endpoint directory * minor cleanup * clean up old artifacts * Use diff appropriately * Fix download * schedule task on interval * Split up into client/manager * more mocks * config interval * Fixing download tests and adding cache tests * lint * mo money, mo progress * Converting to io-ts * More tests and mocks * even more tests and mocks * Merging both refactors * Adding more tests for the convertion layer * fix conflicts * Adding lzma types * Bug fixes * lint * resolve some type errors * Adding back in cache * Fixing download test * Changing cache to be sized * Fix manifest manager initialization * Hook up datasource service * Fix download tests * Incremental progress * Adds integration with ingest manager for auth * Update test fixture * Add manifest dispatch * Refactoring to use the same SO Client from ingest * bug fixes * build renovate config * Fix endpoint_app_context_services tests * Only index the fields that are necessary for searching * Integ test progress * mock and test city * Add task tests * Tests for artifact_client and manifest_client * Add manifest_manager tests * minor refactor * Finish manifest_manager tests * Type errors * Update integ test * Type errors, final cleanup * Fix integration test and add test for invalid api key * minor fixup * Remove compression * Update task interval * Removing .text suffix from translated list * Fixes hashes for unit tests * clean up yarn.lock * Remove lzma-native from package.json * missed updating one of the tests Co-authored-by: Alex Kahan --- x-pack/plugins/lists/server/mocks.ts | 2 +- .../common/endpoint/generate_data.ts | 7 + .../common/endpoint/schema/common.ts | 22 ++ .../common/endpoint/schema/manifest.ts | 27 ++ .../common/endpoint/types.ts | 4 + x-pack/plugins/security_solution/kibana.json | 1 + .../policy/store/policy_details/index.test.ts | 7 + .../endpoint_app_context_services.test.ts | 14 +- .../endpoint/endpoint_app_context_services.ts | 33 +- .../server/endpoint/ingest_integration.ts | 82 +++-- .../endpoint/lib/artifacts/cache.test.ts | 43 +++ .../server/endpoint/lib/artifacts/cache.ts | 37 +++ .../server/endpoint/lib/artifacts/common.ts | 17 + .../server/endpoint/lib/artifacts/index.ts | 12 + .../endpoint/lib/artifacts/lists.test.ts | 196 ++++++++++++ .../server/endpoint/lib/artifacts/lists.ts | 157 +++++++++ .../endpoint/lib/artifacts/manifest.test.ts | 150 +++++++++ .../server/endpoint/lib/artifacts/manifest.ts | 130 ++++++++ .../lib/artifacts/manifest_entry.test.ts | 64 ++++ .../endpoint/lib/artifacts/manifest_entry.ts | 48 +++ .../lib/artifacts/saved_object_mappings.ts | 67 ++++ .../endpoint/lib/artifacts/task.mock.ts | 11 + .../endpoint/lib/artifacts/task.test.ts | 73 +++++ .../server/endpoint/lib/artifacts/task.ts | 107 +++++++ .../server/endpoint/mocks.ts | 48 ++- .../artifacts/download_exception_list.test.ts | 301 ++++++++++++++++++ .../artifacts/download_exception_list.ts | 107 +++++++ .../server/endpoint/routes/artifacts/index.ts | 7 + .../endpoint/schemas/artifacts/common.ts | 19 ++ .../endpoint/schemas/artifacts/index.ts | 11 + .../endpoint/schemas/artifacts/lists.mock.ts | 32 ++ .../endpoint/schemas/artifacts/lists.ts | 65 ++++ .../request/download_artifact_schema.ts | 19 ++ .../schemas/artifacts/request/index.ts | 7 + .../response/download_artifact_schema.ts | 25 ++ .../schemas/artifacts/response/index.ts | 7 + .../schemas/artifacts/saved_objects.mock.ts | 40 +++ .../schemas/artifacts/saved_objects.ts | 31 ++ .../server/endpoint/schemas/index.ts | 7 + .../artifacts/artifact_client.mock.ts | 18 ++ .../artifacts/artifact_client.test.ts | 49 +++ .../services/artifacts/artifact_client.ts | 42 +++ .../endpoint/services/artifacts/index.ts | 8 + .../artifacts/manifest_client.mock.ts | 18 ++ .../artifacts/manifest_client.test.ts | 69 ++++ .../services/artifacts/manifest_client.ts | 85 +++++ .../artifacts/manifest_manager/index.ts | 7 + .../manifest_manager/manifest_manager.mock.ts | 111 +++++++ .../manifest_manager/manifest_manager.test.ts | 82 +++++ .../manifest_manager/manifest_manager.ts | 270 ++++++++++++++++ .../server/endpoint/services/index.ts | 7 + .../server/endpoint/types.ts | 2 +- .../security_solution/server/plugin.ts | 52 ++- .../security_solution/server/saved_objects.ts | 14 +- .../apis/endpoint/artifacts/index.ts | 93 ++++++ .../endpoint/artifacts/api_feature/data.json | 179 +++++++++++ 56 files changed, 3101 insertions(+), 42 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/endpoint/schema/common.ts create mode 100644 x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/index.ts create mode 100644 x-pack/test/api_integration/apis/endpoint/artifacts/index.ts create mode 100644 x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json diff --git a/x-pack/plugins/lists/server/mocks.ts b/x-pack/plugins/lists/server/mocks.ts index aad4a25a900a1..ba565216fe431 100644 --- a/x-pack/plugins/lists/server/mocks.ts +++ b/x-pack/plugins/lists/server/mocks.ts @@ -18,6 +18,6 @@ const createSetupMock = (): jest.Mocked => { export const listMock = { createSetup: createSetupMock, - getExceptionList: getExceptionListClientMock, + getExceptionListClient: getExceptionListClientMock, getListClient: getListClientMock, }; diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index a6fe12a9b029f..6720f3523d5c7 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1034,6 +1034,13 @@ export class EndpointDocGenerator { enabled: true, streams: [], config: { + artifact_manifest: { + value: { + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: {}, + }, + }, policy: { value: policyFactory(), }, diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts new file mode 100644 index 0000000000000..7f8c938d54feb --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const identifier = t.string; + +export const manifestVersion = t.string; + +export const manifestSchemaVersion = t.keyof({ + '1.0.0': null, +}); +export type ManifestSchemaVersion = t.TypeOf; + +export const sha256 = t.string; + +export const size = t.number; + +export const url = t.string; diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts new file mode 100644 index 0000000000000..470e9b13ef78a --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common'; + +export const manifestEntrySchema = t.exact( + t.type({ + url, + sha256, + size, + }) +); + +export const manifestSchema = t.exact( + t.type({ + manifest_version: manifestVersion, + schema_version: manifestSchemaVersion, + artifacts: t.record(identifier, manifestEntrySchema), + }) +); + +export type ManifestEntrySchema = t.TypeOf; +export type ManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 16923829c1605..f2b8acb627cc4 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -5,6 +5,7 @@ */ import { PackageConfig, NewPackageConfig } from '../../../ingest_manager/common'; +import { ManifestSchema } from './schema/manifest'; /** * Object that allows you to maintain stateful information in the location object across navigation events @@ -683,6 +684,9 @@ export type NewPolicyData = NewPackageConfig & { enabled: boolean; streams: []; config: { + artifact_manifest: { + value: ManifestSchema; + }; policy: { value: PolicyConfig; }; diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index 8ce8820a8e57d..f6f2d5171312c 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -12,6 +12,7 @@ "features", "home", "ingestManager", + "taskManager", "inspector", "licensing", "maps", diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 469b71854dfcc..0bd623b27f4fb 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -41,6 +41,13 @@ describe('policy details: ', () => { enabled: true, streams: [], config: { + artifact_manifest: { + value: { + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: {}, + }, + }, policy: { value: policyConfigFactory(), }, diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 8cf2ada9907d3..2daf259941cbf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -3,11 +3,23 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { httpServerMock } from '../../../../../src/core/server/mocks'; import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { - it('should throw error if start is not called', async () => { + it('should throw error on getAgentService if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => endpointAppContextService.getAgentService()).toThrow(Error); }); + it('should return undefined on getManifestManager if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect(endpointAppContextService.getManifestManager()).toEqual(undefined); + }); + it('should throw error on getScopedSavedObjectsClient if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect(() => + endpointAppContextService.getScopedSavedObjectsClient(httpServerMock.createKibanaRequest()) + ).toThrow(Error); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 1fce5d355f5a7..97a82049634c4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -3,14 +3,22 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { + SavedObjectsServiceStart, + KibanaRequest, + SavedObjectsClientContract, +} from 'src/core/server'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; -import { handlePackageConfigCreate } from './ingest_integration'; +import { getPackageConfigCreateCallback } from './ingest_integration'; +import { ManifestManager } from './services/artifacts'; export type EndpointAppContextServiceStartContract = Pick< IngestManagerStartContract, 'agentService' > & { + manifestManager?: ManifestManager | undefined; registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; + savedObjectsStart: SavedObjectsServiceStart; }; /** @@ -19,10 +27,20 @@ export type EndpointAppContextServiceStartContract = Pick< */ export class EndpointAppContextService { private agentService: AgentService | undefined; + private manifestManager: ManifestManager | undefined; + private savedObjectsStart: SavedObjectsServiceStart | undefined; public start(dependencies: EndpointAppContextServiceStartContract) { this.agentService = dependencies.agentService; - dependencies.registerIngestCallback('packageConfigCreate', handlePackageConfigCreate); + this.manifestManager = dependencies.manifestManager; + this.savedObjectsStart = dependencies.savedObjectsStart; + + if (this.manifestManager !== undefined) { + dependencies.registerIngestCallback( + 'packageConfigCreate', + getPackageConfigCreateCallback(this.manifestManager) + ); + } } public stop() {} @@ -33,4 +51,15 @@ export class EndpointAppContextService { } return this.agentService; } + + public getManifestManager(): ManifestManager | undefined { + return this.manifestManager; + } + + public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClientContract { + if (!this.savedObjectsStart) { + throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); + } + return this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] }); + } } diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index bb0b950dad019..67a331f4ba677 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -4,46 +4,62 @@ * you may not use this file except in compliance with the Elastic License. */ +import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; -import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; +import { ManifestManager } from './services/artifacts'; /** - * Callback to handle creation of package configs in Ingest Manager - * @param newPackageConfig + * Callback to handle creation of PackageConfigs in Ingest Manager */ -export const handlePackageConfigCreate = async ( - newPackageConfig: NewPackageConfig -): Promise => { - // We only care about Endpoint package configs - if (newPackageConfig.package?.name !== 'endpoint') { - return newPackageConfig; - } +export const getPackageConfigCreateCallback = ( + manifestManager: ManifestManager +): ((newPackageConfig: NewPackageConfig) => Promise) => { + const handlePackageConfigCreate = async ( + newPackageConfig: NewPackageConfig + ): Promise => { + // We only care about Endpoint package configs + if (newPackageConfig.package?.name !== 'endpoint') { + return newPackageConfig; + } - // We cast the type here so that any changes to the Endpoint specific data - // follow the types/schema expected - let updatedPackageConfig = newPackageConfig as NewPolicyData; + // We cast the type here so that any changes to the Endpoint specific data + // follow the types/schema expected + let updatedPackageConfig = newPackageConfig as NewPolicyData; - // Until we get the Default Policy Configuration in the Endpoint package, - // we will add it here manually at creation time. - // @ts-ignore - if (newPackageConfig.inputs.length === 0) { - updatedPackageConfig = { - ...newPackageConfig, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - policy: { - value: policyConfigFactory(), + const wrappedManifest = await manifestManager.refresh({ initialize: true }); + if (wrappedManifest !== null) { + // Until we get the Default Policy Configuration in the Endpoint package, + // we will add it here manually at creation time. + // @ts-ignore + if (newPackageConfig.inputs.length === 0) { + updatedPackageConfig = { + ...newPackageConfig, + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + artifact_manifest: { + value: wrappedManifest.manifest.toEndpointFormat(), + }, + policy: { + value: policyConfigFactory(), + }, + }, }, - }, - }, - ], - }; - } + ], + }; + } + } + + try { + return updatedPackageConfig; + } finally { + await manifestManager.commit(wrappedManifest); + } + }; - return updatedPackageConfig; + return handlePackageConfigCreate; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts new file mode 100644 index 0000000000000..5a0fb91345552 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ExceptionsCache } from './cache'; + +describe('ExceptionsCache tests', () => { + let cache: ExceptionsCache; + + beforeEach(() => { + jest.clearAllMocks(); + cache = new ExceptionsCache(3); + }); + + test('it should cache', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('test'); + expect(cacheResp).toEqual('body'); + }); + + test('it should handle cache miss', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('not test'); + expect(cacheResp).toEqual(undefined); + }); + + test('it should handle cache eviction', async () => { + cache.set('1', 'a'); + cache.set('2', 'b'); + cache.set('3', 'c'); + const cacheResp = cache.get('1'); + expect(cacheResp).toEqual('a'); + + cache.set('4', 'd'); + const secondResp = cache.get('1'); + expect(secondResp).toEqual(undefined); + expect(cache.get('2')).toEqual('b'); + expect(cache.get('3')).toEqual('c'); + expect(cache.get('4')).toEqual('d'); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts new file mode 100644 index 0000000000000..b7a4c2feb6bf8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const DEFAULT_MAX_SIZE = 10; + +/** + * FIFO cache implementation for artifact downloads. + */ +export class ExceptionsCache { + private cache: Map; + private queue: string[]; + private maxSize: number; + + constructor(maxSize: number) { + this.cache = new Map(); + this.queue = []; + this.maxSize = maxSize || DEFAULT_MAX_SIZE; + } + + set(id: string, body: string) { + if (this.queue.length + 1 > this.maxSize) { + const entry = this.queue.shift(); + if (entry !== undefined) { + this.cache.delete(entry); + } + } + this.queue.push(id); + this.cache.set(id, body); + } + + get(id: string): string | undefined { + return this.cache.get(id); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts new file mode 100644 index 0000000000000..4c3153ca0ef11 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const ArtifactConstants = { + GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', + SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact', + SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], + SCHEMA_VERSION: '1.0.0', +}; + +export const ManifestConstants = { + SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest', + SCHEMA_VERSION: '1.0.0', +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts new file mode 100644 index 0000000000000..ee7d44459aa38 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './cache'; +export * from './common'; +export * from './lists'; +export * from './manifest'; +export * from './manifest_entry'; +export * from './task'; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts new file mode 100644 index 0000000000000..738890fb4038f --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ExceptionListClient } from '../../../../../lists/server'; +import { listMock } from '../../../../../lists/server/mocks'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { EntriesArray, EntryList } from '../../../../../lists/common/schemas/types/entries'; +import { getFullEndpointExceptionList } from './lists'; + +describe('buildEventTypeSignal', () => { + let mockExceptionClient: ExceptionListClient; + + beforeEach(() => { + jest.clearAllMocks(); + mockExceptionClient = listMock.getExceptionListClient(); + }); + + test('it should convert the exception lists response to the proper endpoint format', async () => { + const expectedEndpointExceptions = { + exceptions_list: [ + { + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert simple fields', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { field: 'host.hostname', operator: 'included', type: 'match', value: 'estc' }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_cased', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_cased', + value: 'estc', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert fields case sensitive', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { + field: 'host.hostname.text', + operator: 'included', + type: 'match_any', + value: ['estc', 'kibana'], + }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_caseless_any', + value: ['estc', 'kibana'], + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should ignore unsupported entries', async () => { + // Lists and exists are not supported by the Endpoint + const testEntries: EntriesArray = [ + { field: 'server.domain', operator: 'included', type: 'match', value: 'DOMAIN' }, + { + field: 'server.domain', + operator: 'included', + type: 'list', + list: { + id: 'lists_not_supported', + type: 'keyword', + }, + } as EntryList, + { field: 'server.ip', operator: 'included', type: 'exists' }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_cased', + value: 'DOMAIN', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert the exception lists response to the proper endpoint format while paging', async () => { + // The first call returns one exception + const first = getFoundExceptionListItemSchemaMock(); + + // The second call returns two exceptions + const second = getFoundExceptionListItemSchemaMock(); + second.data.push(getExceptionListItemSchemaMock()); + + // The third call returns no exceptions, paging stops + const third = getFoundExceptionListItemSchemaMock(); + third.data = []; + mockExceptionClient.findExceptionListItem = jest + .fn() + .mockReturnValueOnce(first) + .mockReturnValueOnce(second) + .mockReturnValueOnce(third); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp.exceptions_list.length).toEqual(6); + }); + + test('it should handle no exceptions', async () => { + const exceptionsResponse = getFoundExceptionListItemSchemaMock(); + exceptionsResponse.data = []; + exceptionsResponse.total = 0; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp.exceptions_list.length).toEqual(0); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts new file mode 100644 index 0000000000000..7fd057afdbd55 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; +import { validate } from '../../../../common/validate'; + +import { + Entry, + EntryNested, + EntryMatch, + EntryMatchAny, +} from '../../../../../lists/common/schemas/types/entries'; +import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; +import { ExceptionListClient } from '../../../../../lists/server'; +import { + InternalArtifactSchema, + TranslatedEntry, + TranslatedEntryMatch, + TranslatedEntryMatchAny, + TranslatedEntryNested, + WrappedTranslatedExceptionList, + wrappedExceptionList, +} from '../../schemas'; +import { ArtifactConstants } from './common'; + +export async function buildArtifact( + exceptions: WrappedTranslatedExceptionList, + os: string, + schemaVersion: string +): Promise { + const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions)); + const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex'); + + return { + identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, + sha256, + encoding: 'application/json', + created: Date.now(), + body: exceptionsBuffer.toString('base64'), + size: exceptionsBuffer.byteLength, + }; +} + +export async function getFullEndpointExceptionList( + eClient: ExceptionListClient, + os: string, + schemaVersion: string +): Promise { + const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] }; + let numResponses = 0; + let page = 1; + + do { + const response = await eClient.findExceptionListItem({ + listId: 'endpoint_list', + namespaceType: 'agnostic', + filter: `exception-list-agnostic.attributes._tags:\"os:${os}\"`, + perPage: 100, + page, + sortField: 'created_at', + sortOrder: 'desc', + }); + + if (response?.data !== undefined) { + numResponses = response.data.length; + + exceptions.exceptions_list = exceptions.exceptions_list.concat( + translateToEndpointExceptions(response, schemaVersion) + ); + + page++; + } else { + break; + } + } while (numResponses > 0); + + const [validated, errors] = validate(exceptions, wrappedExceptionList); + if (errors != null) { + throw new Error(errors); + } + return validated as WrappedTranslatedExceptionList; +} + +/** + * Translates Exception list items to Exceptions the endpoint can understand + * @param exc + */ +export function translateToEndpointExceptions( + exc: FoundExceptionListItemSchema, + schemaVersion: string +): TranslatedEntry[] { + const translatedList: TranslatedEntry[] = []; + + if (schemaVersion === '1.0.0') { + exc.data.forEach((list) => { + list.entries.forEach((entry) => { + const tEntry = translateEntry(schemaVersion, entry); + if (tEntry !== undefined) { + translatedList.push(tEntry); + } + }); + }); + } else { + throw new Error('unsupported schemaVersion'); + } + return translatedList; +} + +function translateEntry( + schemaVersion: string, + entry: Entry | EntryNested +): TranslatedEntry | undefined { + let translatedEntry; + switch (entry.type) { + case 'nested': { + const e = (entry as unknown) as EntryNested; + const nestedEntries: TranslatedEntry[] = []; + for (const nestedEntry of e.entries) { + const translation = translateEntry(schemaVersion, nestedEntry); + if (translation !== undefined) { + nestedEntries.push(translation); + } + } + translatedEntry = { + entries: nestedEntries, + field: e.field, + type: 'nested', + } as TranslatedEntryNested; + break; + } + case 'match': { + const e = (entry as unknown) as EntryMatch; + translatedEntry = { + field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, + operator: e.operator, + type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', + value: e.value, + } as TranslatedEntryMatch; + break; + } + case 'match_any': + { + const e = (entry as unknown) as EntryMatchAny; + translatedEntry = { + field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, + operator: e.operator, + type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', + value: e.value, + } as TranslatedEntryMatchAny; + } + break; + } + return translatedEntry || undefined; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts new file mode 100644 index 0000000000000..0434e3d8ffcb2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; +import { InternalArtifactSchema } from '../../schemas'; +import { + getInternalArtifactMock, + getInternalArtifactMockWithDiffs, +} from '../../schemas/artifacts/saved_objects.mock'; +import { Manifest } from './manifest'; + +describe('manifest', () => { + describe('Manifest object sanity checks', () => { + const artifacts: InternalArtifactSchema[] = []; + const now = new Date(); + let manifest1: Manifest; + let manifest2: Manifest; + + beforeAll(async () => { + const artifactLinux = await getInternalArtifactMock('linux', '1.0.0'); + const artifactMacos = await getInternalArtifactMock('macos', '1.0.0'); + const artifactWindows = await getInternalArtifactMock('windows', '1.0.0'); + artifacts.push(artifactLinux); + artifacts.push(artifactMacos); + artifacts.push(artifactWindows); + + manifest1 = new Manifest(now, '1.0.0', 'v0'); + manifest1.addEntry(artifactLinux); + manifest1.addEntry(artifactMacos); + manifest1.addEntry(artifactWindows); + manifest1.setVersion('abcd'); + + const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); + manifest2 = new Manifest(new Date(), '1.0.0', 'v0'); + manifest2.addEntry(newArtifactLinux); + manifest2.addEntry(artifactMacos); + manifest2.addEntry(artifactWindows); + }); + + test('Can create manifest with valid schema version', () => { + const manifest = new Manifest(new Date(), '1.0.0', 'v0'); + expect(manifest).toBeInstanceOf(Manifest); + }); + + test('Cannot create manifest with invalid schema version', () => { + expect(() => { + new Manifest(new Date(), 'abcd' as ManifestSchemaVersion, 'v0'); + }).toThrow(); + }); + + test('Manifest transforms correctly to expected endpoint format', async () => { + expect(manifest1.toEndpointFormat()).toStrictEqual({ + artifacts: { + 'endpoint-exceptionlist-linux-1.0.0': { + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }, + 'endpoint-exceptionlist-macos-1.0.0': { + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }, + 'endpoint-exceptionlist-windows-1.0.0': { + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }, + }, + manifest_version: 'abcd', + schema_version: '1.0.0', + }); + }); + + test('Manifest transforms correctly to expected saved object format', async () => { + expect(manifest1.toSavedObject()).toStrictEqual({ + created: now.getTime(), + ids: [ + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + ], + }); + }); + + test('Manifest returns diffs since supplied manifest', async () => { + const diffs = manifest2.diff(manifest1); + expect(diffs).toEqual([ + { + id: + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + type: 'delete', + }, + { + id: + 'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1', + type: 'add', + }, + ]); + }); + + test('Manifest returns data for given artifact', async () => { + const artifact = artifacts[0]; + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.sha256}`); + expect(returned).toEqual(artifact); + }); + + test('Manifest returns entries map', async () => { + const entries = manifest1.getEntries(); + const keys = Object.keys(entries); + expect(keys).toEqual([ + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + ]); + }); + + test('Manifest returns true if contains artifact', async () => { + const found = manifest1.contains( + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + expect(found).toEqual(true); + }); + + test('Manifest can be created from list of artifacts', async () => { + const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); + expect( + manifest.contains( + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ) + ).toEqual(true); + expect( + manifest.contains( + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ) + ).toEqual(true); + expect( + manifest.contains( + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ) + ).toEqual(true); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts new file mode 100644 index 0000000000000..c343568226e22 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { validate } from '../../../../common/validate'; +import { InternalArtifactSchema, InternalManifestSchema } from '../../schemas/artifacts'; +import { + manifestSchemaVersion, + ManifestSchemaVersion, +} from '../../../../common/endpoint/schema/common'; +import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; +import { ManifestEntry } from './manifest_entry'; + +export interface ManifestDiff { + type: string; + id: string; +} + +export class Manifest { + private created: Date; + private entries: Record; + private schemaVersion: ManifestSchemaVersion; + + // For concurrency control + private version: string; + + constructor(created: Date, schemaVersion: string, version: string) { + this.created = created; + this.entries = {}; + this.version = version; + + const [validated, errors] = validate( + (schemaVersion as unknown) as object, + manifestSchemaVersion + ); + + if (errors != null || validated === null) { + throw new Error(`Invalid manifest version: ${schemaVersion}`); + } + + this.schemaVersion = validated; + } + + public static fromArtifacts( + artifacts: InternalArtifactSchema[], + schemaVersion: string, + version: string + ): Manifest { + const manifest = new Manifest(new Date(), schemaVersion, version); + artifacts.forEach((artifact) => { + manifest.addEntry(artifact); + }); + return manifest; + } + + public getSchemaVersion(): ManifestSchemaVersion { + return this.schemaVersion; + } + + public getVersion(): string { + return this.version; + } + + public setVersion(version: string) { + this.version = version; + } + + public addEntry(artifact: InternalArtifactSchema) { + const entry = new ManifestEntry(artifact); + this.entries[entry.getDocId()] = entry; + } + + public contains(artifactId: string): boolean { + return artifactId in this.entries; + } + + public getEntries(): Record { + return this.entries; + } + + public getArtifact(artifactId: string): InternalArtifactSchema { + return this.entries[artifactId].getArtifact(); + } + + public diff(manifest: Manifest): ManifestDiff[] { + const diffs: ManifestDiff[] = []; + + for (const id in manifest.getEntries()) { + if (!this.contains(id)) { + diffs.push({ type: 'delete', id }); + } + } + + for (const id in this.entries) { + if (!manifest.contains(id)) { + diffs.push({ type: 'add', id }); + } + } + + return diffs; + } + + public toEndpointFormat(): ManifestSchema { + const manifestObj: ManifestSchema = { + manifest_version: this.version ?? 'v0', + schema_version: this.schemaVersion, + artifacts: {}, + }; + + for (const entry of Object.values(this.entries)) { + manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); + } + + const [validated, errors] = validate(manifestObj, manifestSchema); + if (errors != null) { + throw new Error(errors); + } + + return validated as ManifestSchema; + } + + public toSavedObject(): InternalManifestSchema { + return { + created: this.created.getTime(), + ids: Object.keys(this.entries), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts new file mode 100644 index 0000000000000..34bd2b0f388e1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalArtifactSchema } from '../../schemas'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; +import { ManifestEntry } from './manifest_entry'; + +describe('manifest_entry', () => { + describe('ManifestEntry object sanity checks', () => { + let artifact: InternalArtifactSchema; + let manifestEntry: ManifestEntry; + + beforeAll(async () => { + artifact = await getInternalArtifactMock('windows', '1.0.0'); + manifestEntry = new ManifestEntry(artifact); + }); + + test('Can create manifest entry', () => { + expect(manifestEntry).toBeInstanceOf(ManifestEntry); + }); + + test('Correct doc_id is returned', () => { + expect(manifestEntry.getDocId()).toEqual( + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + }); + + test('Correct identifier is returned', () => { + expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-1.0.0'); + }); + + test('Correct sha256 is returned', () => { + expect(manifestEntry.getSha256()).toEqual( + '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + }); + + test('Correct size is returned', () => { + expect(manifestEntry.getSize()).toEqual(268); + }); + + test('Correct url is returned', () => { + expect(manifestEntry.getUrl()).toEqual( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + }); + + test('Correct artifact is returned', () => { + expect(manifestEntry.getArtifact()).toEqual(artifact); + }); + + test('Correct record is returned', () => { + expect(manifestEntry.getRecord()).toEqual({ + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts new file mode 100644 index 0000000000000..00fd446bf14b5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; + +export class ManifestEntry { + private artifact: InternalArtifactSchema; + + constructor(artifact: InternalArtifactSchema) { + this.artifact = artifact; + } + + public getDocId(): string { + return `${this.getIdentifier()}-${this.getSha256()}`; + } + + public getIdentifier(): string { + return this.artifact.identifier; + } + + public getSha256(): string { + return this.artifact.sha256; + } + + public getSize(): number { + return this.artifact.size; + } + + public getUrl(): string { + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getSha256()}`; + } + + public getArtifact(): InternalArtifactSchema { + return this.artifact; + } + + public getRecord(): ManifestEntrySchema { + return { + sha256: this.getSha256(), + size: this.getSize(), + url: this.getUrl(), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts new file mode 100644 index 0000000000000..d38026fbcbbd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsType } from '../../../../../../../src/core/server'; + +import { ArtifactConstants, ManifestConstants } from './common'; + +export const exceptionsArtifactSavedObjectType = ArtifactConstants.SAVED_OBJECT_TYPE; +export const manifestSavedObjectType = ManifestConstants.SAVED_OBJECT_TYPE; + +export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + identifier: { + type: 'keyword', + }, + sha256: { + type: 'keyword', + }, + encoding: { + type: 'keyword', + index: false, + }, + created: { + type: 'date', + index: false, + }, + body: { + type: 'binary', + index: false, + }, + size: { + type: 'long', + index: false, + }, + }, +}; + +export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + created: { + type: 'date', + index: false, + }, + // array of doc ids + ids: { + type: 'keyword', + index: false, + }, + }, +}; + +export const exceptionsArtifactType: SavedObjectsType = { + name: exceptionsArtifactSavedObjectType, + hidden: false, // TODO: should these be hidden? + namespaceType: 'agnostic', + mappings: exceptionsArtifactSavedObjectMappings, +}; + +export const manifestType: SavedObjectsType = { + name: manifestSavedObjectType, + hidden: false, // TODO: should these be hidden? + namespaceType: 'agnostic', + mappings: manifestSavedObjectMappings, +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts new file mode 100644 index 0000000000000..4391d89f3b2b2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestTask } from './task'; + +export class MockManifestTask extends ManifestTask { + public runTask = jest.fn(); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts new file mode 100644 index 0000000000000..daa8a7dd83ee0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../../../task_manager/server'; + +import { createMockEndpointAppContext } from '../../mocks'; + +import { ManifestTaskConstants, ManifestTask } from './task'; +import { MockManifestTask } from './task.mock'; + +describe('task', () => { + describe('Periodic task sanity checks', () => { + test('can create task', () => { + const manifestTask = new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: taskManagerMock.createSetup(), + }); + expect(manifestTask).toBeInstanceOf(ManifestTask); + }); + + test('task should be registered', () => { + const mockTaskManager = taskManagerMock.createSetup(); + new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: mockTaskManager, + }); + expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); + }); + + test('task should be scheduled', async () => { + const mockTaskManagerSetup = taskManagerMock.createSetup(); + const manifestTask = new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: mockTaskManagerSetup, + }); + const mockTaskManagerStart = taskManagerMock.createStart(); + manifestTask.start({ taskManager: mockTaskManagerStart }); + expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); + }); + + test('task should run', async () => { + const mockContext = createMockEndpointAppContext(); + const mockTaskManager = taskManagerMock.createSetup(); + const mockManifestTask = new MockManifestTask({ + endpointAppContext: mockContext, + taskManager: mockTaskManager, + }); + const mockTaskInstance = { + id: ManifestTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: ManifestTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ManifestTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockManifestTask.runTask).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts new file mode 100644 index 0000000000000..08d02e70dac16 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'src/core/server'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../../task_manager/server'; +import { EndpointAppContext } from '../../types'; + +export const ManifestTaskConstants = { + TIMEOUT: '1m', + TYPE: 'securitySolution:endpoint:exceptions-packager', + VERSION: '1.0.0', +}; + +export interface ManifestTaskSetupContract { + endpointAppContext: EndpointAppContext; + taskManager: TaskManagerSetupContract; +} + +export interface ManifestTaskStartContract { + taskManager: TaskManagerStartContract; +} + +export class ManifestTask { + private endpointAppContext: EndpointAppContext; + private logger: Logger; + + constructor(setupContract: ManifestTaskSetupContract) { + this.endpointAppContext = setupContract.endpointAppContext; + this.logger = this.endpointAppContext.logFactory.get(this.getTaskId()); + + setupContract.taskManager.registerTaskDefinitions({ + [ManifestTaskConstants.TYPE]: { + title: 'Security Solution Endpoint Exceptions Handler', + type: ManifestTaskConstants.TYPE, + timeout: ManifestTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + await this.runTask(taskInstance.id); + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + public start = async (startContract: ManifestTaskStartContract) => { + try { + await startContract.taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: ManifestTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + interval: '60s', + }, + state: {}, + params: { version: ManifestTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.debug(`Error scheduling task, received ${e.message}`); + } + }; + + private getTaskId = (): string => { + return `${ManifestTaskConstants.TYPE}:${ManifestTaskConstants.VERSION}`; + }; + + public runTask = async (taskId: string) => { + // Check that this task is current + if (taskId !== this.getTaskId()) { + // old task, return + this.logger.debug(`Outdated task running: ${taskId}`); + return; + } + + const manifestManager = this.endpointAppContext.service.getManifestManager(); + + if (manifestManager === undefined) { + this.logger.debug('Manifest Manager not available.'); + return; + } + + manifestManager + .refresh() + .then((wrappedManifest) => { + if (wrappedManifest) { + return manifestManager.dispatch(wrappedManifest); + } + }) + .then((wrappedManifest) => { + if (wrappedManifest) { + return manifestManager.commit(wrappedManifest); + } + }) + .catch((err) => { + this.logger.error(err); + }); + }; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index ffd919db87fc9..55d7baec36dc6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -5,23 +5,67 @@ */ import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; +import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; import { xpackMocks } from '../../../../mocks'; import { AgentService, IngestManagerStartContract, ExternalCallback, } from '../../../ingest_manager/server'; -import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; import { createPackageConfigServiceMock } from '../../../ingest_manager/server/mocks'; +import { ConfigType } from '../config'; +import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; +import { + EndpointAppContextService, + EndpointAppContextServiceStartContract, +} from './endpoint_app_context_services'; +import { + ManifestManagerMock, + getManifestManagerMock, +} from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { EndpointAppContext } from './types'; + +/** + * Creates a mocked EndpointAppContext. + */ +export const createMockEndpointAppContext = ( + mockManifestManager?: ManifestManagerMock +): EndpointAppContext => { + return { + logFactory: loggingSystemMock.create(), + // @ts-ignore + config: createMockConfig() as ConfigType, + service: createMockEndpointAppContextService(mockManifestManager), + }; +}; + +/** + * Creates a mocked EndpointAppContextService + */ +export const createMockEndpointAppContextService = ( + mockManifestManager?: ManifestManagerMock +): jest.Mocked => { + return { + start: jest.fn(), + stop: jest.fn(), + getAgentService: jest.fn(), + // @ts-ignore + getManifestManager: mockManifestManager ?? jest.fn(), + getScopedSavedObjectsClient: jest.fn(), + }; +}; /** - * Crates a mocked input contract for the `EndpointAppContextService#start()` method + * Creates a mocked input contract for the `EndpointAppContextService#start()` method */ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< EndpointAppContextServiceStartContract > => { return { agentService: createMockAgentService(), + savedObjectsStart: savedObjectsServiceMock.createStartContract(), + // @ts-ignore + manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< ReturnType, Parameters diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts new file mode 100644 index 0000000000000..540976134d8ae --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { + ILegacyClusterClient, + IRouter, + SavedObjectsClientContract, + ILegacyScopedClusterClient, + RouteConfig, + RequestHandler, + KibanaResponseFactory, + RequestHandlerContext, + SavedObject, +} from 'kibana/server'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, + httpServiceMock, + httpServerMock, + loggingSystemMock, +} from 'src/core/server/mocks'; +import { ExceptionsCache } from '../../lib/artifacts/cache'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { registerDownloadExceptionListRoute } from './download_exception_list'; +import { EndpointAppContextService } from '../../endpoint_app_context_services'; +import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; +import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; +import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; + +const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; +const expectedEndpointExceptions: WrappedTranslatedExceptionList = { + exceptions_list: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], +}; +const mockIngestSOResponse = { + page: 1, + per_page: 100, + total: 1, + saved_objects: [ + { + id: 'agent1', + type: 'agent', + references: [], + score: 0, + attributes: { + active: true, + access_api_key_id: 'pedTuHIBTEDt93wW0Fhr', + }, + }, + ], +}; +const AuthHeader = 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw=='; + +describe('test alerts route', () => { + let routerMock: jest.Mocked; + let mockClusterClient: jest.Mocked; + let mockScopedClient: jest.Mocked; + let mockSavedObjectClient: jest.Mocked; + let mockResponse: jest.Mocked; + // @ts-ignore + let routeConfig: RouteConfig; + let routeHandler: RequestHandler; + let endpointAppContextService: EndpointAppContextService; + let cache: ExceptionsCache; + let ingestSavedObjectClient: jest.Mocked; + + beforeEach(() => { + mockClusterClient = elasticsearchServiceMock.createClusterClient(); + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); + mockSavedObjectClient = savedObjectsClientMock.create(); + mockResponse = httpServerMock.createResponseFactory(); + mockClusterClient.asScoped.mockReturnValue(mockScopedClient); + routerMock = httpServiceMock.createRouter(); + endpointAppContextService = new EndpointAppContextService(); + cache = new ExceptionsCache(5); + const startContract = createMockEndpointAppContextServiceStartContract(); + + // The authentication with the Fleet Plugin needs a separate scoped SO Client + ingestSavedObjectClient = savedObjectsClientMock.create(); + ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + // @ts-ignore + startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient); + endpointAppContextService.start(startContract); + + registerDownloadExceptionListRoute( + routerMock, + { + logFactory: loggingSystemMock.create(), + service: endpointAppContextService, + config: () => Promise.resolve(createMockConfig()), + }, + cache + ); + }); + + it('should serve the artifact to download', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, + method: 'get', + params: { sha256: '123456' }, + headers: { + authorization: AuthHeader, + }, + }); + + // Mock the SavedObjectsClient get response for fetching the artifact + const mockArtifact = { + id: '2468', + type: 'test', + references: [], + attributes: { + identifier: mockArtifactName, + schemaVersion: '1.0.0', + sha256: '123456', + encoding: 'application/json', + created: Date.now(), + body: Buffer.from(JSON.stringify(expectedEndpointExceptions)).toString('base64'), + size: 100, + }, + }; + const soFindResp: SavedObject = { + ...mockArtifact, + }; + ingestSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + + const expectedHeaders = { + 'content-encoding': 'application/json', + 'content-disposition': `attachment; filename=${mockArtifactName}.json`, + }; + + expect(mockResponse.ok).toBeCalled(); + expect(mockResponse.ok.mock.calls[0][0]?.headers).toEqual(expectedHeaders); + const artifact = mockResponse.ok.mock.calls[0][0]?.body; + expect(artifact).toEqual(Buffer.from(mockArtifact.attributes.body, 'base64').toString()); + }); + + it('should handle fetching a non-existent artifact', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, + method: 'get', + params: { sha256: '789' }, + headers: { + authorization: AuthHeader, + }, + }); + + ingestSavedObjectClient.get.mockImplementationOnce(() => + // eslint-disable-next-line prefer-promise-reject-errors + Promise.reject({ output: { statusCode: 404 } }) + ); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.notFound).toBeCalled(); + }); + + it('should utilize the cache', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + headers: { + authorization: AuthHeader, + }, + }); + + // Add to the download cache + const mockArtifact = expectedEndpointExceptions; + const cacheKey = `${mockArtifactName}-${mockSha}`; + cache.set(cacheKey, JSON.stringify(mockArtifact)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.ok).toBeCalled(); + // The saved objects client should be bypassed as the cache will contain the download + expect(ingestSavedObjectClient.get.mock.calls.length).toEqual(0); + }); + + it('should respond with a 401 if a valid API Token is not supplied', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + }); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.unauthorized).toBeCalled(); + }); + + it('should respond with a 404 if an agent cannot be linked to the API token', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + headers: { + authorization: AuthHeader, + }, + }); + + // Mock the SavedObjectsClient find response for verifying the API token with no results + mockIngestSOResponse.saved_objects = []; + mockIngestSOResponse.total = 0; + ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.notFound).toBeCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts new file mode 100644 index 0000000000000..337393e768a8f --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + IRouter, + SavedObjectsClientContract, + HttpResponseOptions, + IKibanaResponse, + SavedObject, +} from 'src/core/server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; +import { validate } from '../../../../common/validate'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; +import { ArtifactConstants, ExceptionsCache } from '../../lib/artifacts'; +import { + DownloadArtifactRequestParamsSchema, + downloadArtifactRequestParamsSchema, + downloadArtifactResponseSchema, + InternalArtifactSchema, +} from '../../schemas/artifacts'; +import { EndpointAppContext } from '../../types'; + +const allowlistBaseRoute: string = '/api/endpoint/artifacts'; + +/** + * Registers the exception list route to enable sensors to download an allowlist artifact + */ +export function registerDownloadExceptionListRoute( + router: IRouter, + endpointContext: EndpointAppContext, + cache: ExceptionsCache +) { + router.get( + { + path: `${allowlistBaseRoute}/download/{identifier}/{sha256}`, + validate: { + params: buildRouteValidation< + typeof downloadArtifactRequestParamsSchema, + DownloadArtifactRequestParamsSchema + >(downloadArtifactRequestParamsSchema), + }, + options: { tags: [] }, + }, + // @ts-ignore + async (context, req, res) => { + let scopedSOClient: SavedObjectsClientContract; + const logger = endpointContext.logFactory.get('download_exception_list'); + + // The ApiKey must be associated with an enrolled Fleet agent + try { + scopedSOClient = endpointContext.service.getScopedSavedObjectsClient(req); + await authenticateAgentWithAccessToken(scopedSOClient, req); + } catch (err) { + if (err.output.statusCode === 401) { + return res.unauthorized(); + } else { + return res.notFound(); + } + } + + const buildAndValidateResponse = (artName: string, body: string): IKibanaResponse => { + const artifact: HttpResponseOptions = { + body, + headers: { + 'content-encoding': 'application/json', + 'content-disposition': `attachment; filename=${artName}.json`, + }, + }; + + const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); + if (errors !== null || validated === null) { + return res.internalError({ body: errors! }); + } else { + return res.ok((validated as unknown) as HttpResponseOptions); + } + }; + + const id = `${req.params.identifier}-${req.params.sha256}`; + const cacheResp = cache.get(id); + + if (cacheResp) { + logger.debug(`Cache HIT artifact ${id}`); + return buildAndValidateResponse(req.params.identifier, cacheResp); + } else { + logger.debug(`Cache MISS artifact ${id}`); + return scopedSOClient + .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) + .then((artifact: SavedObject) => { + const body = Buffer.from(artifact.attributes.body, 'base64').toString(); + cache.set(id, body); + return buildAndValidateResponse(artifact.attributes.identifier, body); + }) + .catch((err) => { + if (err?.output?.statusCode === 404) { + return res.notFound({ body: `No artifact found for ${id}` }); + } else { + return res.internalError({ body: err }); + } + }); + } + } + ); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts new file mode 100644 index 0000000000000..945646c73c46c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_exception_list'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts new file mode 100644 index 0000000000000..3c066e150288a --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const body = t.string; + +export const created = t.number; // TODO: Make this into an ISO Date string check + +export const encoding = t.keyof({ + 'application/json': null, +}); + +export const schemaVersion = t.keyof({ + '1.0.0': null, +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts new file mode 100644 index 0000000000000..908fbb698adef --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './common'; +export * from './lists'; +export * from './request'; +export * from './response'; +export * from './saved_objects'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts new file mode 100644 index 0000000000000..7354b5fd0ec4d --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { WrappedTranslatedExceptionList } from './lists'; + +export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList => { + return { + exceptions_list: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts new file mode 100644 index 0000000000000..21d1105a313e7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { operator } from '../../../../../lists/common/schemas'; + +export const translatedEntryMatchAny = t.exact( + t.type({ + field: t.string, + operator, + type: t.keyof({ + exact_cased_any: null, + exact_caseless_any: null, + }), + value: t.array(t.string), + }) +); +export type TranslatedEntryMatchAny = t.TypeOf; + +export const translatedEntryMatch = t.exact( + t.type({ + field: t.string, + operator, + type: t.keyof({ + exact_cased: null, + exact_caseless: null, + }), + value: t.string, + }) +); +export type TranslatedEntryMatch = t.TypeOf; + +export const translatedEntryNested = t.exact( + t.type({ + field: t.string, + type: t.keyof({ nested: null }), + entries: t.array(t.union([translatedEntryMatch, translatedEntryMatchAny])), + }) +); +export type TranslatedEntryNested = t.TypeOf; + +export const translatedEntry = t.union([ + translatedEntryNested, + translatedEntryMatch, + translatedEntryMatchAny, +]); +export type TranslatedEntry = t.TypeOf; + +export const translatedExceptionList = t.exact( + t.type({ + type: t.string, + entries: t.array(translatedEntry), + }) +); +export type TranslatedExceptionList = t.TypeOf; + +export const wrappedExceptionList = t.exact( + t.type({ + exceptions_list: t.array(translatedEntry), + }) +); +export type WrappedTranslatedExceptionList = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts new file mode 100644 index 0000000000000..7a194fdc7b5f4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { identifier, sha256 } from '../../../../../common/endpoint/schema/common'; + +export const downloadArtifactRequestParamsSchema = t.exact( + t.type({ + identifier, + sha256, + }) +); + +export type DownloadArtifactRequestParamsSchema = t.TypeOf< + typeof downloadArtifactRequestParamsSchema +>; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts new file mode 100644 index 0000000000000..13e4165eb5f16 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_artifact_schema'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts new file mode 100644 index 0000000000000..537f7707889e4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { encoding } from '../common'; + +const body = t.string; +const headers = t.exact( + t.type({ + 'content-encoding': encoding, + 'content-disposition': t.string, + }) +); + +export const downloadArtifactResponseSchema = t.exact( + t.type({ + body, + headers, + }) +); + +export type DownloadArtifactResponseSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts new file mode 100644 index 0000000000000..13e4165eb5f16 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_artifact_schema'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts new file mode 100644 index 0000000000000..1a9cc55ca5725 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ArtifactConstants, buildArtifact } from '../../lib/artifacts'; +import { getTranslatedExceptionListMock } from './lists.mock'; +import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; + +export const getInternalArtifactMock = async ( + os: string, + schemaVersion: string +): Promise => { + return buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); +}; + +export const getInternalArtifactMockWithDiffs = async ( + os: string, + schemaVersion: string +): Promise => { + const mock = getTranslatedExceptionListMock(); + mock.exceptions_list.pop(); + return buildArtifact(mock, os, schemaVersion); +}; + +export const getInternalArtifactsMock = async ( + os: string, + schemaVersion: string +): Promise => { + // @ts-ignore + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async () => { + await buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); + }); +}; + +export const getInternalManifestMock = (): InternalManifestSchema => ({ + created: Date.now(), + ids: [], +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts new file mode 100644 index 0000000000000..2e71ef98387f1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { identifier, sha256, size } from '../../../../common/endpoint/schema/common'; +import { body, created, encoding } from './common'; + +export const internalArtifactSchema = t.exact( + t.type({ + identifier, + sha256, + encoding, + created, + body, + size, + }) +); + +export type InternalArtifactSchema = t.TypeOf; + +export const internalManifestSchema = t.exact( + t.type({ + created, + ids: t.array(identifier), + }) +); + +export type InternalManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts new file mode 100644 index 0000000000000..a3b6e68e4ada2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifacts'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts new file mode 100644 index 0000000000000..6392c59b2377c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { SavedObjectsClientContract } from 'src/core/server'; +import { ArtifactClient } from './artifact_client'; + +export const getArtifactClientMock = ( + savedObjectsClient?: SavedObjectsClientContract +): ArtifactClient => { + if (savedObjectsClient !== undefined) { + return new ArtifactClient(savedObjectsClient); + } + return new ArtifactClient(savedObjectsClientMock.create()); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts new file mode 100644 index 0000000000000..08e29b5c6b82b --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; +import { getArtifactClientMock } from './artifact_client.mock'; +import { ArtifactClient } from './artifact_client'; + +describe('artifact_client', () => { + describe('ArtifactClient sanity checks', () => { + test('can create ArtifactClient', () => { + const artifactClient = new ArtifactClient(savedObjectsClientMock.create()); + expect(artifactClient).toBeInstanceOf(ArtifactClient); + }); + + test('can get artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + await artifactClient.getArtifact('abcd'); + expect(savedObjectsClient.get).toHaveBeenCalled(); + }); + + test('can create artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + const artifact = await getInternalArtifactMock('linux', '1.0.0'); + await artifactClient.createArtifact(artifact); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: artifactClient.getArtifactId(artifact) } + ); + }); + + test('can delete artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + await artifactClient.deleteArtifact('abcd'); + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + 'abcd' + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts new file mode 100644 index 0000000000000..4a3dcaae1bd3d --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { InternalArtifactSchema } from '../../schemas/artifacts'; + +export class ArtifactClient { + private savedObjectsClient: SavedObjectsClientContract; + + constructor(savedObjectsClient: SavedObjectsClientContract) { + this.savedObjectsClient = savedObjectsClient; + } + + public getArtifactId(artifact: InternalArtifactSchema) { + return `${artifact.identifier}-${artifact.sha256}`; + } + + public async getArtifact(id: string): Promise> { + return this.savedObjectsClient.get( + ArtifactConstants.SAVED_OBJECT_TYPE, + id + ); + } + + public async createArtifact( + artifact: InternalArtifactSchema + ): Promise> { + return this.savedObjectsClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: this.getArtifactId(artifact) } + ); + } + + public async deleteArtifact(id: string) { + return this.savedObjectsClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, id); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts new file mode 100644 index 0000000000000..44a4d7e77dbcb --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifact_client'; +export * from './manifest_manager'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts new file mode 100644 index 0000000000000..bfeacbcedf2cb --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsClientContract } from 'src/core/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ManifestClient } from './manifest_client'; + +export const getManifestClientMock = ( + savedObjectsClient?: SavedObjectsClientContract +): ManifestClient => { + if (savedObjectsClient !== undefined) { + return new ManifestClient(savedObjectsClient, '1.0.0'); + } + return new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts new file mode 100644 index 0000000000000..5780c6279ee6a --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; +import { ManifestConstants } from '../../lib/artifacts'; +import { getInternalManifestMock } from '../../schemas/artifacts/saved_objects.mock'; +import { getManifestClientMock } from './manifest_client.mock'; +import { ManifestClient } from './manifest_client'; + +describe('manifest_client', () => { + describe('ManifestClient sanity checks', () => { + test('can create ManifestClient', () => { + const manifestClient = new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + expect(manifestClient).toBeInstanceOf(ManifestClient); + }); + + test('cannot create ManifestClient with invalid schema version', () => { + expect(() => { + new ManifestClient(savedObjectsClientMock.create(), 'invalid' as ManifestSchemaVersion); + }).toThrow(); + }); + + test('can get manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + await manifestClient.getManifest(); + expect(savedObjectsClient.get).toHaveBeenCalled(); + }); + + test('can create manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifest = getInternalManifestMock(); + await manifestClient.createManifest(manifest); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest, + { id: manifestClient.getManifestId() } + ); + }); + + test('can update manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifest = getInternalManifestMock(); + await manifestClient.updateManifest(manifest, { version: 'abcd' }); + expect(savedObjectsClient.update).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifestClient.getManifestId(), + manifest, + { version: 'abcd' } + ); + }); + + test('can delete manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + await manifestClient.deleteManifest(); + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifestClient.getManifestId() + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts new file mode 100644 index 0000000000000..45182841e56fc --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObject, + SavedObjectsClientContract, + SavedObjectsUpdateResponse, +} from 'src/core/server'; +import { + manifestSchemaVersion, + ManifestSchemaVersion, +} from '../../../../common/endpoint/schema/common'; +import { validate } from '../../../../common/validate'; +import { ManifestConstants } from '../../lib/artifacts'; +import { InternalManifestSchema } from '../../schemas/artifacts'; + +interface UpdateManifestOpts { + version: string; +} + +export class ManifestClient { + private schemaVersion: ManifestSchemaVersion; + private savedObjectsClient: SavedObjectsClientContract; + + constructor( + savedObjectsClient: SavedObjectsClientContract, + schemaVersion: ManifestSchemaVersion + ) { + this.savedObjectsClient = savedObjectsClient; + + const [validated, errors] = validate( + (schemaVersion as unknown) as object, + manifestSchemaVersion + ); + + if (errors != null || validated === null) { + throw new Error(`Invalid manifest version: ${schemaVersion}`); + } + + this.schemaVersion = validated; + } + + public getManifestId(): string { + return `endpoint-manifest-${this.schemaVersion}`; + } + + public async getManifest(): Promise> { + return this.savedObjectsClient.get( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId() + ); + } + + public async createManifest( + manifest: InternalManifestSchema + ): Promise> { + return this.savedObjectsClient.create( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest, + { id: this.getManifestId() } + ); + } + + public async updateManifest( + manifest: InternalManifestSchema, + opts?: UpdateManifestOpts + ): Promise> { + return this.savedObjectsClient.update( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId(), + manifest, + opts + ); + } + + public async deleteManifest() { + return this.savedObjectsClient.delete( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId() + ); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts new file mode 100644 index 0000000000000..03d5d27b3ff78 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './manifest_manager'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts new file mode 100644 index 0000000000000..cd70b11aef305 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// eslint-disable-next-line max-classes-per-file +import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks'; +import { Logger } from 'src/core/server'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { listMock } from '../../../../../../lists/server/mocks'; +import { + ExceptionsCache, + Manifest, + buildArtifact, + getFullEndpointExceptionList, +} from '../../../lib/artifacts'; +import { InternalArtifactSchema } from '../../../schemas/artifacts'; +import { getArtifactClientMock } from '../artifact_client.mock'; +import { getManifestClientMock } from '../manifest_client.mock'; +import { ManifestManager } from './manifest_manager'; + +function getMockPackageConfig() { + return { + id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', + inputs: [ + { + config: {}, + }, + ], + revision: 1, + version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992) + updated_at: '2020-06-25T16:03:38.159292', + updated_by: 'kibana', + created_at: '2020-06-25T16:03:38.159292', + created_by: 'kibana', + }; +} + +class PackageConfigServiceMock { + public create = jest.fn().mockResolvedValue(getMockPackageConfig()); + public get = jest.fn().mockResolvedValue(getMockPackageConfig()); + public getByIds = jest.fn().mockResolvedValue([getMockPackageConfig()]); + public list = jest.fn().mockResolvedValue({ + items: [getMockPackageConfig()], + total: 1, + page: 1, + perPage: 20, + }); + public update = jest.fn().mockResolvedValue(getMockPackageConfig()); +} + +export function getPackageConfigServiceMock() { + return new PackageConfigServiceMock(); +} + +async function mockBuildExceptionListArtifacts( + os: string, + schemaVersion: string +): Promise { + const mockExceptionClient = listMock.getExceptionListClient(); + const first = getFoundExceptionListItemSchemaMock(); + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + const exceptions = await getFullEndpointExceptionList(mockExceptionClient, os, schemaVersion); + return [await buildArtifact(exceptions, os, schemaVersion)]; +} + +// @ts-ignore +export class ManifestManagerMock extends ManifestManager { + // @ts-ignore + private buildExceptionListArtifacts = async () => { + return mockBuildExceptionListArtifacts('linux', '1.0.0'); + }; + + // @ts-ignore + private getLastDispatchedManifest = jest + .fn() + .mockResolvedValue(new Manifest(new Date(), '1.0.0', 'v0')); + + // @ts-ignore + private getManifestClient = jest + .fn() + .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); +} + +export const getManifestManagerMock = (opts?: { + packageConfigService?: PackageConfigServiceMock; + savedObjectsClient?: ReturnType; +}): ManifestManagerMock => { + let packageConfigService = getPackageConfigServiceMock(); + if (opts?.packageConfigService !== undefined) { + packageConfigService = opts.packageConfigService; + } + + let savedObjectsClient = savedObjectsClientMock.create(); + if (opts?.savedObjectsClient !== undefined) { + savedObjectsClient = opts.savedObjectsClient; + } + + const manifestManager = new ManifestManagerMock({ + artifactClient: getArtifactClientMock(savedObjectsClient), + cache: new ExceptionsCache(5), + // @ts-ignore + packageConfigService, + exceptionListClient: listMock.getExceptionListClient(), + logger: loggingSystemMock.create().get() as jest.Mocked, + savedObjectsClient, + }); + + return manifestManager; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts new file mode 100644 index 0000000000000..bbb6fdfd50810 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ArtifactConstants, ManifestConstants, Manifest } from '../../../lib/artifacts'; +import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_manager.mock'; + +describe('manifest_manager', () => { + describe('ManifestManager sanity checks', () => { + test('ManifestManager can refresh manifest', async () => { + const manifestManager = getManifestManagerMock(); + const manifestWrapper = await manifestManager.refresh(); + expect(manifestWrapper!.diffs).toEqual([ + { + id: + 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', + type: 'add', + }, + ]); + expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); + }); + + test('ManifestManager can dispatch manifest', async () => { + const packageConfigService = getPackageConfigServiceMock(); + const manifestManager = getManifestManagerMock({ packageConfigService }); + const manifestWrapperRefresh = await manifestManager.refresh(); + const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch); + const entries = manifestWrapperDispatch!.manifest.getEntries(); + const artifact = Object.values(entries)[0].getArtifact(); + expect( + packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value + ).toEqual({ + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: { + [artifact.identifier]: { + sha256: artifact.sha256, + size: artifact.size, + url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.sha256}`, + }, + }, + }); + }); + + test('ManifestManager can commit manifest', async () => { + const savedObjectsClient: ReturnType = savedObjectsClientMock.create(); + const manifestManager = getManifestManagerMock({ + savedObjectsClient, + }); + + const manifestWrapperRefresh = await manifestManager.refresh(); + const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + const diff = { + id: 'abcd', + type: 'delete', + }; + manifestWrapperDispatch!.diffs.push(diff); + + await manifestManager.commit(manifestWrapperDispatch); + + // created new artifact + expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( + ArtifactConstants.SAVED_OBJECT_TYPE + ); + + // deleted old artifact + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + 'abcd' + ); + + // committed new manifest + expect(savedObjectsClient.create.mock.calls[1][0]).toEqual( + ManifestConstants.SAVED_OBJECT_TYPE + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts new file mode 100644 index 0000000000000..33b0d5db575c6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -0,0 +1,270 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, SavedObjectsClientContract, SavedObject } from 'src/core/server'; +import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; +import { ExceptionListClient } from '../../../../../../lists/server'; +import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; +import { + ArtifactConstants, + ManifestConstants, + Manifest, + buildArtifact, + getFullEndpointExceptionList, + ExceptionsCache, + ManifestDiff, +} from '../../../lib/artifacts'; +import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; +import { ArtifactClient } from '../artifact_client'; +import { ManifestClient } from '../manifest_client'; + +export interface ManifestManagerContext { + savedObjectsClient: SavedObjectsClientContract; + artifactClient: ArtifactClient; + exceptionListClient: ExceptionListClient; + packageConfigService: PackageConfigServiceInterface; + logger: Logger; + cache: ExceptionsCache; +} + +export interface ManifestRefreshOpts { + initialize?: boolean; +} + +export interface WrappedManifest { + manifest: Manifest; + diffs: ManifestDiff[]; +} + +export class ManifestManager { + protected artifactClient: ArtifactClient; + protected exceptionListClient: ExceptionListClient; + protected packageConfigService: PackageConfigServiceInterface; + protected savedObjectsClient: SavedObjectsClientContract; + protected logger: Logger; + protected cache: ExceptionsCache; + + constructor(context: ManifestManagerContext) { + this.artifactClient = context.artifactClient; + this.exceptionListClient = context.exceptionListClient; + this.packageConfigService = context.packageConfigService; + this.savedObjectsClient = context.savedObjectsClient; + this.logger = context.logger; + this.cache = context.cache; + } + + private getManifestClient(schemaVersion: string): ManifestClient { + return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); + } + + private async buildExceptionListArtifacts( + schemaVersion: string + ): Promise { + const artifacts: InternalArtifactSchema[] = []; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + const exceptionList = await getFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const artifact = await buildArtifact(exceptionList, os, schemaVersion); + + artifacts.push(artifact); + } + + return artifacts; + } + + private async getLastDispatchedManifest(schemaVersion: string): Promise { + return this.getManifestClient(schemaVersion) + .getManifest() + .then(async (manifestSo: SavedObject) => { + if (manifestSo.version === undefined) { + throw new Error('No version returned for manifest.'); + } + const manifest = new Manifest( + new Date(manifestSo.attributes.created), + schemaVersion, + manifestSo.version + ); + + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } + + return manifest; + }) + .catch((err) => { + if (err.output.statusCode !== 404) { + throw err; + } + return null; + }); + } + + public async refresh(opts?: ManifestRefreshOpts): Promise { + let oldManifest: Manifest | null; + + // Get the last-dispatched manifest + oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + + if (oldManifest === null && opts !== undefined && opts.initialize) { + oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION, 'v0'); // create empty manifest + } else if (oldManifest == null) { + this.logger.debug('Manifest does not exist yet. Waiting...'); + return null; + } + + // Build new exception list artifacts + const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + + // Build new manifest + const newManifest = Manifest.fromArtifacts( + artifacts, + ManifestConstants.SCHEMA_VERSION, + oldManifest.getVersion() + ); + + // Get diffs + const diffs = newManifest.diff(oldManifest); + + // Create new artifacts + for (const diff of diffs) { + if (diff.type === 'add') { + const artifact = newManifest.getArtifact(diff.id); + try { + await this.artifactClient.createArtifact(artifact); + // Cache the body of the artifact + this.cache.set(diff.id, artifact.body); + } catch (err) { + if (err.status === 409) { + // This artifact already existed... + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + throw err; + } + } + } + } + + return { + manifest: newManifest, + diffs, + }; + } + + /** + * Dispatches the manifest by writing it to the endpoint packageConfig. + * + * @return {WrappedManifest | null} WrappedManifest if all dispatched, else null + */ + public async dispatch(wrappedManifest: WrappedManifest | null): Promise { + if (wrappedManifest === null) { + this.logger.debug('wrappedManifest was null, aborting dispatch'); + return null; + } + + function showDiffs(diffs: ManifestDiff[]) { + return diffs.map((diff) => { + const op = diff.type === 'add' ? '(+)' : '(-)'; + return `${op}${diff.id}`; + }); + } + + if (wrappedManifest.diffs.length > 0) { + this.logger.info(`Dispatching new manifest with diffs: ${showDiffs(wrappedManifest.diffs)}`); + + let paging = true; + let success = true; + + while (paging) { + const { items, total, page } = await this.packageConfigService.list( + this.savedObjectsClient, + { + page: 1, + perPage: 20, + kuery: 'ingest-package-configs.package.name:endpoint', + } + ); + + for (const packageConfig of items) { + const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; + + if ( + newPackageConfig.inputs.length > 0 && + newPackageConfig.inputs[0].config !== undefined + ) { + const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + artifactManifest.value = wrappedManifest.manifest.toEndpointFormat(); + newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; + + await this.packageConfigService + .update(this.savedObjectsClient, id, newPackageConfig) + .then((response) => { + this.logger.debug(`Updated package config ${id}`); + }) + .catch((err) => { + success = false; + this.logger.debug(`Error updating package config ${id}`); + this.logger.error(err); + }); + } else { + success = false; + this.logger.debug(`Package config ${id} has no config.`); + } + } + + paging = page * items.length < total; + } + + return success ? wrappedManifest : null; + } else { + this.logger.debug('No manifest diffs [no-op]'); + } + + return null; + } + + public async commit(wrappedManifest: WrappedManifest | null) { + if (wrappedManifest === null) { + this.logger.debug('wrappedManifest was null, aborting commit'); + return; + } + + const manifestClient = this.getManifestClient(wrappedManifest.manifest.getSchemaVersion()); + + // Commit the new manifest + if (wrappedManifest.manifest.getVersion() === 'v0') { + await manifestClient.createManifest(wrappedManifest.manifest.toSavedObject()); + } else { + const version = wrappedManifest.manifest.getVersion(); + if (version === 'v0') { + throw new Error('Updating existing manifest with baseline version. Bad state.'); + } + await manifestClient.updateManifest(wrappedManifest.manifest.toSavedObject(), { + version, + }); + } + + this.logger.info(`Commited manifest ${wrappedManifest.manifest.getVersion()}`); + + // Clean up old artifacts + for (const diff of wrappedManifest.diffs) { + try { + if (diff.type === 'delete') { + await this.artifactClient.deleteArtifact(diff.id); + this.logger.info(`Cleaned up artifact ${diff.id}`); + } + } catch (err) { + this.logger.error(err); + } + } + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/index.ts new file mode 100644 index 0000000000000..a3b6e68e4ada2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifacts'; diff --git a/x-pack/plugins/security_solution/server/endpoint/types.ts b/x-pack/plugins/security_solution/server/endpoint/types.ts index fbcc5bc833d73..3c6630db8ebd8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/types.ts +++ b/x-pack/plugins/security_solution/server/endpoint/types.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { LoggerFactory } from 'kibana/server'; -import { EndpointAppContextService } from './endpoint_app_context_services'; import { ConfigType } from '../config'; +import { EndpointAppContextService } from './endpoint_app_context_services'; /** * The context for Endpoint apps. diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 9bb1bea0949e0..a97f1eee56342 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -11,9 +11,10 @@ import { i18n } from '@kbn/i18n'; import { CoreSetup, CoreStart, + Logger, Plugin as IPlugin, PluginInitializerContext, - Logger, + SavedObjectsClient, } from '../../../../src/core/server'; import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; @@ -24,6 +25,7 @@ import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from ' import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; import { IngestManagerStartContract } from '../../ingest_manager/server'; +import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { initServer } from './init_server'; import { compose } from './lib/compose/kibana'; import { initRoutes } from './routes'; @@ -32,6 +34,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; +import { ManifestTask, ExceptionsCache } from './endpoint/lib/artifacts'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -40,8 +43,10 @@ import { APP_ID, APP_ICON, SERVER_APP_ID } from '../common/constants'; import { registerEndpointRoutes } from './endpoint/routes/metadata'; import { registerResolverRoutes } from './endpoint/routes/resolver'; import { registerPolicyRoutes } from './endpoint/routes/policy'; +import { ArtifactClient, ManifestManager } from './endpoint/services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; +import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; export interface SetupPlugins { alerts: AlertingSetup; @@ -50,12 +55,14 @@ export interface SetupPlugins { licensing: LicensingPluginSetup; security?: SecuritySetup; spaces?: SpacesSetup; + taskManager: TaskManagerSetupContract; ml?: MlSetup; lists?: ListPluginSetup; } export interface StartPlugins { ingestManager: IngestManagerStartContract; + taskManager: TaskManagerStartContract; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -70,11 +77,17 @@ export class Plugin implements IPlugin type.name); diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts new file mode 100644 index 0000000000000..2721592ba3350 --- /dev/null +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const supertestWithoutAuth = getSupertestWithoutAuth(providerContext); + let agentAccessAPIKey: string; + + describe('artifact download', () => { + setupIngest(providerContext); + before(async () => { + await esArchiver.load('endpoint/artifacts/api_feature', { useCreate: true }); + + const { body: enrollmentApiKeysResponse } = await supertest + .get(`/api/ingest_manager/fleet/enrollment-api-keys`) + .expect(200); + expect(enrollmentApiKeysResponse.list).length(2); + + const { body: enrollmentApiKeyResponse } = await supertest + .get( + `/api/ingest_manager/fleet/enrollment-api-keys/${enrollmentApiKeysResponse.list[0].id}` + ) + .expect(200); + expect(enrollmentApiKeyResponse.item).to.have.key('api_key'); + const enrollmentAPIToken = enrollmentApiKeyResponse.item.api_key; + + // 2. Enroll agent + const { body: enrollmentResponse } = await supertestWithoutAuth + .post(`/api/ingest_manager/fleet/agents/enroll`) + .set('kbn-xsrf', 'xxx') + .set('Authorization', `ApiKey ${enrollmentAPIToken}`) + .send({ + type: 'PERMANENT', + metadata: { + local: { + elastic: { + agent: { + version: '7.0.0', + }, + }, + }, + user_provided: {}, + }, + }) + .expect(200); + expect(enrollmentResponse.success).to.eql(true); + + agentAccessAPIKey = enrollmentResponse.item.access_api_key; + }); + after(() => esArchiver.unload('endpoint/artifacts/api_feature')); + + it('should fail to find artifact with invalid hash', async () => { + await supertestWithoutAuth + .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/abcd') + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(404); + }); + + it('should download an artifact with correct hash', async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200); + }); + + it('should fail on invalid api key', async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey iNvAlId`) + .send() + .expect(401); + }); + }); +} diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json new file mode 100644 index 0000000000000..a886b60e7e0dc --- /dev/null +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -0,0 +1,179 @@ +{ + "type": "doc", + "value": { + "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "index": ".kibana", + "source": { + "references": [ + ], + "endpoint:exceptions-artifact": { + "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", + "created": 1593016187465, + "encoding": "application/json", + "identifier": "endpoint-exceptionlist-linux-1.0.0", + "sha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "size": 22 + }, + "type": "endpoint:exceptions-artifact", + "updated_at": "2020-06-24T16:29:47.584Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "endpoint:exceptions-manifest:endpoint-manifest-1.0.0", + "index": ".kibana", + "source": { + "references": [ + ], + "endpoint:exceptions-manifest": { + "created": 1593183699663, + "ids": [ + "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-macos-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" + ] + }, + "type": "endpoint:exceptions-manifest", + "updated_at": "2020-06-26T15:01:39.704Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "exception-list-agnostic:13a7ef40-b63b-11ea-ace9-591c8e572c76", + "index": ".kibana", + "source": { + "exception-list-agnostic": { + "_tags": [ + "endpoint", + "process", + "malware", + "os:linux" + ], + "created_at": "2020-06-24T16:52:23.689Z", + "created_by": "akahan", + "description": "This is a sample agnostic endpoint type exception", + "list_id": "endpoint_list", + "list_type": "list", + "name": "Sample Endpoint Exception List", + "tags": [ + "user added string for a tag", + "malware" + ], + "tie_breaker_id": "e3b20e6e-c023-4575-a033-47990115969c", + "type": "endpoint", + "updated_by": "akahan" + }, + "references": [ + ], + "type": "exception-list-agnostic", + "updated_at": "2020-06-24T16:52:23.732Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "exception-list-agnostic:679b95a0-b714-11ea-a4c9-0963ae39bc3d", + "index": ".kibana", + "source": { + "exception-list-agnostic": { + "_tags": [ + "os:windows" + ], + "comments": [ + ], + "created_at": "2020-06-25T18:48:05.326Z", + "created_by": "akahan", + "description": "This is a sample endpoint type exception", + "entries": [ + { + "field": "actingProcess.file.signer", + "operator": "included", + "type": "match", + "value": "Elastic, N.V." + }, + { + "field": "event.category", + "operator": "included", + "type": "match_any", + "value": [ + "process", + "malware" + ] + } + ], + "item_id": "61142b8f-5876-4709-9952-95160cd58f2f", + "list_id": "endpoint_list", + "list_type": "item", + "name": "Sample Endpoint Exception List", + "tags": [ + "user added string for a tag", + "malware" + ], + "tie_breaker_id": "b36176d2-bc75-4641-a8e3-e811c6bc30d8", + "type": "endpoint", + "updated_by": "akahan" + }, + "references": [ + ], + "type": "exception-list-agnostic", + "updated_at": "2020-06-25T18:48:05.369Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "fleet-agents:a34d87c1-726e-4c30-b2ff-1b4b95f59d2a", + "index": ".kibana", + "source": { + "fleet-agents": { + "access_api_key_id": "8ZnT7HIBwLFvkUEPQaT3", + "active": true, + "config_id": "2dd2a110-b6f6-11ea-a66d-63cf082a3b58", + "enrolled_at": "2020-06-25T18:52:47.290Z", + "local_metadata": { + "os": "macos" + }, + "type": "PERMANENT", + "user_provided_metadata": { + "region": "us-east" + } + }, + "references": [ + ], + "type": "fleet-agents", + "updated_at": "2020-06-25T18:52:48.464Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "fleet-enrollment-api-keys:8178eb66-392f-4b76-9dc9-704ed1a5c56e", + "index": ".kibana", + "source": { + "fleet-enrollment-api-keys": { + "active": true, + "api_key": "8ZnT7HIBwLFvkUEPQaT3", + "api_key_id": "8ZnT7HIBwLFvkUEPQaT3", + "config_id": "2dd2a110-b6f6-11ea-a66d-63cf082a3b58", + "created_at": "2020-06-25T17:25:30.065Z", + "name": "Default (93aa98c8-d650-422e-aa7b-663dae3dff83)" + }, + "references": [ + ], + "type": "fleet-enrollment-api-keys", + "updated_at": "2020-06-25T17:25:30.114Z" + } + } +} \ No newline at end of file