From b34f4d41e2041138cca48619bdfceeb450bfeef2 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 16 Nov 2022 14:15:58 +0100 Subject: [PATCH 01/13] changes to fleet usage task, agent version telemetry --- .../server/collectors/agent_collectors.ts | 17 ++++ .../fleet/server/collectors/register.ts | 17 +++- x-pack/plugins/fleet/server/plugin.ts | 16 ++- .../server/services/fleet_usage_sender.ts | 97 +++++++++++++------ 4 files changed, 108 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 4828967462116..6c8f452f45ca2 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -8,6 +8,7 @@ import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; import type { FleetConfigType } from '../../common/types'; +import { AGENTS_INDEX } from '../../common'; import * as AgentService from '../services/agents'; export interface AgentUsage { @@ -47,3 +48,19 @@ export const getAgentUsage = async ( updating, }; }; + +export const getAgentVersions = async (esClient?: ElasticsearchClient) => { + if (!esClient) { + return {}; + } + const response = await esClient.search({ + index: AGENTS_INDEX, + size: 0, + aggs: { + versions: { + terms: { field: 'agent.version' }, + }, + }, + }); + return ((response?.aggregations?.versions as any).buckets ?? []).map((bucket: any) => bucket.key); +}; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index a194ff9b560e5..6bc65d8bf040c 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -11,7 +11,7 @@ import type { CoreSetup } from '@kbn/core/server'; import type { FleetConfigType } from '..'; import { getIsAgentsEnabled } from './config_collectors'; -import { getAgentUsage } from './agent_collectors'; +import { getAgentUsage, getAgentVersions } from './agent_collectors'; import type { AgentUsage } from './agent_collectors'; import { getInternalClients } from './helpers'; import { getPackageUsage } from './package_collectors'; @@ -26,7 +26,20 @@ export interface Usage { fleet_server: FleetServerUsage; } -export const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => { +export const fetchFleetUsage = async (core: CoreSetup, config: FleetConfigType) => { + const [soClient, esClient] = await getInternalClients(core); + const usage = { + agents_enabled: getIsAgentsEnabled(config), + agents: await getAgentUsage(config, soClient, esClient), + fleet_server: await getFleetServerUsage(soClient, esClient), + packages: await getPackageUsage(soClient), + agent_versions: await getAgentVersions(esClient), + }; + return usage; +}; + +// used by kibana daily collector +const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => { const [soClient, esClient] = await getInternalClients(core); const usage = { agents_enabled: getIsAgentsEnabled(config), diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 63ef9a304ade8..94fba065049bf 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -101,7 +101,11 @@ import { AgentServiceImpl, PackageServiceImpl, } from './services'; -import { registerFleetUsageCollector, fetchUsage, fetchAgentsUsage } from './collectors/register'; +import { + registerFleetUsageCollector, + fetchAgentsUsage, + fetchFleetUsage, +} from './collectors/register'; import { getAuthzFromRequest, makeRouterWithFleetAuthz } from './routes/security'; import { FleetArtifactsClient } from './services/artifacts'; import type { FleetRouter } from './types/request_context'; @@ -383,14 +387,8 @@ export class FleetPlugin // Register usage collection registerFleetUsageCollector(core, config, deps.usageCollection); - const fetch = async () => fetchUsage(core, config); - this.fleetUsageSender = new FleetUsageSender( - deps.taskManager, - core, - fetch, - this.kibanaVersion, - this.isProductionMode - ); + const fetch = async () => fetchFleetUsage(core, config); + this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch); registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config)); const router: FleetRouter = core.http.createRouter(); diff --git a/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts b/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts index ada764fcff927..5a5d76337e12d 100644 --- a/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts +++ b/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts @@ -9,6 +9,7 @@ import type { TaskManagerStartContract, TaskManagerSetupContract, } from '@kbn/task-manager-plugin/server'; +import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server'; import type { CoreSetup } from '@kbn/core/server'; import type { Usage } from '../collectors/register'; @@ -19,15 +20,15 @@ const EVENT_TYPE = 'fleet_usage'; export class FleetUsageSender { private taskManager?: TaskManagerStartContract; - private taskId = 'Fleet-Usage-Sender-Task'; + private taskVersion = '1.0.0'; private taskType = 'Fleet-Usage-Sender'; + private wasStarted: boolean = false; + private interval = '1h'; constructor( taskManager: TaskManagerSetupContract, core: CoreSetup, - fetchUsage: () => Promise, - kibanaVersion: string, - isProductionMode: boolean + fetchUsage: () => Promise ) { taskManager.registerTaskDefinitions({ [this.taskType]: { @@ -36,21 +37,11 @@ export class FleetUsageSender { maxAttempts: 1, createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { - async run() { - appContextService.getLogger().info('Running Fleet Usage telemetry send task'); - - try { - const usageData = await fetchUsage(); - appContextService.getLogger().debug(JSON.stringify(usageData)); - core.analytics.reportEvent(EVENT_TYPE, usageData); - } catch (error) { - appContextService - .getLogger() - .error('Error occurred while sending Fleet Usage telemetry: ' + error); - } + run: async () => { + return this.runTask(taskInstance, core, fetchUsage); }, - async cancel() {}, + cancel: async () => {}, }; }, }, @@ -58,20 +49,63 @@ export class FleetUsageSender { this.registerTelemetryEventType(core); } + private runTask = async ( + taskInstance: ConcreteTaskInstance, + core: CoreSetup, + fetchUsage: () => Promise + ) => { + if (!this.wasStarted) { + appContextService.getLogger().debug('[runTask()] Aborted. Task not started yet'); + return; + } + // Check that this task is current + if (taskInstance.id !== this.taskId) { + throwUnrecoverableError(new Error('Outdated task version for task: ' + taskInstance.id)); + return; + } + appContextService.getLogger().info('Running Fleet Usage telemetry send task'); + + try { + const usageData = await fetchUsage(); + appContextService.getLogger().debug(JSON.stringify(usageData)); + core.analytics.reportEvent(EVENT_TYPE, usageData); + } catch (error) { + appContextService + .getLogger() + .error('Error occurred while sending Fleet Usage telemetry: ' + error); + } + }; + + private get taskId() { + return `${this.taskType}-${this.taskVersion}`; + } + public async start(taskManager: TaskManagerStartContract) { this.taskManager = taskManager; - appContextService.getLogger().info(`Task ${this.taskId} scheduled with interval 1h`); - await this.taskManager?.ensureScheduled({ - id: this.taskId, - taskType: this.taskType, - schedule: { - interval: '1h', - }, - scope: ['fleet'], - state: {}, - params: {}, - }); + if (!taskManager) { + appContextService.getLogger().error('missing required service during start'); + return; + } + + this.wasStarted = true; + + try { + appContextService.getLogger().info(`Task ${this.taskId} scheduled with interval 1h`); + + await this.taskManager.ensureScheduled({ + id: this.taskId, + taskType: this.taskType, + schedule: { + interval: this.interval, + }, + scope: ['fleet'], + state: {}, + params: {}, + }); + } catch (e) { + appContextService.getLogger().error(`Error scheduling task, received error: ${e}`); + } } /** @@ -181,6 +215,13 @@ export class FleetUsageSender { }, }, }, + agent_versions: { + type: 'array', + items: { + type: 'keyword', + _meta: { description: 'The agent versions enrolled in this deployment.' }, + }, + }, }, }); } From aa0196de7a341449456bc7008d526bf2dea96b26 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 16 Nov 2022 15:21:01 +0100 Subject: [PATCH 02/13] added fleet server config --- .../collectors/fleet_server_collector.ts | 31 +++++++++++++++++ .../fleet/server/collectors/register.ts | 3 +- .../server/services/fleet_usage_sender.ts | 34 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index 4438b23c8a285..5b769cffc8b88 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -7,6 +7,8 @@ import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; +import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../constants'; + import { packagePolicyService } from '../services'; import { getAgentStatusForAgentPolicy } from '../services/agents'; import { listFleetServerHosts } from '../services/fleet_server_host'; @@ -84,3 +86,32 @@ export const getFleetServerUsage = async ( num_host_urls: numHostsUrls, }; }; + +export const getFleetServerConfig = async ( + soClient?: SavedObjectsClient, + esClient?: ElasticsearchClient +): Promise => { + if (!soClient || !esClient) { + return {}; + } + const fleetServerHosts = await listFleetServerHosts(soClient); + const hosts = fleetServerHosts.items.map((item) => ({ + ...item, + proxy_id: (item as any).proxy_id ?? '', + })); + + const res = await packagePolicyService.list(soClient, { + page: 1, + perPage: SO_SEARCH_LIMIT, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:fleet_server`, + }); + const policies = res.items.map((item) => ({ + id: item.id, + name: item.name, + enabled: item.enabled, + policy_id: item.policy_id, + input_config: (item.inputs[0] ?? {}).compiled_input, + })); + + return { hosts, policies }; +}; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 6bc65d8bf040c..2a536dc801040 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -16,7 +16,7 @@ import type { AgentUsage } from './agent_collectors'; import { getInternalClients } from './helpers'; import { getPackageUsage } from './package_collectors'; import type { PackageUsage } from './package_collectors'; -import { getFleetServerUsage } from './fleet_server_collector'; +import { getFleetServerUsage, getFleetServerConfig } from './fleet_server_collector'; import type { FleetServerUsage } from './fleet_server_collector'; export interface Usage { @@ -34,6 +34,7 @@ export const fetchFleetUsage = async (core: CoreSetup, config: FleetConfigType) fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), agent_versions: await getAgentVersions(esClient), + fleet_server_config: await getFleetServerConfig(soClient, esClient), }; return usage; }; diff --git a/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts b/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts index 5a5d76337e12d..05d62112537cc 100644 --- a/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts +++ b/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts @@ -222,6 +222,40 @@ export class FleetUsageSender { _meta: { description: 'The agent versions enrolled in this deployment.' }, }, }, + fleet_server_config: { + properties: { + hosts: { + type: 'array', + items: { + properties: { + id: { type: 'keyword' }, + name: { type: 'keyword' }, + host_urls: { + type: 'array', + items: { + type: 'keyword', + }, + }, + is_default: { type: 'boolean' }, + is_preconfigured: { type: 'boolean' }, + proxy_id: { type: 'keyword' }, + }, + }, + }, + policies: { + type: 'array', + items: { + properties: { + id: { type: 'keyword' }, + name: { type: 'keyword' }, + enabled: { type: 'boolean' }, + policy_id: { type: 'keyword' }, + input_config: { type: 'pass_through' }, + }, + }, + }, + }, + }, }, }); } From 25e1e09dec2b8982d3f9aa56af012a5c3504952c Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 17 Nov 2022 15:56:59 +0100 Subject: [PATCH 03/13] added agent policies telemetry --- .../server/collectors/agent_collectors.ts | 34 ++- .../fleet/server/collectors/agent_policies.ts | 42 +++ .../fleet/server/collectors/register.ts | 2 + .../server/services/fleet_usage_sender.ts | 262 ------------------ x-pack/plugins/fleet/server/services/index.ts | 2 +- .../services/telemetry/fleet_usage_sender.ts | 122 ++++++++ .../services/telemetry/fleet_usages_schema.ts | 167 +++++++++++ 7 files changed, 357 insertions(+), 274 deletions(-) create mode 100644 x-pack/plugins/fleet/server/collectors/agent_policies.ts delete mode 100644 x-pack/plugins/fleet/server/services/fleet_usage_sender.ts create mode 100644 x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts create mode 100644 x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 6c8f452f45ca2..418cbf643f873 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -10,6 +10,7 @@ import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; import type { FleetConfigType } from '../../common/types'; import { AGENTS_INDEX } from '../../common'; import * as AgentService from '../services/agents'; +import { appContextService } from '../services'; export interface AgentUsage { total_enrolled: number; @@ -49,18 +50,29 @@ export const getAgentUsage = async ( }; }; -export const getAgentVersions = async (esClient?: ElasticsearchClient) => { +export const getAgentVersions = async (esClient?: ElasticsearchClient): Promise => { if (!esClient) { - return {}; + return []; } - const response = await esClient.search({ - index: AGENTS_INDEX, - size: 0, - aggs: { - versions: { - terms: { field: 'agent.version' }, + try { + const response = await esClient.search({ + index: AGENTS_INDEX, + size: 0, + aggs: { + versions: { + terms: { field: 'agent.version' }, + }, }, - }, - }); - return ((response?.aggregations?.versions as any).buckets ?? []).map((bucket: any) => bucket.key); + }); + return ((response?.aggregations?.versions as any).buckets ?? []).map( + (bucket: any) => bucket.key + ); + } catch (error) { + if (error.statusCode === 404) { + appContextService.getLogger().debug('Index .fleet-agents does not exist yet.'); + } else { + throw error; + } + return []; + } }; diff --git a/x-pack/plugins/fleet/server/collectors/agent_policies.ts b/x-pack/plugins/fleet/server/collectors/agent_policies.ts new file mode 100644 index 0000000000000..2f98556fd3b7a --- /dev/null +++ b/x-pack/plugins/fleet/server/collectors/agent_policies.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; + +import { AGENT_POLICY_INDEX } from '../../common'; +import { ES_SEARCH_LIMIT } from '../../common/constants'; + +export const getAgentPoliciesUsage = async ( + soClient?: SavedObjectsClient, + esClient?: ElasticsearchClient +): Promise => { + if (!soClient || !esClient) { + return {}; + } + + const res = await esClient.search({ + index: AGENT_POLICY_INDEX, + size: ES_SEARCH_LIMIT, + track_total_hits: true, + rest_total_hits_as_int: true, + }); + + const agentPolicies = res.hits.hits; + + const outputTypes = new Set(); + agentPolicies.forEach((item) => { + const source = (item._source as any) ?? {}; + Object.keys(source.data.outputs).forEach((output) => { + outputTypes.add(source.data.outputs[output].type); + }); + }); + + return { + count: res.hits.total, + output_types: Array.from(outputTypes), + }; +}; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 2a536dc801040..9600ffac339c5 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -18,6 +18,7 @@ import { getPackageUsage } from './package_collectors'; import type { PackageUsage } from './package_collectors'; import { getFleetServerUsage, getFleetServerConfig } from './fleet_server_collector'; import type { FleetServerUsage } from './fleet_server_collector'; +import { getAgentPoliciesUsage } from './agent_policies'; export interface Usage { agents_enabled: boolean; @@ -35,6 +36,7 @@ export const fetchFleetUsage = async (core: CoreSetup, config: FleetConfigType) packages: await getPackageUsage(soClient), agent_versions: await getAgentVersions(esClient), fleet_server_config: await getFleetServerConfig(soClient, esClient), + agent_policies: await getAgentPoliciesUsage(soClient, esClient), }; return usage; }; diff --git a/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts b/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts deleted file mode 100644 index 05d62112537cc..0000000000000 --- a/x-pack/plugins/fleet/server/services/fleet_usage_sender.ts +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { - ConcreteTaskInstance, - TaskManagerStartContract, - TaskManagerSetupContract, -} from '@kbn/task-manager-plugin/server'; -import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server'; -import type { CoreSetup } from '@kbn/core/server'; - -import type { Usage } from '../collectors/register'; - -import { appContextService } from './app_context'; - -const EVENT_TYPE = 'fleet_usage'; - -export class FleetUsageSender { - private taskManager?: TaskManagerStartContract; - private taskVersion = '1.0.0'; - private taskType = 'Fleet-Usage-Sender'; - private wasStarted: boolean = false; - private interval = '1h'; - - constructor( - taskManager: TaskManagerSetupContract, - core: CoreSetup, - fetchUsage: () => Promise - ) { - taskManager.registerTaskDefinitions({ - [this.taskType]: { - title: 'Fleet Usage Sender', - timeout: '1m', - maxAttempts: 1, - createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { - return { - run: async () => { - return this.runTask(taskInstance, core, fetchUsage); - }, - - cancel: async () => {}, - }; - }, - }, - }); - this.registerTelemetryEventType(core); - } - - private runTask = async ( - taskInstance: ConcreteTaskInstance, - core: CoreSetup, - fetchUsage: () => Promise - ) => { - if (!this.wasStarted) { - appContextService.getLogger().debug('[runTask()] Aborted. Task not started yet'); - return; - } - // Check that this task is current - if (taskInstance.id !== this.taskId) { - throwUnrecoverableError(new Error('Outdated task version for task: ' + taskInstance.id)); - return; - } - appContextService.getLogger().info('Running Fleet Usage telemetry send task'); - - try { - const usageData = await fetchUsage(); - appContextService.getLogger().debug(JSON.stringify(usageData)); - core.analytics.reportEvent(EVENT_TYPE, usageData); - } catch (error) { - appContextService - .getLogger() - .error('Error occurred while sending Fleet Usage telemetry: ' + error); - } - }; - - private get taskId() { - return `${this.taskType}-${this.taskVersion}`; - } - - public async start(taskManager: TaskManagerStartContract) { - this.taskManager = taskManager; - - if (!taskManager) { - appContextService.getLogger().error('missing required service during start'); - return; - } - - this.wasStarted = true; - - try { - appContextService.getLogger().info(`Task ${this.taskId} scheduled with interval 1h`); - - await this.taskManager.ensureScheduled({ - id: this.taskId, - taskType: this.taskType, - schedule: { - interval: this.interval, - }, - scope: ['fleet'], - state: {}, - params: {}, - }); - } catch (e) { - appContextService.getLogger().error(`Error scheduling task, received error: ${e}`); - } - } - - /** - * took schema from [here](https://github.com/elastic/kibana/blob/main/x-pack/plugins/fleet/server/collectors/register.ts#L53) and adapted to EBT format - */ - private registerTelemetryEventType(core: CoreSetup): void { - core.analytics.registerEventType({ - eventType: EVENT_TYPE, - schema: { - agents_enabled: { type: 'boolean', _meta: { description: 'agents enabled' } }, - agents: { - properties: { - total_enrolled: { - type: 'long', - _meta: { - description: 'The total number of enrolled agents, in any state', - }, - }, - healthy: { - type: 'long', - _meta: { - description: 'The total number of enrolled agents in a healthy state', - }, - }, - unhealthy: { - type: 'long', - _meta: { - description: 'The total number of enrolled agents in an unhealthy state', - }, - }, - updating: { - type: 'long', - _meta: { - description: 'The total number of enrolled agents in an updating state', - }, - }, - offline: { - type: 'long', - _meta: { - description: 'The total number of enrolled agents currently offline', - }, - }, - total_all_statuses: { - type: 'long', - _meta: { - description: 'The total number of agents in any state, both enrolled and inactive', - }, - }, - }, - }, - fleet_server: { - properties: { - total_enrolled: { - type: 'long', - _meta: { - description: 'The total number of enrolled Fleet Server agents, in any state', - }, - }, - total_all_statuses: { - type: 'long', - _meta: { - description: - 'The total number of Fleet Server agents in any state, both enrolled and inactive.', - }, - }, - healthy: { - type: 'long', - _meta: { - description: 'The total number of enrolled Fleet Server agents in a healthy state.', - }, - }, - unhealthy: { - type: 'long', - _meta: { - description: - 'The total number of enrolled Fleet Server agents in an unhealthy state', - }, - }, - updating: { - type: 'long', - _meta: { - description: - 'The total number of enrolled Fleet Server agents in an updating state', - }, - }, - offline: { - type: 'long', - _meta: { - description: 'The total number of enrolled Fleet Server agents currently offline', - }, - }, - num_host_urls: { - type: 'long', - _meta: { - description: 'The number of Fleet Server hosts configured in Fleet settings.', - }, - }, - }, - }, - packages: { - type: 'array', - items: { - properties: { - name: { type: 'keyword' }, - version: { type: 'keyword' }, - enabled: { type: 'boolean' }, - }, - }, - }, - agent_versions: { - type: 'array', - items: { - type: 'keyword', - _meta: { description: 'The agent versions enrolled in this deployment.' }, - }, - }, - fleet_server_config: { - properties: { - hosts: { - type: 'array', - items: { - properties: { - id: { type: 'keyword' }, - name: { type: 'keyword' }, - host_urls: { - type: 'array', - items: { - type: 'keyword', - }, - }, - is_default: { type: 'boolean' }, - is_preconfigured: { type: 'boolean' }, - proxy_id: { type: 'keyword' }, - }, - }, - }, - policies: { - type: 'array', - items: { - properties: { - id: { type: 'keyword' }, - name: { type: 'keyword' }, - enabled: { type: 'boolean' }, - policy_id: { type: 'keyword' }, - input_config: { type: 'pass_through' }, - }, - }, - }, - }, - }, - }, - }); - } -} diff --git a/x-pack/plugins/fleet/server/services/index.ts b/x-pack/plugins/fleet/server/services/index.ts index 6078e2696f3b9..133d5cf88d71c 100644 --- a/x-pack/plugins/fleet/server/services/index.ts +++ b/x-pack/plugins/fleet/server/services/index.ts @@ -63,4 +63,4 @@ export type { PackageService, PackageClient } from './epm'; // Fleet server policy config export { migrateSettingsToFleetServerHost } from './fleet_server_host'; -export { FleetUsageSender } from './fleet_usage_sender'; +export { FleetUsageSender } from './telemetry/fleet_usage_sender'; diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts new file mode 100644 index 0000000000000..4ec10110b3ff5 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + ConcreteTaskInstance, + TaskManagerStartContract, + TaskManagerSetupContract, +} from '@kbn/task-manager-plugin/server'; +import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server'; +import type { CoreSetup } from '@kbn/core/server'; + +import type { Usage } from '../../collectors/register'; + +import { appContextService } from '../app_context'; + +import { fleetUsagesSchema } from './fleet_usages_schema'; + +const EVENT_TYPE = 'fleet_usage'; + +export class FleetUsageSender { + private taskManager?: TaskManagerStartContract; + private taskVersion = '1.0.0'; + private taskType = 'Fleet-Usage-Sender'; + private wasStarted: boolean = false; + private interval = '1h'; + + constructor( + taskManager: TaskManagerSetupContract, + core: CoreSetup, + fetchUsage: () => Promise + ) { + taskManager.registerTaskDefinitions({ + [this.taskType]: { + title: 'Fleet Usage Sender', + timeout: '1m', + maxAttempts: 1, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + return this.runTask(taskInstance, core, fetchUsage); + }, + + cancel: async () => {}, + }; + }, + }, + }); + this.registerTelemetryEventType(core); + } + + private runTask = async ( + taskInstance: ConcreteTaskInstance, + core: CoreSetup, + fetchUsage: () => Promise + ) => { + if (!this.wasStarted) { + appContextService.getLogger().debug('[runTask()] Aborted. Task not started yet'); + return; + } + // Check that this task is current + if (taskInstance.id !== this.taskId) { + throwUnrecoverableError(new Error('Outdated task version for task: ' + taskInstance.id)); + return; + } + appContextService.getLogger().info('Running Fleet Usage telemetry send task'); + + try { + const usageData = await fetchUsage(); + appContextService.getLogger().debug(JSON.stringify(usageData)); + core.analytics.reportEvent(EVENT_TYPE, usageData); + } catch (error) { + appContextService + .getLogger() + .error('Error occurred while sending Fleet Usage telemetry: ' + error); + } + }; + + private get taskId() { + return `${this.taskType}-${this.taskVersion}`; + } + + public async start(taskManager: TaskManagerStartContract) { + this.taskManager = taskManager; + + if (!taskManager) { + appContextService.getLogger().error('missing required service during start'); + return; + } + + this.wasStarted = true; + + try { + appContextService.getLogger().info(`Task ${this.taskId} scheduled with interval 1h`); + + await this.taskManager.ensureScheduled({ + id: this.taskId, + taskType: this.taskType, + schedule: { + interval: this.interval, + }, + scope: ['fleet'], + state: {}, + params: {}, + }); + } catch (e) { + appContextService.getLogger().error(`Error scheduling task, received error: ${e}`); + } + } + + /** + * took schema from [here](https://github.com/elastic/kibana/blob/main/x-pack/plugins/fleet/server/collectors/register.ts#L53) and adapted to EBT format + */ + private registerTelemetryEventType(core: CoreSetup): void { + core.analytics.registerEventType({ + eventType: EVENT_TYPE, + schema: fleetUsagesSchema, + }); + } +} diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts new file mode 100644 index 0000000000000..306fcd8af6f1b --- /dev/null +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RootSchema } from '@kbn/analytics-client'; + +export const fleetUsagesSchema: RootSchema = { + agents_enabled: { type: 'boolean', _meta: { description: 'agents enabled' } }, + agents: { + properties: { + total_enrolled: { + type: 'long', + _meta: { + description: 'The total number of enrolled agents, in any state', + }, + }, + healthy: { + type: 'long', + _meta: { + description: 'The total number of enrolled agents in a healthy state', + }, + }, + unhealthy: { + type: 'long', + _meta: { + description: 'The total number of enrolled agents in an unhealthy state', + }, + }, + updating: { + type: 'long', + _meta: { + description: 'The total number of enrolled agents in an updating state', + }, + }, + offline: { + type: 'long', + _meta: { + description: 'The total number of enrolled agents currently offline', + }, + }, + total_all_statuses: { + type: 'long', + _meta: { + description: 'The total number of agents in any state, both enrolled and inactive', + }, + }, + }, + }, + fleet_server: { + properties: { + total_enrolled: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents, in any state', + }, + }, + total_all_statuses: { + type: 'long', + _meta: { + description: + 'The total number of Fleet Server agents in any state, both enrolled and inactive.', + }, + }, + healthy: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents in a healthy state.', + }, + }, + unhealthy: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents in an unhealthy state', + }, + }, + updating: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents in an updating state', + }, + }, + offline: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents currently offline', + }, + }, + num_host_urls: { + type: 'long', + _meta: { + description: 'The number of Fleet Server hosts configured in Fleet settings.', + }, + }, + }, + }, + packages: { + type: 'array', + items: { + properties: { + name: { type: 'keyword' }, + version: { type: 'keyword' }, + enabled: { type: 'boolean' }, + }, + }, + }, + agent_versions: { + type: 'array', + items: { + type: 'keyword', + _meta: { description: 'The agent versions enrolled in this deployment.' }, + }, + }, + fleet_server_config: { + properties: { + hosts: { + type: 'array', + items: { + properties: { + id: { type: 'keyword' }, + name: { type: 'keyword' }, + host_urls: { + type: 'array', + items: { + type: 'keyword', + }, + }, + is_default: { type: 'boolean' }, + is_preconfigured: { type: 'boolean' }, + proxy_id: { type: 'keyword' }, + }, + }, + }, + policies: { + type: 'array', + items: { + properties: { + id: { type: 'keyword' }, + name: { type: 'keyword' }, + enabled: { type: 'boolean' }, + policy_id: { type: 'keyword' }, + input_config: { type: 'pass_through' }, + }, + }, + }, + }, + }, + agent_policies: { + properties: { + count: { + type: 'long', + _meta: { + description: 'Number of agent policies', + }, + }, + output_types: { + type: 'array', + items: { + type: 'keyword', + _meta: { description: 'Output types of agent policies' }, + }, + }, + }, + }, +}; From 8d54d369152d49c23bcdd57bc8ea3322e295cd3a Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 17 Nov 2022 17:22:42 +0100 Subject: [PATCH 04/13] added last checkin status --- .../server/collectors/agent_collectors.ts | 35 ++++++++++++++++--- .../fleet/server/collectors/register.ts | 4 +-- .../services/telemetry/fleet_usages_schema.ts | 16 +++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 418cbf643f873..0460a36628b6d 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -50,9 +50,22 @@ export const getAgentUsage = async ( }; }; -export const getAgentVersions = async (esClient?: ElasticsearchClient): Promise => { +export interface AgentData { + agent_versions: string[]; + agent_last_checkin_status: { + error: number; + degraded: number; + }; +} + +const DEFAULT_AGENT_DATA = { + agent_versions: [], + agent_last_checkin_status: { error: 0, degraded: 0 }, +}; + +export const getAgentData = async (esClient?: ElasticsearchClient): Promise => { if (!esClient) { - return []; + return DEFAULT_AGENT_DATA; } try { const response = await esClient.search({ @@ -62,17 +75,31 @@ export const getAgentVersions = async (esClient?: ElasticsearchClient): Promise< versions: { terms: { field: 'agent.version' }, }, + last_checkin_status: { + terms: { field: 'last_checkin_status' }, + }, }, }); - return ((response?.aggregations?.versions as any).buckets ?? []).map( + const versions = ((response?.aggregations?.versions as any).buckets ?? []).map( (bucket: any) => bucket.key ); + const statuses = ((response?.aggregations?.last_checkin_status as any).buckets ?? []).reduce( + (acc: any, bucket: any) => { + if (acc[bucket.key]) acc[bucket.key] = bucket.doc_count; + return acc; + }, + { error: 0, degraded: 0 } + ); + return { + agent_versions: versions, + agent_last_checkin_status: statuses, + }; } catch (error) { if (error.statusCode === 404) { appContextService.getLogger().debug('Index .fleet-agents does not exist yet.'); } else { throw error; } - return []; + return DEFAULT_AGENT_DATA; } }; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 9600ffac339c5..e1c7406abb1d5 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -11,7 +11,7 @@ import type { CoreSetup } from '@kbn/core/server'; import type { FleetConfigType } from '..'; import { getIsAgentsEnabled } from './config_collectors'; -import { getAgentUsage, getAgentVersions } from './agent_collectors'; +import { getAgentUsage, getAgentData } from './agent_collectors'; import type { AgentUsage } from './agent_collectors'; import { getInternalClients } from './helpers'; import { getPackageUsage } from './package_collectors'; @@ -34,7 +34,7 @@ export const fetchFleetUsage = async (core: CoreSetup, config: FleetConfigType) agents: await getAgentUsage(config, soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), - agent_versions: await getAgentVersions(esClient), + ...(await getAgentData(esClient)), fleet_server_config: await getFleetServerConfig(soClient, esClient), agent_policies: await getAgentPoliciesUsage(soClient, esClient), }; diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index 306fcd8af6f1b..664f024912307 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -164,4 +164,20 @@ export const fleetUsagesSchema: RootSchema = { }, }, }, + agent_last_checkin_status: { + properties: { + error: { + type: 'long', + _meta: { + description: 'Count of agent last checkin status error', + }, + }, + degraded: { + type: 'long', + _meta: { + description: 'Count of agent last checkin status degraded', + }, + }, + }, + }, }; From cefd8487cf81df30fc0506405607fdc527a7fca6 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Thu, 17 Nov 2022 17:29:01 +0100 Subject: [PATCH 05/13] added last checkin status - fix --- x-pack/plugins/fleet/server/collectors/agent_collectors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 0460a36628b6d..cf194ca62b076 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -85,7 +85,7 @@ export const getAgentData = async (esClient?: ElasticsearchClient): Promise { - if (acc[bucket.key]) acc[bucket.key] = bucket.doc_count; + if (acc[bucket.key] !== undefined) acc[bucket.key] = bucket.doc_count; return acc; }, { error: 0, degraded: 0 } From d95598a7ed623f636521c834584f400a7f6bb99f Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Fri, 18 Nov 2022 12:47:43 +0100 Subject: [PATCH 06/13] added last checkin status for last 1h --- .../server/collectors/agent_collectors.ts | 58 ++++++++++++++++--- .../services/telemetry/fleet_usages_schema.ts | 18 +++++- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index cf194ca62b076..daeda1d4dd5d9 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -52,7 +52,11 @@ export const getAgentUsage = async ( export interface AgentData { agent_versions: string[]; - agent_last_checkin_status: { + agent_checkin_status: { + error: number; + degraded: number; + }; + agent_checkin_status_last_1h: { error: number; degraded: number; }; @@ -60,7 +64,8 @@ export interface AgentData { const DEFAULT_AGENT_DATA = { agent_versions: [], - agent_last_checkin_status: { error: 0, degraded: 0 }, + agent_checkin_status: { error: 0, degraded: 0 }, + agent_checkin_status_last_1h: { error: 0, degraded: 0 }, }; export const getAgentData = async (esClient?: ElasticsearchClient): Promise => { @@ -68,6 +73,14 @@ export const getAgentData = async (esClient?: ElasticsearchClient): Promise + ((resp?.aggregations?.last_checkin_status as any).buckets ?? []).reduce( + (acc: any, bucket: any) => { + if (acc[bucket.key] !== undefined) acc[bucket.key] = bucket.doc_count; + return acc; + }, + { error: 0, degraded: 0 } + ); const response = await esClient.search({ index: AGENTS_INDEX, size: 0, @@ -83,16 +96,43 @@ export const getAgentData = async (esClient?: ElasticsearchClient): Promise bucket.key ); - const statuses = ((response?.aggregations?.last_checkin_status as any).buckets ?? []).reduce( - (acc: any, bucket: any) => { - if (acc[bucket.key] !== undefined) acc[bucket.key] = bucket.doc_count; - return acc; + const statuses = transformLastCheckinStatusBuckets(response); + + const responseLast1h = await esClient.search({ + index: AGENTS_INDEX, + size: 0, + query: { + bool: { + filter: [ + { + bool: { + must: [ + { + range: { + last_checkin: { + gte: 'now-1h/h', + lt: 'now/h', + }, + }, + }, + ], + }, + }, + ], + }, }, - { error: 0, degraded: 0 } - ); + aggs: { + last_checkin_status: { + terms: { field: 'last_checkin_status' }, + }, + }, + }); + const statusesLast1h = transformLastCheckinStatusBuckets(responseLast1h); + return { agent_versions: versions, - agent_last_checkin_status: statuses, + agent_checkin_status: statuses, + agent_checkin_status_last_1h: statusesLast1h, }; } catch (error) { if (error.statusCode === 404) { diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index 664f024912307..c86dde0988686 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -164,7 +164,23 @@ export const fleetUsagesSchema: RootSchema = { }, }, }, - agent_last_checkin_status: { + agent_checkin_status: { + properties: { + error: { + type: 'long', + _meta: { + description: 'Count of agent last checkin status error', + }, + }, + degraded: { + type: 'long', + _meta: { + description: 'Count of agent last checkin status degraded', + }, + }, + }, + }, + agent_checkin_status_last_1h: { properties: { error: { type: 'long', From 4c5f29e1b54110414617fc0c759b5b5fe20058c3 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Fri, 18 Nov 2022 16:19:54 +0100 Subject: [PATCH 07/13] added abortController, apm span --- .../server/collectors/agent_collectors.ts | 80 ++++++++++--------- .../fleet/server/collectors/agent_policies.ts | 25 +++--- .../collectors/fleet_server_collector.ts | 8 +- .../fleet/server/collectors/register.ts | 15 +++- x-pack/plugins/fleet/server/plugin.ts | 3 +- .../services/telemetry/fleet_usage_sender.ts | 20 +++-- 6 files changed, 84 insertions(+), 67 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index daeda1d4dd5d9..0f960df4545ba 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -68,10 +68,10 @@ const DEFAULT_AGENT_DATA = { agent_checkin_status_last_1h: { error: 0, degraded: 0 }, }; -export const getAgentData = async (esClient?: ElasticsearchClient): Promise => { - if (!esClient) { - return DEFAULT_AGENT_DATA; - } +export const getAgentData = async ( + esClient: ElasticsearchClient, + abortController: AbortController +): Promise => { try { const transformLastCheckinStatusBuckets = (resp: any) => ((resp?.aggregations?.last_checkin_status as any).buckets ?? []).reduce( @@ -81,52 +81,58 @@ export const getAgentData = async (esClient?: ElasticsearchClient): Promise bucket.key ); const statuses = transformLastCheckinStatusBuckets(response); - const responseLast1h = await esClient.search({ - index: AGENTS_INDEX, - size: 0, - query: { - bool: { - filter: [ - { - bool: { - must: [ - { - range: { - last_checkin: { - gte: 'now-1h/h', - lt: 'now/h', + const responseLast1h = await esClient.search( + { + index: AGENTS_INDEX, + size: 0, + query: { + bool: { + filter: [ + { + bool: { + must: [ + { + range: { + last_checkin: { + gte: 'now-1h/h', + lt: 'now/h', + }, }, }, - }, - ], + ], + }, }, - }, - ], + ], + }, }, - }, - aggs: { - last_checkin_status: { - terms: { field: 'last_checkin_status' }, + aggs: { + last_checkin_status: { + terms: { field: 'last_checkin_status' }, + }, }, }, - }); + { signal: abortController.signal } + ); const statusesLast1h = transformLastCheckinStatusBuckets(responseLast1h); return { diff --git a/x-pack/plugins/fleet/server/collectors/agent_policies.ts b/x-pack/plugins/fleet/server/collectors/agent_policies.ts index 2f98556fd3b7a..10c8daba37a24 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_policies.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_policies.ts @@ -5,25 +5,24 @@ * 2.0. */ -import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; +import type { ElasticsearchClient } from '@kbn/core/server'; import { AGENT_POLICY_INDEX } from '../../common'; import { ES_SEARCH_LIMIT } from '../../common/constants'; export const getAgentPoliciesUsage = async ( - soClient?: SavedObjectsClient, - esClient?: ElasticsearchClient + esClient: ElasticsearchClient, + abortController: AbortController ): Promise => { - if (!soClient || !esClient) { - return {}; - } - - const res = await esClient.search({ - index: AGENT_POLICY_INDEX, - size: ES_SEARCH_LIMIT, - track_total_hits: true, - rest_total_hits_as_int: true, - }); + const res = await esClient.search( + { + index: AGENT_POLICY_INDEX, + size: ES_SEARCH_LIMIT, + track_total_hits: true, + rest_total_hits_as_int: true, + }, + { signal: abortController.signal } + ); const agentPolicies = res.hits.hits; diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index 5b769cffc8b88..f3254ceee7a6d 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -87,13 +87,7 @@ export const getFleetServerUsage = async ( }; }; -export const getFleetServerConfig = async ( - soClient?: SavedObjectsClient, - esClient?: ElasticsearchClient -): Promise => { - if (!soClient || !esClient) { - return {}; - } +export const getFleetServerConfig = async (soClient: SavedObjectsClient): Promise => { const fleetServerHosts = await listFleetServerHosts(soClient); const hosts = fleetServerHosts.items.map((item) => ({ ...item, diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index e1c7406abb1d5..051bb902c5479 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -27,16 +27,23 @@ export interface Usage { fleet_server: FleetServerUsage; } -export const fetchFleetUsage = async (core: CoreSetup, config: FleetConfigType) => { +export const fetchFleetUsage = async ( + core: CoreSetup, + config: FleetConfigType, + abortController: AbortController +) => { const [soClient, esClient] = await getInternalClients(core); + if (!soClient || !esClient) { + return; + } const usage = { agents_enabled: getIsAgentsEnabled(config), agents: await getAgentUsage(config, soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), - ...(await getAgentData(esClient)), - fleet_server_config: await getFleetServerConfig(soClient, esClient), - agent_policies: await getAgentPoliciesUsage(soClient, esClient), + ...(await getAgentData(esClient, abortController)), + fleet_server_config: await getFleetServerConfig(soClient), + agent_policies: await getAgentPoliciesUsage(esClient, abortController), }; return usage; }; diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index d6591ce0b11a6..cadb7859cc2e5 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -374,7 +374,8 @@ export class FleetPlugin // Register usage collection registerFleetUsageCollector(core, config, deps.usageCollection); - const fetch = async () => fetchFleetUsage(core, config); + const fetch = async (abortController: AbortController) => + await fetchFleetUsage(core, config, abortController); this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch); registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config)); diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts index 4ec10110b3ff5..6e788e58fde57 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts @@ -11,6 +11,7 @@ import type { } from '@kbn/task-manager-plugin/server'; import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server'; import type { CoreSetup } from '@kbn/core/server'; +import { withSpan } from '@kbn/apm-utils'; import type { Usage } from '../../collectors/register'; @@ -26,24 +27,30 @@ export class FleetUsageSender { private taskType = 'Fleet-Usage-Sender'; private wasStarted: boolean = false; private interval = '1h'; + private timeout = '1m'; + private abortController = new AbortController(); constructor( taskManager: TaskManagerSetupContract, core: CoreSetup, - fetchUsage: () => Promise + fetchUsage: (abortController: AbortController) => Promise ) { taskManager.registerTaskDefinitions({ [this.taskType]: { title: 'Fleet Usage Sender', - timeout: '1m', + timeout: this.timeout, maxAttempts: 1, createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - return this.runTask(taskInstance, core, fetchUsage); + return withSpan({ name: this.taskType, type: 'telemetry' }, () => + this.runTask(taskInstance, core, () => fetchUsage(this.abortController)) + ); }, - cancel: async () => {}, + cancel: async () => { + this.abortController.abort('task timed out'); + }, }; }, }, @@ -54,7 +61,7 @@ export class FleetUsageSender { private runTask = async ( taskInstance: ConcreteTaskInstance, core: CoreSetup, - fetchUsage: () => Promise + fetchUsage: () => Promise ) => { if (!this.wasStarted) { appContextService.getLogger().debug('[runTask()] Aborted. Task not started yet'); @@ -69,6 +76,9 @@ export class FleetUsageSender { try { const usageData = await fetchUsage(); + if (!usageData) { + return; + } appContextService.getLogger().debug(JSON.stringify(usageData)); core.analytics.reportEvent(EVENT_TYPE, usageData); } catch (error) { From b29541b1c5f0fc2621abdcf0b8f904ce46f95276 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 21 Nov 2022 10:08:10 +0100 Subject: [PATCH 08/13] removed fleet server hosts info --- .../collectors/fleet_server_collector.ts | 12 +--------- .../services/telemetry/fleet_usages_schema.ts | 22 ------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index f3254ceee7a6d..7d49089d02309 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -88,24 +88,14 @@ export const getFleetServerUsage = async ( }; export const getFleetServerConfig = async (soClient: SavedObjectsClient): Promise => { - const fleetServerHosts = await listFleetServerHosts(soClient); - const hosts = fleetServerHosts.items.map((item) => ({ - ...item, - proxy_id: (item as any).proxy_id ?? '', - })); - const res = await packagePolicyService.list(soClient, { page: 1, perPage: SO_SEARCH_LIMIT, kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:fleet_server`, }); const policies = res.items.map((item) => ({ - id: item.id, - name: item.name, - enabled: item.enabled, - policy_id: item.policy_id, input_config: (item.inputs[0] ?? {}).compiled_input, })); - return { hosts, policies }; + return { policies }; }; diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index c86dde0988686..959510e6f7faa 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -115,32 +115,10 @@ export const fleetUsagesSchema: RootSchema = { }, fleet_server_config: { properties: { - hosts: { - type: 'array', - items: { - properties: { - id: { type: 'keyword' }, - name: { type: 'keyword' }, - host_urls: { - type: 'array', - items: { - type: 'keyword', - }, - }, - is_default: { type: 'boolean' }, - is_preconfigured: { type: 'boolean' }, - proxy_id: { type: 'keyword' }, - }, - }, - }, policies: { type: 'array', items: { properties: { - id: { type: 'keyword' }, - name: { type: 'keyword' }, - enabled: { type: 'boolean' }, - policy_id: { type: 'keyword' }, input_config: { type: 'pass_through' }, }, }, From 6c6de5758e74461706286ed210ad2f06d34d35b0 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 21 Nov 2022 10:23:26 +0100 Subject: [PATCH 09/13] removed host port from policy config, added try catch --- .../fleet/server/collectors/agent_policies.ts | 66 ++++++++++++------- .../collectors/fleet_server_collector.ts | 11 +++- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_policies.ts b/x-pack/plugins/fleet/server/collectors/agent_policies.ts index 10c8daba37a24..95c840536b6e5 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_policies.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_policies.ts @@ -9,33 +9,53 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { AGENT_POLICY_INDEX } from '../../common'; import { ES_SEARCH_LIMIT } from '../../common/constants'; +import { appContextService } from '../services'; + +interface AgentPoliciesUsage { + count: number; + output_types: string[]; +} + +const DEFAULT_AGENT_POLICIES_USAGE = { + count: 0, + output_types: [], +}; export const getAgentPoliciesUsage = async ( esClient: ElasticsearchClient, abortController: AbortController -): Promise => { - const res = await esClient.search( - { - index: AGENT_POLICY_INDEX, - size: ES_SEARCH_LIMIT, - track_total_hits: true, - rest_total_hits_as_int: true, - }, - { signal: abortController.signal } - ); - - const agentPolicies = res.hits.hits; - - const outputTypes = new Set(); - agentPolicies.forEach((item) => { - const source = (item._source as any) ?? {}; - Object.keys(source.data.outputs).forEach((output) => { - outputTypes.add(source.data.outputs[output].type); +): Promise => { + try { + const res = await esClient.search( + { + index: AGENT_POLICY_INDEX, + size: ES_SEARCH_LIMIT, + track_total_hits: true, + rest_total_hits_as_int: true, + }, + { signal: abortController.signal } + ); + + const agentPolicies = res.hits.hits; + + const outputTypes = new Set(); + agentPolicies.forEach((item) => { + const source = (item._source as any) ?? {}; + Object.keys(source.data.outputs).forEach((output) => { + outputTypes.add(source.data.outputs[output].type); + }); }); - }); - return { - count: res.hits.total, - output_types: Array.from(outputTypes), - }; + return { + count: res.hits.total as number, + output_types: Array.from(outputTypes), + }; + } catch (error) { + if (error.statusCode === 404) { + appContextService.getLogger().debug('Index .fleet-policies does not exist yet.'); + } else { + throw error; + } + return DEFAULT_AGENT_POLICIES_USAGE; + } }; diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index 7d49089d02309..f48e166209edf 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -93,8 +93,17 @@ export const getFleetServerConfig = async (soClient: SavedObjectsClient): Promis perPage: SO_SEARCH_LIMIT, kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:fleet_server`, }); + const getInputConfig = (item: any) => { + let config = (item.inputs[0] ?? {}).compiled_input; + if (config.server) { + config = { ...config, server: { ...config.server } }; + delete config.server.host; + delete config.server.port; + } + return config; + }; const policies = res.items.map((item) => ({ - input_config: (item.inputs[0] ?? {}).compiled_input, + input_config: getInputConfig(item), })); return { policies }; From 43788cbf32f5df807683de24aa2b89aab6ee18d8 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Mon, 21 Nov 2022 10:49:34 +0100 Subject: [PATCH 10/13] fix checks --- x-pack/plugins/fleet/server/collectors/agent_policies.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_policies.ts b/x-pack/plugins/fleet/server/collectors/agent_policies.ts index 95c840536b6e5..bd8075b09fd06 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_policies.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_policies.ts @@ -11,7 +11,7 @@ import { AGENT_POLICY_INDEX } from '../../common'; import { ES_SEARCH_LIMIT } from '../../common/constants'; import { appContextService } from '../services'; -interface AgentPoliciesUsage { +export interface AgentPoliciesUsage { count: number; output_types: string[]; } From 99f5a61bf3ade36afe13ad0237830f53717aed81 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 22 Nov 2022 13:29:15 +0100 Subject: [PATCH 11/13] integration test, removed last1h checkin errors, added agents per policy --- .../server/collectors/agent_collectors.ts | 59 ++-- .../fleet/server/collectors/register.ts | 6 +- .../fleet_usage_telemetry.test.ts | 256 ++++++++++++++++++ .../services/telemetry/fleet_usages_schema.ts | 23 +- 4 files changed, 285 insertions(+), 59 deletions(-) create mode 100644 x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 0f960df4545ba..7861e7ac606a4 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -7,7 +7,6 @@ import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server'; -import type { FleetConfigType } from '../../common/types'; import { AGENTS_INDEX } from '../../common'; import * as AgentService from '../services/agents'; import { appContextService } from '../services'; @@ -22,7 +21,6 @@ export interface AgentUsage { } export const getAgentUsage = async ( - config: FleetConfigType, soClient?: SavedObjectsClient, esClient?: ElasticsearchClient ): Promise => { @@ -56,16 +54,13 @@ export interface AgentData { error: number; degraded: number; }; - agent_checkin_status_last_1h: { - error: number; - degraded: number; - }; + agents_per_policy: number[]; } const DEFAULT_AGENT_DATA = { agent_versions: [], agent_checkin_status: { error: 0, degraded: 0 }, - agent_checkin_status_last_1h: { error: 0, degraded: 0 }, + agents_per_policy: [], }; export const getAgentData = async ( @@ -84,6 +79,17 @@ export const getAgentData = async ( const response = await esClient.search( { index: AGENTS_INDEX, + query: { + bool: { + filter: [ + { + term: { + active: 'true', + }, + }, + ], + }, + }, size: 0, aggs: { versions: { @@ -92,6 +98,9 @@ export const getAgentData = async ( last_checkin_status: { terms: { field: 'last_checkin_status' }, }, + policies: { + terms: { field: 'policy_id' }, + }, }, }, { signal: abortController.signal } @@ -101,44 +110,14 @@ export const getAgentData = async ( ); const statuses = transformLastCheckinStatusBuckets(response); - const responseLast1h = await esClient.search( - { - index: AGENTS_INDEX, - size: 0, - query: { - bool: { - filter: [ - { - bool: { - must: [ - { - range: { - last_checkin: { - gte: 'now-1h/h', - lt: 'now/h', - }, - }, - }, - ], - }, - }, - ], - }, - }, - aggs: { - last_checkin_status: { - terms: { field: 'last_checkin_status' }, - }, - }, - }, - { signal: abortController.signal } + const agentsPerPolicy = ((response?.aggregations?.policies as any).buckets ?? []).map( + (bucket: any) => bucket.doc_count ); - const statusesLast1h = transformLastCheckinStatusBuckets(responseLast1h); return { agent_versions: versions, agent_checkin_status: statuses, - agent_checkin_status_last_1h: statusesLast1h, + agents_per_policy: agentsPerPolicy, }; } catch (error) { if (error.statusCode === 404) { diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 051bb902c5479..2892de0685e2f 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -38,7 +38,7 @@ export const fetchFleetUsage = async ( } const usage = { agents_enabled: getIsAgentsEnabled(config), - agents: await getAgentUsage(config, soClient, esClient), + agents: await getAgentUsage(soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), ...(await getAgentData(esClient, abortController)), @@ -53,7 +53,7 @@ const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => { const [soClient, esClient] = await getInternalClients(core); const usage = { agents_enabled: getIsAgentsEnabled(config), - agents: await getAgentUsage(config, soClient, esClient), + agents: await getAgentUsage(soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), }; @@ -64,7 +64,7 @@ export const fetchAgentsUsage = async (core: CoreSetup, config: FleetConfigType) const [soClient, esClient] = await getInternalClients(core); const usage = { agents_enabled: getIsAgentsEnabled(config), - agents: await getAgentUsage(config, soClient, esClient), + agents: await getAgentUsage(soClient, esClient), fleet_server: await getFleetServerUsage(soClient, esClient), }; return usage; diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts new file mode 100644 index 0000000000000..fff7291856af2 --- /dev/null +++ b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import path from 'path'; + +import * as kbnTestServer from '@kbn/core/test_helpers/kbn_server'; + +import { fetchFleetUsage } from '../collectors/register'; + +import { waitForFleetSetup } from './helpers'; + +const logFilePath = path.join(__dirname, 'logs.log'); + +describe('fleet usage telemetry', () => { + let core: any; + let esServer: kbnTestServer.TestElasticsearchUtils; + let kbnServer: kbnTestServer.TestKibanaUtils; + + const startServers = async () => { + const { startES } = kbnTestServer.createTestServers({ + adjustTimeout: (t) => jest.setTimeout(t), + settings: { + es: { + license: 'trial', + }, + kbn: {}, + }, + }); + + esServer = await startES(); + const startKibana = async () => { + const root = kbnTestServer.createRootWithCorePlugins( + { + xpack: { + fleet: { + agentPolicies: [ + { + name: 'Second preconfigured policy', + description: 'second policy', + is_default: false, + is_managed: true, + id: 'test-456789', + namespace: 'default', + monitoring_enabled: [], + package_policies: [], + }, + ], + }, + }, + logging: { + appenders: { + file: { + type: 'file', + fileName: logFilePath, + layout: { + type: 'json', + }, + }, + }, + loggers: [ + { + name: 'root', + appenders: ['file'], + }, + { + name: 'plugins.fleet', + level: 'info', + }, + ], + }, + }, + { oss: false } + ); + + await root.preboot(); + const coreSetup = await root.setup(); + const coreStart = await root.start(); + + return { + root, + coreSetup, + coreStart, + stop: async () => await root.shutdown(), + }; + }; + kbnServer = await startKibana(); + await waitForFleetSetup(kbnServer.root); + }; + + const stopServers = async () => { + if (kbnServer) { + await kbnServer.stop(); + } + + if (esServer) { + await esServer.stop(); + } + + await new Promise((res) => setTimeout(res, 10000)); + }; + + beforeAll(async () => { + await startServers(); + + const esClient = kbnServer.coreStart.elasticsearch.client.asInternalUser; + await esClient.bulk({ + index: '.fleet-agents', + body: [ + { + create: { + _id: 'agent1', + }, + }, + { + agent: { + version: '8.6.0', + }, + last_checkin_status: 'error', + last_checkin: '2022-11-21T12:26:24Z', + active: true, + policy_id: 'policy1', + }, + { + create: { + _id: 'agent2', + }, + }, + { + agent: { + version: '8.5.1', + }, + last_checkin_status: 'degraded', + last_checkin: '2022-11-21T12:27:24Z', + active: true, + policy_id: 'policy1', + }, + { + create: { + _id: 'inactive', + }, + }, + { + agent: { + version: '8.5.1', + }, + last_checkin_status: 'online', + last_checkin: '2021-11-21T12:27:24Z', + active: false, + policy_id: 'policy1', + }, + ], + refresh: 'wait_for', + }); + + await esClient.create({ + index: '.fleet-policies', + id: 'policy1', + body: { + data: { + id: 'fleet-server-policy', + outputs: { + default: { + type: 'elasticsearch', + }, + }, + }, + }, + refresh: 'wait_for', + }); + + const soClient = kbnServer.coreStart.savedObjects.createInternalRepository(); + await soClient.create('ingest-package-policies', { + name: 'fleet_server-1', + namespace: 'default', + package: { + name: 'fleet_server', + title: 'Fleet Server', + version: '1.2.0', + }, + enabled: true, + policy_id: 'fleet-server-policy', + inputs: [ + { + compiled_input: { + server: { + port: 8220, + host: '0.0.0.0', + 'limits.max_agents': 3000, + }, + 'server.runtime': 'gc_percent:20', + }, + }, + ], + }); + }); + + afterAll(async () => { + await stopServers(); + }); + + beforeEach(() => { + core = { getStartServices: jest.fn().mockResolvedValue([kbnServer.coreStart]) }; + }); + + it('should fetch usage telemetry', async () => { + const usage = await fetchFleetUsage(core, { agents: { enabled: true } }, new AbortController()); + + expect(usage).toEqual( + expect.objectContaining({ + agents_enabled: true, + agents: { + total_enrolled: 2, + healthy: 0, + unhealthy: 0, + offline: 2, + total_all_statuses: 3, + updating: 0, + }, + fleet_server: { + total_all_statuses: 0, + total_enrolled: 0, + healthy: 0, + unhealthy: 0, + offline: 0, + updating: 0, + num_host_urls: 0, + }, + packages: [ + expect.objectContaining({ + name: 'synthetics', + }), + ], + agent_versions: ['8.5.1', '8.6.0'], + agent_checkin_status: { error: 1, degraded: 1 }, + agents_per_policy: [2], + fleet_server_config: { + policies: [ + { + input_config: { + server: { + 'limits.max_agents': 3000, + }, + 'server.runtime': 'gc_percent:20', + }, + }, + ], + }, + agent_policies: { count: 3, output_types: ['elasticsearch'] }, + }) + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index 959510e6f7faa..9eeb867bd9b91 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -113,6 +113,13 @@ export const fleetUsagesSchema: RootSchema = { _meta: { description: 'The agent versions enrolled in this deployment.' }, }, }, + agents_per_policy: { + type: 'array', + items: { + type: 'long', + _meta: { description: 'Agent counts enrolled per agent policy.' }, + }, + }, fleet_server_config: { properties: { policies: { @@ -158,20 +165,4 @@ export const fleetUsagesSchema: RootSchema = { }, }, }, - agent_checkin_status_last_1h: { - properties: { - error: { - type: 'long', - _meta: { - description: 'Count of agent last checkin status error', - }, - }, - degraded: { - type: 'long', - _meta: { - description: 'Count of agent last checkin status degraded', - }, - }, - }, - }, }; From 1fa218b060141cea5e32998355caea3991c30388 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Tue, 22 Nov 2022 14:35:54 +0100 Subject: [PATCH 12/13] setting registry to localhost to prevent synthetics from installing --- .../integration_tests/fleet_usage_telemetry.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts index fff7291856af2..f05725c45bee0 100644 --- a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts @@ -19,6 +19,7 @@ describe('fleet usage telemetry', () => { let core: any; let esServer: kbnTestServer.TestElasticsearchUtils; let kbnServer: kbnTestServer.TestKibanaUtils; + const registryUrl = 'http://localhost'; const startServers = async () => { const { startES } = kbnTestServer.createTestServers({ @@ -37,6 +38,7 @@ describe('fleet usage telemetry', () => { { xpack: { fleet: { + registryUrl, agentPolicies: [ { name: 'Second preconfigured policy', @@ -229,11 +231,7 @@ describe('fleet usage telemetry', () => { updating: 0, num_host_urls: 0, }, - packages: [ - expect.objectContaining({ - name: 'synthetics', - }), - ], + packages: [], agent_versions: ['8.5.1', '8.6.0'], agent_checkin_status: { error: 1, degraded: 1 }, agents_per_policy: [2], From 241a5e917a5d7cd6fa31654e5c1c103328335658 Mon Sep 17 00:00:00 2001 From: Julia Bardi Date: Wed, 23 Nov 2022 09:08:56 +0100 Subject: [PATCH 13/13] whitelist fleet server config to only server limits, timeouts and runtime --- .../collectors/fleet_server_collector.ts | 34 +++++++++++++++---- .../fleet_usage_telemetry.test.ts | 2 ++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index f48e166209edf..4d587e78563f6 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -94,13 +94,35 @@ export const getFleetServerConfig = async (soClient: SavedObjectsClient): Promis kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:fleet_server`, }); const getInputConfig = (item: any) => { - let config = (item.inputs[0] ?? {}).compiled_input; - if (config.server) { - config = { ...config, server: { ...config.server } }; - delete config.server.host; - delete config.server.port; + const config = (item.inputs[0] ?? {}).compiled_input; + if (config?.server) { + // whitelist only server limits, timeouts and runtime, sometimes fields are coming in "server.limits" format instead of nested object + const newConfig = Object.keys(config) + .filter((key) => key.startsWith('server')) + .reduce((acc: any, curr: string) => { + if (curr === 'server') { + acc.server = {}; + Object.keys(config.server) + .filter( + (key) => + key.startsWith('limits') || + key.startsWith('timeouts') || + key.startsWith('runtime') + ) + .forEach((serverKey: string) => { + acc.server[serverKey] = config.server[serverKey]; + return acc; + }); + } else { + acc[curr] = config[curr]; + } + return acc; + }, {}); + + return newConfig; + } else { + return {}; } - return config; }; const policies = res.items.map((item) => ({ input_config: getInputConfig(item), diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts index f05725c45bee0..5197b34fc89fe 100644 --- a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts @@ -192,8 +192,10 @@ describe('fleet usage telemetry', () => { port: 8220, host: '0.0.0.0', 'limits.max_agents': 3000, + other: 'other', }, 'server.runtime': 'gc_percent:20', + ssl: 'ssl', }, }, ],