diff --git a/packages/elastic-apm-synthtrace/src/index.ts b/packages/elastic-apm-synthtrace/src/index.ts index ab6a3e3731be7..3e7a2f1d59190 100644 --- a/packages/elastic-apm-synthtrace/src/index.ts +++ b/packages/elastic-apm-synthtrace/src/index.ts @@ -9,6 +9,7 @@ export { timerange } from './lib/timerange'; export { apm } from './lib/apm'; export { stackMonitoring } from './lib/stack_monitoring'; +export { observer } from './lib/agent_config'; export { cleanWriteTargets } from './lib/utils/clean_write_targets'; export { createLogger, LogLevel } from './lib/utils/create_logger'; diff --git a/packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config.ts b/packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config.ts new file mode 100644 index 0000000000000..5ec90035141da --- /dev/null +++ b/packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { AgentConfigFields } from './agent_config_fields'; +import { Metricset } from '../apm/metricset'; + +export class AgentConfig extends Metricset { + constructor() { + super({ + 'metricset.name': 'agent_config', + agent_config_applied: 1, + }); + } + + etag(etag: string) { + this.fields['labels.etag'] = etag; + return this; + } +} diff --git a/packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config_fields.ts b/packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config_fields.ts new file mode 100644 index 0000000000000..82b0963cee6e6 --- /dev/null +++ b/packages/elastic-apm-synthtrace/src/lib/agent_config/agent_config_fields.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ApmFields } from '../apm/apm_fields'; + +export type AgentConfigFields = Pick< + ApmFields, + | '@timestamp' + | 'processor.event' + | 'processor.name' + | 'metricset.name' + | 'observer' + | 'ecs.version' + | 'event.ingested' +> & + Partial<{ + 'labels.etag': string; + agent_config_applied: number; + 'event.agent_id_status': string; + }>; diff --git a/packages/elastic-apm-synthtrace/src/lib/agent_config/index.ts b/packages/elastic-apm-synthtrace/src/lib/agent_config/index.ts new file mode 100644 index 0000000000000..204a12386b275 --- /dev/null +++ b/packages/elastic-apm-synthtrace/src/lib/agent_config/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { observer } from './observer'; diff --git a/packages/elastic-apm-synthtrace/src/lib/agent_config/observer.ts b/packages/elastic-apm-synthtrace/src/lib/agent_config/observer.ts new file mode 100644 index 0000000000000..189f3f62abb39 --- /dev/null +++ b/packages/elastic-apm-synthtrace/src/lib/agent_config/observer.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { AgentConfigFields } from './agent_config_fields'; +import { AgentConfig } from './agent_config'; +import { Entity } from '../entity'; + +export class Observer extends Entity { + agentConfig() { + return new AgentConfig(); + } +} + +export function observer() { + return new Observer({}); +} diff --git a/packages/elastic-apm-synthtrace/src/lib/apm/instance.ts b/packages/elastic-apm-synthtrace/src/lib/apm/instance.ts index 4051d7e8241da..9a7664e9518ce 100644 --- a/packages/elastic-apm-synthtrace/src/lib/apm/instance.ts +++ b/packages/elastic-apm-synthtrace/src/lib/apm/instance.ts @@ -45,7 +45,7 @@ export class Instance extends Entity { } appMetrics(metrics: ApmApplicationMetricFields) { - return new Metricset({ + return new Metricset({ ...this.fields, 'metricset.name': 'app', ...metrics, diff --git a/packages/elastic-apm-synthtrace/src/lib/apm/metricset.ts b/packages/elastic-apm-synthtrace/src/lib/apm/metricset.ts index 88177e816a852..515af829c6a5a 100644 --- a/packages/elastic-apm-synthtrace/src/lib/apm/metricset.ts +++ b/packages/elastic-apm-synthtrace/src/lib/apm/metricset.ts @@ -7,10 +7,10 @@ */ import { Serializable } from '../serializable'; -import { ApmFields } from './apm_fields'; +import { Fields } from '../entity'; -export class Metricset extends Serializable { - constructor(fields: ApmFields) { +export class Metricset extends Serializable { + constructor(fields: TFields) { super({ 'processor.event': 'metric', 'processor.name': 'metric', diff --git a/packages/elastic-apm-synthtrace/src/lib/stream_processor.ts b/packages/elastic-apm-synthtrace/src/lib/stream_processor.ts index e1cb332996e23..a6f8f923b3714 100644 --- a/packages/elastic-apm-synthtrace/src/lib/stream_processor.ts +++ b/packages/elastic-apm-synthtrace/src/lib/stream_processor.ts @@ -211,7 +211,9 @@ export class StreamProcessor { const eventType = d.processor.event as keyof ApmElasticsearchOutputWriteTargets; let dataStream = writeTargets[eventType]; if (eventType === 'metric') { - if (!d.service?.name) { + if (d.metricset?.name === 'agent_config') { + dataStream = 'metrics-apm.internal-default'; + } else if (!d.service?.name) { dataStream = 'metrics-apm.app-default'; } else { if (!d.transaction && !d.span) { diff --git a/packages/elastic-apm-synthtrace/src/scripts/examples/04_agent_config.ts b/packages/elastic-apm-synthtrace/src/scripts/examples/04_agent_config.ts new file mode 100644 index 0000000000000..ec6d57eba4b61 --- /dev/null +++ b/packages/elastic-apm-synthtrace/src/scripts/examples/04_agent_config.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { observer, timerange } from '../..'; +import { Scenario } from '../scenario'; +import { getLogger } from '../utils/get_common_services'; +import { RunOptions } from '../utils/parse_run_cli_flags'; +import { AgentConfigFields } from '../../lib/agent_config/agent_config_fields'; + +const scenario: Scenario = async (runOptions: RunOptions) => { + const logger = getLogger(runOptions); + + return { + generate: ({ from, to }) => { + const agentConfig = observer().agentConfig(); + + const range = timerange(from, to); + return range + .interval('30s') + .rate(1) + .generator((timestamp) => { + const events = logger.perf('generating_agent_config_events', () => { + return agentConfig.etag('test-etag').timestamp(timestamp); + }); + return events; + }); + }, + }; +}; + +export default scenario; diff --git a/x-pack/plugins/apm/common/agent_configuration/configuration_types.d.ts b/x-pack/plugins/apm/common/agent_configuration/configuration_types.d.ts index 0f315c1583f1a..88302dea91200 100644 --- a/x-pack/plugins/apm/common/agent_configuration/configuration_types.d.ts +++ b/x-pack/plugins/apm/common/agent_configuration/configuration_types.d.ts @@ -15,6 +15,6 @@ export type AgentConfigurationIntake = t.TypeOf< export type AgentConfiguration = { '@timestamp': number; applied_by_agent?: boolean; - etag?: string; + etag: string; agent_name?: string; } & AgentConfigurationIntake; diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/convert_settings_to_string.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/convert_settings_to_string.ts index d52b048bc6b46..a0b3fa2e45c54 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/convert_settings_to_string.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/convert_settings_to_string.ts @@ -13,17 +13,27 @@ import { AgentConfiguration } from '../../../../common/agent_configuration/confi export function convertConfigSettingsToString( hit: SearchHit ) { - const config = hit._source; + const { settings } = hit._source; - if (config.settings?.transaction_sample_rate) { - config.settings.transaction_sample_rate = - config.settings.transaction_sample_rate.toString(); - } + const convertedConfigSettings = { + ...settings, + ...(settings?.transaction_sample_rate + ? { + transaction_sample_rate: settings.transaction_sample_rate.toString(), + } + : {}), + ...(settings?.transaction_max_spans + ? { + transaction_max_spans: settings.transaction_max_spans.toString(), + } + : {}), + }; - if (config.settings?.transaction_max_spans) { - config.settings.transaction_max_spans = - config.settings.transaction_max_spans.toString(); - } - - return hit; + return { + ...hit, + _source: { + ...hit._source, + settings: convertedConfigSettings, + }, + }; } diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts index 18e2fe0f34a6d..f32e53a1ad1dd 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts @@ -13,6 +13,7 @@ import { } from '../../../../common/elasticsearch_fieldnames'; import { Setup } from '../../../lib/helpers/setup_request'; import { convertConfigSettingsToString } from './convert_settings_to_string'; +import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet'; export async function findExactConfiguration({ service, @@ -40,16 +41,27 @@ export async function findExactConfiguration({ }, }; - const resp = await internalClient.search( - 'find_exact_agent_configuration', - params - ); + const [agentConfig, configsAppliedToAgentsThroughFleet] = await Promise.all([ + internalClient.search( + 'find_exact_agent_configuration', + params + ), + getConfigsAppliedToAgentsThroughFleet({ setup }), + ]); - const hit = resp.hits.hits[0] as SearchHit | undefined; + const hit = agentConfig.hits.hits[0] as + | SearchHit + | undefined; if (!hit) { return; } - return convertConfigSettingsToString(hit); + return { + id: hit._id, + ...convertConfigSettingsToString(hit)._source, + applied_by_agent: + hit._source.applied_by_agent || + configsAppliedToAgentsThroughFleet.hasOwnProperty(hit._source.etag), + }; } diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_config_applied_to_agent_through_fleet.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_config_applied_to_agent_through_fleet.ts new file mode 100644 index 0000000000000..351c21b43c1e9 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/get_config_applied_to_agent_through_fleet.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { termQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import datemath from '@kbn/datemath'; +import { METRICSET_NAME } from '../../../../common/elasticsearch_fieldnames'; +import { Setup } from '../../../lib/helpers/setup_request'; + +export async function getConfigsAppliedToAgentsThroughFleet({ + setup, +}: { + setup: Setup; +}) { + const { internalClient, indices } = setup; + + const params = { + index: indices.metric, + size: 0, + body: { + query: { + bool: { + filter: [ + ...termQuery(METRICSET_NAME, 'agent_config'), + ...rangeQuery( + datemath.parse('now-15m')!.valueOf(), + datemath.parse('now')!.valueOf() + ), + ], + }, + }, + aggs: { + config_by_etag: { + terms: { + field: 'labels.etag', + size: 200, + }, + }, + }, + }, + }; + + const response = await internalClient.search( + 'get_config_applied_to_agent_through_fleet', + params + ); + + return ( + response.aggregations?.config_by_etag.buckets.reduce( + (configsAppliedToAgentsThroughFleet, bucket) => { + configsAppliedToAgentsThroughFleet[bucket.key as string] = true; + return configsAppliedToAgentsThroughFleet; + }, + {} as Record + ) ?? {} + ); +} diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts index bc105106cb5e4..416cb50c0a801 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts @@ -8,6 +8,7 @@ import { Setup } from '../../../lib/helpers/setup_request'; import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { convertConfigSettingsToString } from './convert_settings_to_string'; +import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet'; export async function listConfigurations({ setup }: { setup: Setup }) { const { internalClient, indices } = setup; @@ -17,12 +18,22 @@ export async function listConfigurations({ setup }: { setup: Setup }) { size: 200, }; - const resp = await internalClient.search( - 'list_agent_configuration', - params - ); + const [agentConfigs, configsAppliedToAgentsThroughFleet] = await Promise.all([ + internalClient.search( + 'list_agent_configuration', + params + ), + getConfigsAppliedToAgentsThroughFleet({ setup }), + ]); - return resp.hits.hits + return agentConfigs.hits.hits .map(convertConfigSettingsToString) - .map((hit) => hit._source); + .map((hit) => { + return { + ...hit._source, + applied_by_agent: + hit._source.applied_by_agent || + configsAppliedToAgentsThroughFleet.hasOwnProperty(hit._source.etag), + }; + }); } diff --git a/x-pack/plugins/apm/server/routes/settings/agent_configuration/route.ts b/x-pack/plugins/apm/server/routes/settings/agent_configuration/route.ts index 72869ef165fa2..3d9abebeeef2b 100644 --- a/x-pack/plugins/apm/server/routes/settings/agent_configuration/route.ts +++ b/x-pack/plugins/apm/server/routes/settings/agent_configuration/route.ts @@ -38,7 +38,9 @@ const agentConfigurationRoute = createApmServerRoute({ >; }> => { const setup = await setupRequest(resources); + const configurations = await listConfigurations({ setup }); + return { configurations }; }, }); @@ -71,7 +73,7 @@ const getSingleAgentConfigurationRoute = createApmServerRoute({ throw Boom.notFound(); } - return config._source; + return config; }, }); @@ -102,11 +104,11 @@ const deleteAgentConfigurationRoute = createApmServerRoute({ } logger.info( - `Deleting config ${service.name}/${service.environment} (${config._id})` + `Deleting config ${service.name}/${service.environment} (${config.id})` ); const deleteConfigurationResult = await deleteConfiguration({ - configurationId: config._id, + configurationId: config.id, setup, }); @@ -162,7 +164,7 @@ const createOrUpdateAgentConfigurationRoute = createApmServerRoute({ ); await createOrUpdateConfiguration({ - configurationId: config?._id, + configurationId: config?.id, configurationIntake: body, setup, }); diff --git a/x-pack/test/apm_api_integration/tests/settings/agent_configuration/add_agent_config_metrics.ts b/x-pack/test/apm_api_integration/tests/settings/agent_configuration/add_agent_config_metrics.ts new file mode 100644 index 0000000000000..f0329a220c71a --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/settings/agent_configuration/add_agent_config_metrics.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { timerange, observer } from '@elastic/apm-synthtrace'; +import type { ApmSynthtraceEsClient } from '@elastic/apm-synthtrace'; + +export async function addAgentConfigMetrics({ + synthtraceEsClient, + start, + end, + etag, +}: { + synthtraceEsClient: ApmSynthtraceEsClient; + start: number; + end: number; + etag?: string; +}) { + const agentConfig = observer().agentConfig(); + + const agentConfigEvents = [ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => agentConfig.etag(etag ?? 'test-etag').timestamp(timestamp)), + ]; + + await synthtraceEsClient.index(agentConfigEvents); +} diff --git a/x-pack/test/apm_api_integration/tests/settings/agent_configuration.spec.ts b/x-pack/test/apm_api_integration/tests/settings/agent_configuration/agent_configuration.spec.ts similarity index 85% rename from x-pack/test/apm_api_integration/tests/settings/agent_configuration.spec.ts rename to x-pack/test/apm_api_integration/tests/settings/agent_configuration/agent_configuration.spec.ts index ecf5b87e82d70..e4960791eee5a 100644 --- a/x-pack/test/apm_api_integration/tests/settings/agent_configuration.spec.ts +++ b/x-pack/test/apm_api_integration/tests/settings/agent_configuration/agent_configuration.spec.ts @@ -11,14 +11,17 @@ import expect from '@kbn/expect'; import { omit, orderBy } from 'lodash'; import { AgentConfigurationIntake } from '@kbn/apm-plugin/common/agent_configuration/configuration_types'; import { AgentConfigSearchParams } from '@kbn/apm-plugin/server/routes/settings/agent_configuration/route'; - -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import moment from 'moment'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { addAgentConfigMetrics } from './add_agent_config_metrics'; export default function agentConfigurationTests({ getService }: FtrProviderContext) { const registry = getService('registry'); const apmApiClient = getService('apmApiClient'); const log = getService('log'); + const synthtraceEsClient = getService('synthtraceEsClient'); const archiveName = 'apm_8.0.0'; @@ -77,6 +80,18 @@ export default function agentConfigurationTests({ getService }: FtrProviderConte }); } + function findExactConfiguration(name: string, environment: string) { + return apmApiClient.readUser({ + endpoint: 'GET /api/apm/settings/agent-configuration/view', + params: { + query: { + name, + environment, + }, + }, + }); + } + registry.when( 'agent configuration when no data is loaded', { config: 'basic', archives: [] }, @@ -297,7 +312,7 @@ export default function agentConfigurationTests({ getService }: FtrProviderConte service: { name: 'myservice', environment: 'production' }, settings: { transaction_sample_rate: '0.9' }, }; - let etag: string | undefined; + let etag: string; before(async () => { log.debug('creating agent configuration'); @@ -371,6 +386,74 @@ export default function agentConfigurationTests({ getService }: FtrProviderConte } ); + registry.when( + 'Agent configurations through fleet', + { config: 'basic', archives: ['apm_mappings_only_8.0.0'] }, + () => { + const name = 'myservice'; + const environment = 'development'; + const testConfig = { + service: { name, environment }, + settings: { transaction_sample_rate: '0.9' }, + }; + + let agentConfiguration: + | APIReturnType<'GET /api/apm/settings/agent-configuration/view'> + | undefined; + + before(async () => { + log.debug('creating agent configuration'); + await createConfiguration(testConfig); + const { body } = await findExactConfiguration(name, environment); + agentConfiguration = body; + }); + + after(async () => { + await deleteConfiguration(testConfig); + }); + + it(`should have 'applied_by_agent=false' when there are no agent config metrics for this etag`, async () => { + expect(agentConfiguration?.applied_by_agent).to.be(false); + }); + + describe('when there are agent config metrics for this etag', () => { + before(async () => { + const start = new Date().getTime(); + const end = moment(start).add(15, 'minutes').valueOf(); + + await addAgentConfigMetrics({ + synthtraceEsClient, + start, + end, + etag: agentConfiguration?.etag, + }); + }); + + after(() => synthtraceEsClient.clean()); + + it(`should have 'applied_by_agent=true' when getting a config from all configurations`, async () => { + const { + body: { configurations }, + } = await getAllConfigurations(); + + const updatedConfig = configurations.find( + (x) => x.service.name === name && x.service.environment === environment + ); + + expect(updatedConfig?.applied_by_agent).to.be(true); + }); + + it(`should have 'applied_by_agent=true' when getting a single config`, async () => { + const { + body: { applied_by_agent: appliedByAgent }, + } = await findExactConfiguration(name, environment); + + expect(appliedByAgent).to.be(true); + }); + }); + } + ); + registry.when( 'agent configuration when data is loaded', { config: 'basic', archives: [archiveName] },