From 078619049e9be645891f18f7372ee7348baff547 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Thu, 9 Jul 2020 15:54:02 -0500 Subject: [PATCH] remove script; update docs; add schema --- .../__snapshots__/apm_telemetry.test.ts.snap | 2 +- .../plugins/apm/common/apm_telemetry.test.ts | 49 +++++---------- x-pack/plugins/apm/common/apm_telemetry.ts | 37 ----------- x-pack/plugins/apm/dev_docs/telemetry.md | 14 +---- .../apm/scripts/merge-telemetry-mapping.js | 21 ------- .../scripts/merge-telemetry-mapping/index.ts | 63 ------------------- .../apm/server/lib/apm_telemetry/index.ts | 12 ++-- 7 files changed, 25 insertions(+), 173 deletions(-) delete mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping.js delete mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index d08c16455f03f..59c0d3459008d 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object with the full path telemetry mapping 1`] = ` +exports[`APM telemetry helpers getApmTelemetryFullPath generates a JSON object with the telemetry mapping 1`] = ` { "properties": { "stack_stats": { diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts index ae76221c289fe..92c71f33f0d99 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.test.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -4,11 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - getApmTelemetryMapping, - getApmTelemetryMappingFullPath, - mergeApmTelemetryMapping, -} from './apm_telemetry'; +import { getApmTelemetryMapping } from './apm_telemetry'; // Add this snapshot serializer for this test. The default snapshot serializer // prints "Object" next to objects in the JSON output, but we want to be able to @@ -23,41 +19,28 @@ expect.addSnapshotSerializer({ describe('APM telemetry helpers', () => { describe('getApmTelemetryFullPath', () => { - it('generates a JSON object with the full path telemetry mapping', () => { - expect(getApmTelemetryMappingFullPath()).toMatchSnapshot(); - }); - }); - - describe('mergeApmTelemetryMapping', () => { - describe('with an invalid mapping', () => { - it('throws an error', () => { - expect(() => mergeApmTelemetryMapping({})).toThrowError(); - }); - }); - - describe('with a valid mapping', () => { - it('merges the mapping', () => { - // This is "valid" in the sense that it has all of the deep fields - // needed to merge. It's not a valid mapping opbject. - const validTelemetryMapping = { - mappings: { + // This test creates a snapshot with the JSON of our full telemetry mapping + // that can be PUT in a query to the index on the telemetry cluster. Sharing + // the contents of the snapshot with the telemetry team can provide them with + // useful information about changes to our telmetry. + it('generates a JSON object with the telemetry mapping', () => { + expect({ + properties: { + stack_stats: { properties: { - stack_stats: { + kibana: { properties: { - kibana: { - properties: { plugins: { properties: { apm: {} } } }, + plugins: { + properties: { + apm: getApmTelemetryMapping(), + }, }, }, }, }, }, - }; - - expect( - mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties - .stack_stats.properties.kibana.properties.plugins.properties.apm - ).toEqual(getApmTelemetryMapping()); - }); + }, + }).toMatchSnapshot(); }); }); }); diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 0a56923d0c14f..cfc3a75dc8984 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { produce } from 'immer'; import { AGENT_NAMES } from './agent_name'; /** @@ -222,39 +221,3 @@ export function getApmTelemetryMapping() { }, }; } - -/** - * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json) - * with the output from `getApmTelemetryMapping`. - */ -export function mergeApmTelemetryMapping( - xpackPhoneHomeMapping: Record -) { - return produce(xpackPhoneHomeMapping, (draft: Record) => { - draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping(); - return draft; - }); -} - -/** - * Create the just the mapping at its full path - */ -export function getApmTelemetryMappingFullPath() { - return { - properties: { - stack_stats: { - properties: { - kibana: { - properties: { - plugins: { - properties: { - apm: getApmTelemetryMapping(), - }, - }, - }, - }, - }, - }, - }, - }; -} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index f2815f1d1262e..92614f80f3b6c 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -55,22 +55,10 @@ The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function. -To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. +The `schema` property of the `makeUsageCollector` call in the [`createApmTelemetry` function](../server/lib/apm_telemetry/index.ts) contains the output of `getTelemetryMapping`. Pull requests with changes to the schema should automatically notify the Telemetry team so they can update the mapping in the telemetry clusters. When adding a task, the key of the task and the `took` properties need to be added under the `tasks` properties in the mapping, as when tasks run they report the time they took. -If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: - -```bash -node ./scripts/merge-telemetry-mapping.js ../../../../telemetry slug-for-my-change 7.9 -``` - -this will replace the contents of the mapping in the repository checkout with the updated mapping. - -It will also create a file in the mapping_migrations directory named "00XX-slug-for-my-change". You'll need to rename the file to replace the "XX" with the next sequential migration number based on what's there already and in open pull requests. - -You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. - The queries for the stats are in the [collect data telemetry tasks](../server/lib/apm_telemetry/collect_data_telemetry/tasks.ts). The collection tasks also use the [`APMDataTelemetry` type](../server/lib/apm_telemetry/types.ts) which also needs to be updated with any changes to the fields. diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js deleted file mode 100644 index 741df981a9cb0..0000000000000 --- a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// compile typescript on the fly -// eslint-disable-next-line import/no-extraneous-dependencies -require('@babel/register')({ - extensions: ['.ts'], - plugins: [ - '@babel/plugin-proposal-optional-chaining', - '@babel/plugin-proposal-nullish-coalescing-operator', - ], - presets: [ - '@babel/typescript', - ['@babel/preset-env', { targets: { node: 'current' } }], - ], -}); - -require('./merge-telemetry-mapping/index.ts'); diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts deleted file mode 100644 index b4d1817681cde..0000000000000 --- a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { readFileSync, truncateSync, writeFileSync } from 'fs'; -import { join, resolve } from 'path'; -import { argv } from 'yargs'; -import { - getApmTelemetryMappingFullPath, - mergeApmTelemetryMapping, -} from '../../common/apm_telemetry'; - -function errorExit(error?: Error) { - // eslint-disable-next-line no-console - console.error( - `\n usage: ${argv.$0} /path/to/telemetry/repo migration-slug stack-minor-version\n` - ); - if (error) { - throw error; - } - process.exit(1); -} - -try { - const telemetryRepoPath = argv._[0]; - const filename = resolve( - join(telemetryRepoPath, 'config/templates/xpack-phone-home.json') - ); - const slug = argv._[1]; - const stackMinorVersion = argv._[2]; - - if (!slug || !telemetryRepoPath || !stackMinorVersion) { - errorExit(); - } - - const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); - const migrationFilename = resolve( - join(telemetryRepoPath, 'mapping_migrations', `00XX-${slug}`) - ); - const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); - // The suffix for the all-xpack-phone-home index. Will be "202*" for the 2020s - const allSuffix = new Date().getFullYear().toString().replace(/.$/, '*'); - const versionQuery = { - query: { range: { 'version.minor': { gte: String(stackMinorVersion) } } }, - }; - truncateSync(filename); - writeFileSync(filename, JSON.stringify(newMapping, null, 2)); - - writeFileSync( - migrationFilename, - [ - `PUT xpack-phone-home,all-xpack-phone-home-${allSuffix}/_mapping`, - JSON.stringify(getApmTelemetryMappingFullPath(), null, 2), - '', - `POST xpack-phone-home,all-xpack-phone-home-${allSuffix}/_update_by_query?wait_for_completion=false&conflicts=proceed`, - JSON.stringify(versionQuery, null, 2), - ].join('\n') - ); -} catch (error) { - errorExit(error); -} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index 632e653a2f6e9..d04d8f106b217 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -3,25 +3,26 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup, Logger } from 'src/core/server'; import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; +import { CoreSetup, Logger } from 'src/core/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { getApmTelemetryMapping } from '../../../common/apm_telemetry'; +import { APMConfig } from '../..'; import { - TaskManagerStartContract, TaskManagerSetupContract, + TaskManagerStartContract, } from '../../../../task_manager/server'; -import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { APM_TELEMETRY_SAVED_OBJECT_ID, APM_TELEMETRY_SAVED_OBJECT_TYPE, } from '../../../common/apm_saved_object_constants'; +import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; +import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { collectDataTelemetry, CollectTelemetryParams, } from './collect_data_telemetry'; -import { APMConfig } from '../..'; -import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task'; @@ -97,6 +98,7 @@ export async function createApmTelemetry({ const collector = usageCollector.makeUsageCollector({ type: 'apm', + schema: getApmTelemetryMapping(), fetch: async () => { try { const data = (