From 71a81f7cd6bc8aec7f52d8c26b1b1aee937859e4 Mon Sep 17 00:00:00 2001 From: Ashokaditya Date: Mon, 14 Jun 2021 15:39:11 +0200 Subject: [PATCH 01/91] [Security Solution][Endpoint] Actions Log API (#101032) * WIP add tabs for endpoint details * fetch activity log for endpoint this is work in progress with dummy data * refactor to hold host details and activity log within endpointDetails * api for fetching actions log * add a selector for getting selected agent id * use the new api to show actions log * review changes * move util function to common/utils in order to use it in endpoint_hosts as well as in trusted _apps review suggestion * use util function to get API path review suggestion * sync url params with details active tab review suggestion * fix types due to merge commit refs 3722552f739f74d3c457e8ed6cf80444aa6dfd06 * use AsyncResourseState type review suggestions * sort entries chronologically with recent at the top * adjust icon sizes within entries to match mocks * remove endpoint list paging stuff (not for now) * fix import after sync with master * make the search bar work (sort of) this needs to be fleshed out in a later PR * add tests to middleware for now * use snake case for naming routes review changes * rename and use own relative time function review change * use euiTheme tokens review change * add a comment review changes * log errors to kibana log and unwind stack review changes * search on two indices * fix types * use modified data * distinguish between responses and actions and respective states in UI * use indices explicitly and tune the query * fix types after sync with master * fix lint * do better types review suggestion * add paging to API call * add paging info to redux store for activityLog * decouple paging action from other API requests * use a button for now to fetch more data * add index to fleet indices else we get a type check error about the constant not being exported correctly from `x-pack/plugins/fleet/common/constants/agent` * add tests for audit log API * do semantic paging from first request * fix ts error review changes * add document id and total to API review suggestions * update test * update frontend to consume the modified api correctly * update mock * rename action review changes * wrap mock into function to create anew on each test review changes * wrap with schema.maybe and increase page size review changes * ignore 404 review changes * use i18n review changes * abstract logEntry component logic review changes * move handler logic to a service review changes * update response object review changes * fix paging to use 50 as initial fetch size * fix translations and move custom hook to component file review changes * add return type review changes * update default value for page_size review changes * remove default values review changes https://github.com/elastic/kibana/tree/master/packages/kbn-config-schema#schemamaybe https://github.com/elastic/kibana/tree/master/packages/kbn-config-schema#default-values * fix mock data refs 1f9ae7019498c616455f1109a524741e212106ca * add selectors for data review changes Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/fleet/common/constants/agent.ts | 1 + .../plugins/fleet/common/constants/index.ts | 1 + .../common/endpoint/schema/actions.ts | 11 +- .../common/endpoint/types/actions.ts | 26 +++ .../pages/endpoint_hosts/store/action.ts | 10 +- .../pages/endpoint_hosts/store/builders.ts | 6 +- .../pages/endpoint_hosts/store/index.test.ts | 4 +- .../endpoint_hosts/store/middleware.test.ts | 45 +++- .../pages/endpoint_hosts/store/middleware.ts | 70 +++++- .../pages/endpoint_hosts/store/reducer.ts | 30 ++- .../pages/endpoint_hosts/store/selectors.ts | 28 ++- .../management/pages/endpoint_hosts/types.ts | 8 +- .../view/details/components/log_entry.tsx | 199 ++++++++++++++---- .../components/log_entry_timeline_icon.tsx | 42 ++++ .../view/details/endpoint_activity_log.tsx | 49 ++++- .../view/details/endpoints.stories.tsx | 142 ++++++++----- .../endpoint_hosts/view/details/index.tsx | 12 +- .../pages/endpoint_hosts/view/translations.ts | 50 ++++- .../endpoint/routes/actions/audit_log.test.ts | 186 ++++++++++++++++ .../routes/actions/audit_log_handler.ts | 49 ++--- .../server/endpoint/routes/actions/mocks.ts | 23 ++ .../server/endpoint/routes/actions/service.ts | 110 ++++++++++ 22 files changed, 919 insertions(+), 183 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry_timeline_icon.tsx create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/actions/service.ts diff --git a/x-pack/plugins/fleet/common/constants/agent.ts b/x-pack/plugins/fleet/common/constants/agent.ts index 6d85f658f2240..e38b7a6b5832b 100644 --- a/x-pack/plugins/fleet/common/constants/agent.ts +++ b/x-pack/plugins/fleet/common/constants/agent.ts @@ -25,3 +25,4 @@ export const AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL = 5; export const AGENTS_INDEX = '.fleet-agents'; export const AGENT_ACTIONS_INDEX = '.fleet-actions'; +export const AGENT_ACTIONS_RESULTS_INDEX = '.fleet-actions-results'; diff --git a/x-pack/plugins/fleet/common/constants/index.ts b/x-pack/plugins/fleet/common/constants/index.ts index e3001542e3e6f..ef8cb63f132b4 100644 --- a/x-pack/plugins/fleet/common/constants/index.ts +++ b/x-pack/plugins/fleet/common/constants/index.ts @@ -31,6 +31,7 @@ export const FLEET_SERVER_SERVERS_INDEX = '.fleet-servers'; export const FLEET_SERVER_INDICES = [ '.fleet-actions', + '.fleet-actions-results', '.fleet-agents', FLEET_SERVER_ARTIFACTS_INDEX, '.fleet-enrollment-api-keys', diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts b/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts index 09776b57ed8ea..f58dd1f3370d4 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/actions.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { schema } from '@kbn/config-schema'; +import { schema, TypeOf } from '@kbn/config-schema'; export const HostIsolationRequestSchema = { body: schema.object({ @@ -22,13 +22,18 @@ export const HostIsolationRequestSchema = { }; export const EndpointActionLogRequestSchema = { - // TODO improve when using pagination with query params - query: schema.object({}), + query: schema.object({ + page: schema.number({ defaultValue: 1, min: 1 }), + page_size: schema.number({ defaultValue: 10, min: 1, max: 100 }), + }), params: schema.object({ agent_id: schema.string(), }), }; +export type EndpointActionLogRequestParams = TypeOf; +export type EndpointActionLogRequestQuery = TypeOf; + export const ActionStatusRequestSchema = { query: schema.object({ agent_ids: schema.oneOf([ diff --git a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts index 937025f35dadc..99753242e7627 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts @@ -38,6 +38,32 @@ export interface EndpointActionResponse { action_data: EndpointActionData; } +export interface ActivityLogAction { + type: 'action'; + item: { + // document _id + id: string; + // document _source + data: EndpointAction; + }; +} +export interface ActivityLogActionResponse { + type: 'response'; + item: { + // document id + id: string; + // document _source + data: EndpointActionResponse; + }; +} +export type ActivityLogEntry = ActivityLogAction | ActivityLogActionResponse; +export interface ActivityLog { + total: number; + page: number; + pageSize: number; + data: ActivityLogEntry[]; +} + export type HostIsolationRequestBody = TypeOf; export interface HostIsolationResponse { diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts index 178f27caa1085..a8885b904d9c8 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts @@ -148,8 +148,15 @@ export type EndpointIsolationRequestStateChange = Action<'endpointIsolationReque payload: EndpointState['isolationRequestState']; }; +export interface AppRequestedEndpointActivityLog { + type: 'appRequestedEndpointActivityLog'; + payload: { + page: number; + pageSize: number; + }; +} export type EndpointDetailsActivityLogChanged = Action<'endpointDetailsActivityLogChanged'> & { - payload: EndpointState['endpointDetails']['activityLog']; + payload: EndpointState['endpointDetails']['activityLog']['logData']; }; export type EndpointAction = @@ -157,6 +164,7 @@ export type EndpointAction = | ServerFailedToReturnEndpointList | ServerReturnedEndpointDetails | ServerFailedToReturnEndpointDetails + | AppRequestedEndpointActivityLog | EndpointDetailsActivityLogChanged | ServerReturnedEndpointPolicyResponse | ServerFailedToReturnEndpointPolicyResponse diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/builders.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/builders.ts index d5416d9f8ec96..273b4279851fd 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/builders.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/builders.ts @@ -19,7 +19,11 @@ export const initialEndpointPageState = (): Immutable => { loading: false, error: undefined, endpointDetails: { - activityLog: createUninitialisedResourceState(), + activityLog: { + page: 1, + pageSize: 50, + logData: createUninitialisedResourceState(), + }, hostDetails: { details: undefined, detailsLoading: false, diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts index 5be67a3581c9e..455c6538bcdf2 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts @@ -43,7 +43,9 @@ describe('EndpointList store concerns', () => { error: undefined, endpointDetails: { activityLog: { - type: 'UninitialisedResourceState', + page: 1, + pageSize: 50, + logData: { type: 'UninitialisedResourceState' }, }, hostDetails: { details: undefined, diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts index 98ef5a341ac9e..130f8a56fd026 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts @@ -18,7 +18,7 @@ import { Immutable, HostResultList, HostIsolationResponse, - EndpointAction, + ActivityLog, ISOLATION_ACTIONS, } from '../../../../../common/endpoint/types'; import { AppAction } from '../../../../common/store/actions'; @@ -233,16 +233,40 @@ describe('endpoint list middleware', () => { }, }); }; + const fleetActionGenerator = new FleetActionGenerator(Math.random().toString()); - const activityLog = [ - fleetActionGenerator.generate({ - agents: [endpointList.hosts[0].metadata.agent.id], - }), - ]; + const actionData = fleetActionGenerator.generate({ + agents: [endpointList.hosts[0].metadata.agent.id], + }); + const responseData = fleetActionGenerator.generateResponse({ + agent_id: endpointList.hosts[0].metadata.agent.id, + }); + const getMockEndpointActivityLog = () => + ({ + total: 2, + page: 1, + pageSize: 50, + data: [ + { + type: 'response', + item: { + id: '', + data: responseData, + }, + }, + { + type: 'action', + item: { + id: '', + data: actionData, + }, + }, + ], + } as ActivityLog); const dispatchGetActivityLog = () => { dispatch({ type: 'endpointDetailsActivityLogChanged', - payload: createLoadedResourceState(activityLog), + payload: createLoadedResourceState(getMockEndpointActivityLog()), }); }; @@ -270,11 +294,10 @@ describe('endpoint list middleware', () => { dispatchGetActivityLog(); const loadedDispatchedResponse = await loadedDispatched; - const activityLogData = (loadedDispatchedResponse.payload as LoadedResourceState< - EndpointAction[] - >).data; + const activityLogData = (loadedDispatchedResponse.payload as LoadedResourceState) + .data; - expect(activityLogData).toEqual(activityLog); + expect(activityLogData).toEqual(getMockEndpointActivityLog()); }); }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts index a1da3c072293e..aa0afe5ec980a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts @@ -8,7 +8,7 @@ import { Dispatch } from 'redux'; import { CoreStart, HttpStart } from 'kibana/public'; import { - EndpointAction, + ActivityLog, HostInfo, HostIsolationRequestBody, HostIsolationResponse, @@ -32,6 +32,8 @@ import { getIsIsolationRequestPending, getCurrentIsolationRequestState, getActivityLogData, + getActivityLogDataPaging, + getLastLoadedActivityLogData, } from './selectors'; import { EndpointState, PolicyIds } from '../types'; import { @@ -336,21 +338,25 @@ export const endpointMiddlewareFactory: ImmutableMiddlewareFactory(getActivityLogData(getState())), + payload: createLoadingResourceState(getActivityLogData(getState())), }); try { - const activityLog = await coreStart.http.get( - resolvePathVariables(ENDPOINT_ACTION_LOG_ROUTE, { agent_id: selectedAgent(getState()) }) - ); + const { page, pageSize } = getActivityLogDataPaging(getState()); + const route = resolvePathVariables(ENDPOINT_ACTION_LOG_ROUTE, { + agent_id: selectedAgent(getState()), + }); + const activityLog = await coreStart.http.get(route, { + query: { page, page_size: pageSize }, + }); dispatch({ type: 'endpointDetailsActivityLogChanged', - payload: createLoadedResourceState(activityLog), + payload: createLoadedResourceState(activityLog), }); } catch (error) { dispatch({ type: 'endpointDetailsActivityLogChanged', - payload: createFailedResourceState(error.body ?? error), + payload: createFailedResourceState(error.body ?? error), }); } @@ -371,6 +377,56 @@ export const endpointMiddlewareFactory: ImmutableMiddlewareFactory(getActivityLogData(getState())), + }); + + try { + const { page, pageSize } = getActivityLogDataPaging(getState()); + const route = resolvePathVariables(ENDPOINT_ACTION_LOG_ROUTE, { + agent_id: selectedAgent(getState()), + }); + const activityLog = await coreStart.http.get(route, { + query: { page, page_size: pageSize }, + }); + + const lastLoadedLogData = getLastLoadedActivityLogData(getState()); + if (lastLoadedLogData !== undefined) { + const updatedLogDataItems = [ + ...new Set([...lastLoadedLogData.data, ...activityLog.data]), + ] as ActivityLog['data']; + + const updatedLogData = { + total: activityLog.total, + page: activityLog.page, + pageSize: activityLog.pageSize, + data: updatedLogDataItems, + }; + dispatch({ + type: 'endpointDetailsActivityLogChanged', + payload: createLoadedResourceState(updatedLogData), + }); + // TODO dispatch 'noNewLogData' if !activityLog.length + // resets paging to previous state + } else { + dispatch({ + type: 'endpointDetailsActivityLogChanged', + payload: createLoadedResourceState(activityLog), + }); + } + } catch (error) { + dispatch({ + type: 'endpointDetailsActivityLogChanged', + payload: createFailedResourceState(error.body ?? error), + }); + } + } + // Isolate Host if (action.type === 'endpointIsolationRequest') { return handleIsolateEndpointHost(store, action); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts index 19235b792b270..b580664512eb6 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts @@ -33,7 +33,10 @@ const handleEndpointDetailsActivityLogChanged: CaseReducer +): Immutable> => { + return { + page: state.endpointDetails.activityLog.page, + pageSize: state.endpointDetails.activityLog.pageSize, + }; +}; + export const getActivityLogData = ( state: Immutable -): Immutable => state.endpointDetails.activityLog; +): Immutable => + state.endpointDetails.activityLog.logData; + +export const getLastLoadedActivityLogData: ( + state: Immutable +) => Immutable | undefined = createSelector(getActivityLogData, (activityLog) => { + return getLastLoadedResourceState(activityLog)?.data; +}); export const getActivityLogRequestLoading: ( state: Immutable @@ -375,6 +394,13 @@ export const getActivityLogRequestLoaded: ( isLoadedResourceState(activityLog) ); +export const getActivityLogIterableData: ( + state: Immutable +) => Immutable = createSelector(getActivityLogData, (activityLog) => { + const emptyArray: ActivityLog['data'] = []; + return isLoadedResourceState(activityLog) ? activityLog.data.data : emptyArray; +}); + export const getActivityLogError: ( state: Immutable ) => ServerApiError | undefined = createSelector(getActivityLogData, (activityLog) => { diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts index 53ddfaee7aa05..eed2182d41809 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts @@ -6,6 +6,7 @@ */ import { + ActivityLog, HostInfo, Immutable, HostMetadata, @@ -14,7 +15,6 @@ import { PolicyData, MetadataQueryStrategyVersions, HostStatus, - EndpointAction, HostIsolationResponse, } from '../../../../common/endpoint/types'; import { ServerApiError } from '../../../common/types'; @@ -36,7 +36,11 @@ export interface EndpointState { /** api error from retrieving host list */ error?: ServerApiError; endpointDetails: { - activityLog: AsyncResourceState; + activityLog: { + page: number; + pageSize: number; + logData: AsyncResourceState; + }; hostDetails: { /** details data for a specific host */ details?: Immutable; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry.tsx index de6d2ecf36ecc..f8574da7f0a03 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry.tsx @@ -5,53 +5,162 @@ * 2.0. */ -import React, { memo } from 'react'; +import React, { memo, useMemo } from 'react'; +import styled from 'styled-components'; -import { EuiAvatar, EuiComment, EuiText } from '@elastic/eui'; -import { Immutable, EndpointAction } from '../../../../../../../common/endpoint/types'; +import { EuiComment, EuiText, EuiAvatarProps, EuiCommentProps, IconType } from '@elastic/eui'; +import { Immutable, ActivityLogEntry } from '../../../../../../../common/endpoint/types'; import { FormattedDateAndTime } from '../../../../../../common/components/endpoint/formatted_date_time'; -import { useEuiTheme } from '../../../../../../common/lib/theme/use_eui_theme'; - -export const LogEntry = memo( - ({ endpointAction }: { endpointAction: Immutable }) => { - const euiTheme = useEuiTheme(); - const isIsolated = endpointAction?.data.command === 'isolate'; - - // do this better when we can distinguish between endpoint events vs user events - const iconType = endpointAction.user_id === 'sys' ? 'dot' : isIsolated ? 'lock' : 'lockOpen'; - const commentType = endpointAction.user_id === 'sys' ? 'update' : 'regular'; - const timelineIcon = ( - - ); - const event = `${isIsolated ? 'isolated' : 'unisolated'} host`; - const hasComment = !!endpointAction.data.comment; - - return ( - - {hasComment ? ( - -

{endpointAction.data.comment}

- - ) : undefined} - - ); +import { LogEntryTimelineIcon } from './log_entry_timeline_icon'; + +import * as i18 from '../../translations'; + +const useLogEntryUIProps = ( + logEntry: Immutable +): { + actionEventTitle: string; + avatarSize: EuiAvatarProps['size']; + commentText: string; + commentType: EuiCommentProps['type']; + displayComment: boolean; + displayResponseEvent: boolean; + iconType: IconType; + isResponseEvent: boolean; + isSuccessful: boolean; + responseEventTitle: string; + username: string | React.ReactNode; +} => { + return useMemo(() => { + let iconType: IconType = 'dot'; + let commentType: EuiCommentProps['type'] = 'update'; + let commentText: string = ''; + let avatarSize: EuiAvatarProps['size'] = 's'; + let isIsolateAction: boolean = false; + let isResponseEvent: boolean = false; + let isSuccessful: boolean = false; + let displayComment: boolean = false; + let displayResponseEvent: boolean = true; + let username: EuiCommentProps['username'] = ''; + + if (logEntry.type === 'action') { + avatarSize = 'm'; + commentType = 'regular'; + commentText = logEntry.item.data.data.comment ?? ''; + displayResponseEvent = false; + iconType = 'lockOpen'; + username = logEntry.item.data.user_id; + if (logEntry.item.data.data) { + const data = logEntry.item.data.data; + if (data.command === 'isolate') { + iconType = 'lock'; + isIsolateAction = true; + } + if (data.comment) { + displayComment = true; + } + } + } else if (logEntry.type === 'response') { + isResponseEvent = true; + if (logEntry.item.data.action_data.command === 'isolate') { + isIsolateAction = true; + } + if (!!logEntry.item.data.completed_at && !logEntry.item.data.error) { + isSuccessful = true; + } + } + + const actionEventTitle = isIsolateAction + ? i18.ACTIVITY_LOG.LogEntry.action.isolatedAction + : i18.ACTIVITY_LOG.LogEntry.action.unisolatedAction; + + const getResponseEventTitle = () => { + if (isIsolateAction) { + if (isSuccessful) { + return i18.ACTIVITY_LOG.LogEntry.response.isolationSuccessful; + } else { + return i18.ACTIVITY_LOG.LogEntry.response.isolationSuccessful; + } + } else { + if (isSuccessful) { + return i18.ACTIVITY_LOG.LogEntry.response.unisolationSuccessful; + } else { + return i18.ACTIVITY_LOG.LogEntry.response.unisolationFailed; + } + } + }; + + return { + actionEventTitle, + avatarSize, + commentText, + commentType, + displayComment, + displayResponseEvent, + iconType, + isResponseEvent, + isSuccessful, + responseEventTitle: getResponseEventTitle(), + username, + }; + }, [logEntry]); +}; + +const StyledEuiComment = styled(EuiComment)` + .euiCommentEvent__headerTimestamp { + display: flex; + :before { + content: ''; + background-color: ${(props) => props.theme.eui.euiColorInk}; + display: block; + width: ${(props) => props.theme.eui.euiBorderWidthThick}; + height: ${(props) => props.theme.eui.euiBorderWidthThick}; + margin: 0 ${(props) => props.theme.eui.euiSizeXS} 0 ${(props) => props.theme.eui.euiSizeS}; + border-radius: 50%; + content: ''; + margin: 0 8px 0 4px; + border-radius: 50%; + position: relative; + top: 10px; + } } -); +`; + +export const LogEntry = memo(({ logEntry }: { logEntry: Immutable }) => { + const { + actionEventTitle, + avatarSize, + commentText, + commentType, + displayComment, + displayResponseEvent, + iconType, + isResponseEvent, + isSuccessful, + responseEventTitle, + username, + } = useLogEntryUIProps(logEntry); + + return ( + {displayResponseEvent ? responseEventTitle : actionEventTitle}} + timelineIcon={ + + } + data-test-subj="timelineEntry" + > + {displayComment ? ( + +

{commentText}

+ + ) : undefined} + + ); +}); LogEntry.displayName = 'LogEntry'; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry_timeline_icon.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry_timeline_icon.tsx new file mode 100644 index 0000000000000..3ff311cd8a139 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/components/log_entry_timeline_icon.tsx @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo } from 'react'; +import { EuiAvatar, EuiAvatarProps } from '@elastic/eui'; +import { useEuiTheme } from '../../../../../../common/lib/theme/use_eui_theme'; + +export const LogEntryTimelineIcon = memo( + ({ + avatarSize, + isResponseEvent, + isSuccessful, + iconType, + }: { + avatarSize: EuiAvatarProps['size']; + isResponseEvent: boolean; + isSuccessful: boolean; + iconType: EuiAvatarProps['iconType']; + }) => { + const euiTheme = useEuiTheme(); + + return ( + + ); + } +); + +LogEntryTimelineIcon.displayName = 'LogEntryTimelineIcon'; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx index 50c91730e332c..4395e3965ea00 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoint_activity_log.tsx @@ -7,21 +7,48 @@ import React, { memo, useCallback } from 'react'; -import { EuiEmptyPrompt, EuiSpacer } from '@elastic/eui'; +import { EuiButton, EuiEmptyPrompt, EuiLoadingContent, EuiSpacer } from '@elastic/eui'; +import { useDispatch } from 'react-redux'; import { LogEntry } from './components/log_entry'; import * as i18 from '../translations'; import { SearchBar } from '../../../../components/search_bar'; -import { Immutable, EndpointAction } from '../../../../../../common/endpoint/types'; +import { Immutable, ActivityLog } from '../../../../../../common/endpoint/types'; import { AsyncResourceState } from '../../../../state'; +import { useEndpointSelector } from '../hooks'; +import { EndpointAction } from '../../store/action'; +import { + getActivityLogDataPaging, + getActivityLogError, + getActivityLogIterableData, + getActivityLogRequestLoaded, + getActivityLogRequestLoading, +} from '../../store/selectors'; export const EndpointActivityLog = memo( - ({ endpointActions }: { endpointActions: AsyncResourceState> }) => { + ({ activityLog }: { activityLog: AsyncResourceState> }) => { + const activityLogLoading = useEndpointSelector(getActivityLogRequestLoading); + const activityLogLoaded = useEndpointSelector(getActivityLogRequestLoaded); + const activityLogData = useEndpointSelector(getActivityLogIterableData); + const activityLogError = useEndpointSelector(getActivityLogError); + const dispatch = useDispatch<(a: EndpointAction) => void>(); + const { page, pageSize } = useEndpointSelector(getActivityLogDataPaging); // TODO const onSearch = useCallback(() => {}, []); + + const getActivityLog = useCallback(() => { + dispatch({ + type: 'appRequestedEndpointActivityLog', + payload: { + page: page + 1, + pageSize, + }, + }); + }, [dispatch, page, pageSize]); + return ( <> - {endpointActions.type !== 'LoadedResourceState' || !endpointActions.data.length ? ( + {activityLogLoading || activityLogError ? ( - {endpointActions.data.map((endpointAction) => ( - - ))} + {activityLogLoading ? ( + + ) : ( + activityLogLoaded && + activityLogData.map((logEntry) => ( + + )) + )} + + {'show more'} + )} diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx index fccd48cbde67c..d839bbfaae875 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/endpoints.stories.tsx @@ -8,7 +8,7 @@ import React, { ComponentType } from 'react'; import moment from 'moment'; -import { EndpointAction, Immutable } from '../../../../../../common/endpoint/types'; +import { ActivityLog, Immutable } from '../../../../../../common/endpoint/types'; import { EndpointDetailsFlyoutTabs } from './components/endpoint_details_tabs'; import { EndpointActivityLog } from './endpoint_activity_log'; import { EndpointDetailsFlyout } from '.'; @@ -17,63 +17,93 @@ import { AsyncResourceState } from '../../../../state'; export const dummyEndpointActivityLog = ( selectedEndpoint: string = '' -): AsyncResourceState> => ({ +): AsyncResourceState> => ({ type: 'LoadedResourceState', - data: [ - { - action_id: '1', - '@timestamp': moment().subtract(1, 'hours').fromNow().toString(), - expiration: moment().add(3, 'day').fromNow().toString(), - type: 'INPUT_ACTION', - input_type: 'endpoint', - agents: [`${selectedEndpoint}`], - user_id: 'sys', - data: { - command: 'isolate', + data: { + total: 20, + page: 1, + pageSize: 50, + data: [ + { + type: 'action', + item: { + id: '', + data: { + action_id: '1', + '@timestamp': moment().subtract(1, 'hours').fromNow().toString(), + expiration: moment().add(3, 'day').fromNow().toString(), + type: 'INPUT_ACTION', + input_type: 'endpoint', + agents: [`${selectedEndpoint}`], + user_id: 'sys', + data: { + command: 'isolate', + }, + }, + }, }, - }, - { - action_id: '2', - '@timestamp': moment().subtract(2, 'hours').fromNow().toString(), - expiration: moment().add(1, 'day').fromNow().toString(), - type: 'INPUT_ACTION', - input_type: 'endpoint', - agents: [`${selectedEndpoint}`], - user_id: 'ash', - data: { - command: 'isolate', - comment: 'Sem et tortor consequat id porta nibh venenatis cras sed.', + { + type: 'action', + item: { + id: '', + data: { + action_id: '2', + '@timestamp': moment().subtract(2, 'hours').fromNow().toString(), + expiration: moment().add(1, 'day').fromNow().toString(), + type: 'INPUT_ACTION', + input_type: 'endpoint', + agents: [`${selectedEndpoint}`], + user_id: 'ash', + data: { + command: 'isolate', + comment: 'Sem et tortor consequat id porta nibh venenatis cras sed.', + }, + }, + }, }, - }, - { - action_id: '3', - '@timestamp': moment().subtract(4, 'hours').fromNow().toString(), - expiration: moment().add(1, 'day').fromNow().toString(), - type: 'INPUT_ACTION', - input_type: 'endpoint', - agents: [`${selectedEndpoint}`], - user_id: 'someone', - data: { - command: 'unisolate', - comment: 'Turpis egestas pretium aenean pharetra.', + { + type: 'action', + item: { + id: '', + data: { + action_id: '3', + '@timestamp': moment().subtract(4, 'hours').fromNow().toString(), + expiration: moment().add(1, 'day').fromNow().toString(), + type: 'INPUT_ACTION', + input_type: 'endpoint', + agents: [`${selectedEndpoint}`], + user_id: 'someone', + data: { + command: 'unisolate', + comment: 'Turpis egestas pretium aenean pharetra.', + }, + }, + }, }, - }, - { - action_id: '4', - '@timestamp': moment().subtract(1, 'day').fromNow().toString(), - expiration: moment().add(3, 'day').fromNow().toString(), - type: 'INPUT_ACTION', - input_type: 'endpoint', - agents: [`${selectedEndpoint}`], - user_id: 'ash', - data: { - command: 'isolate', - comment: - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, \ - sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', + { + type: 'action', + item: { + id: '', + data: { + action_id: '4', + '@timestamp': moment().subtract(1, 'day').fromNow().toString(), + expiration: moment().add(3, 'day').fromNow().toString(), + type: 'INPUT_ACTION', + input_type: 'endpoint', + agents: [`${selectedEndpoint}`], + user_id: 'ash', + data: { + command: 'isolate', + comment: + 'Lorem \ + ipsum dolor sit amet, consectetur adipiscing elit, \ + sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', + }, + }, + }, }, - }, - ], + ], + }, }); export default { @@ -100,12 +130,12 @@ export const Tabs = () => ( { id: 'activity_log', name: 'Activity Log', - content: ActivityLog(), + content: ActivityLogMarkup(), }, ]} /> ); -export const ActivityLog = () => ( - +export const ActivityLogMarkup = () => ( + ); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/index.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/index.tsx index 89c0e3e6a3e06..c39a17e98c76a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/details/index.tsx @@ -33,7 +33,6 @@ import { detailsLoading, getActivityLogData, getActivityLogError, - getActivityLogRequestLoading, showView, policyResponseConfigurations, policyResponseActions, @@ -87,7 +86,6 @@ export const EndpointDetailsFlyout = memo(() => { } = queryParams; const activityLog = useEndpointSelector(getActivityLogData); - const activityLoading = useEndpointSelector(getActivityLogRequestLoading); const activityError = useEndpointSelector(getActivityLogError); const hostDetails = useEndpointSelector(detailsData); const hostDetailsLoading = useEndpointSelector(detailsLoading); @@ -121,12 +119,8 @@ export const EndpointDetailsFlyout = memo(() => { }, { id: EndpointDetailsTabsTypes.activityLog, - name: i18.ACTIVITY_LOG, - content: activityLoading ? ( - ContentLoadingMarkup - ) : ( - - ), + name: i18.ACTIVITY_LOG.tabTitle, + content: , }, ]; @@ -175,7 +169,7 @@ export const EndpointDetailsFlyout = memo(() => { paddingSize="m" > - {hostDetailsLoading || activityLoading ? ( + {hostDetailsLoading ? ( ) : ( diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts index fd2806713183b..1a7889f22db16 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/translations.ts @@ -11,9 +11,53 @@ export const OVERVIEW = i18n.translate('xpack.securitySolution.endpointDetails.o defaultMessage: 'Overview', }); -export const ACTIVITY_LOG = i18n.translate('xpack.securitySolution.endpointDetails.activityLog', { - defaultMessage: 'Activity Log', -}); +export const ACTIVITY_LOG = { + tabTitle: i18n.translate('xpack.securitySolution.endpointDetails.activityLog', { + defaultMessage: 'Activity Log', + }), + LogEntry: { + action: { + isolatedAction: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.action.isolated', + { + defaultMessage: 'isolated host', + } + ), + unisolatedAction: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.action.unisolated', + { + defaultMessage: 'unisolated host', + } + ), + }, + response: { + isolationSuccessful: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.isolationSuccessful', + { + defaultMessage: 'host isolation successful', + } + ), + isolationFailed: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.isolationFailed', + { + defaultMessage: 'host isolation failed', + } + ), + unisolationSuccessful: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.unisolationSuccessful', + { + defaultMessage: 'host unisolation successful', + } + ), + unisolationFailed: i18n.translate( + 'xpack.securitySolution.endpointDetails.activityLog.logEntry.response.unisolationFailed', + { + defaultMessage: 'host unisolation failed', + } + ), + }, + }, +}; export const SEARCH_ACTIVITY_LOG = i18n.translate( 'xpack.securitySolution.endpointDetails.activityLog.search', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts new file mode 100644 index 0000000000000..9b73721775382 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log.test.ts @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { KibanaResponseFactory, RequestHandler, RouteConfig } from 'kibana/server'; +import { + elasticsearchServiceMock, + httpServerMock, + httpServiceMock, + loggingSystemMock, + savedObjectsClientMock, +} from 'src/core/server/mocks'; +import { + EndpointActionLogRequestParams, + EndpointActionLogRequestQuery, + EndpointActionLogRequestSchema, +} from '../../../../common/endpoint/schema/actions'; +import { ENDPOINT_ACTION_LOG_ROUTE } from '../../../../common/endpoint/constants'; +import { parseExperimentalConfigValue } from '../../../../common/experimental_features'; +import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; +import { EndpointAppContextService } from '../../endpoint_app_context_services'; +import { + createMockEndpointAppContextServiceStartContract, + createRouteHandlerContext, +} from '../../mocks'; +import { registerActionAuditLogRoutes } from './audit_log'; +import uuid from 'uuid'; +import { aMockAction, aMockResponse, MockAction, mockAuditLog, MockResponse } from './mocks'; +import { SecuritySolutionRequestHandlerContext } from '../../../types'; +import { ActivityLog } from '../../../../common/endpoint/types'; + +describe('Action Log API', () => { + describe('schema', () => { + it('should require at least 1 agent ID', () => { + expect(() => { + EndpointActionLogRequestSchema.params.validate({}); // no agent_ids provided + }).toThrow(); + }); + + it('should accept a single agent ID', () => { + expect(() => { + EndpointActionLogRequestSchema.params.validate({ agent_id: uuid.v4() }); + }).not.toThrow(); + }); + + it('should work without query params', () => { + expect(() => { + EndpointActionLogRequestSchema.query.validate({}); + }).not.toThrow(); + }); + + it('should work with query params', () => { + expect(() => { + EndpointActionLogRequestSchema.query.validate({ page: 10, page_size: 100 }); + }).not.toThrow(); + }); + + it('should not work without allowed page and page_size params', () => { + expect(() => { + EndpointActionLogRequestSchema.query.validate({ page_size: 101 }); + }).toThrow(); + }); + }); + + describe('response', () => { + const mockID = 'XYZABC-000'; + const actionID = 'some-known-actionid'; + let endpointAppContextService: EndpointAppContextService; + + // convenience for calling the route and handler for audit log + let getActivityLog: ( + query?: EndpointActionLogRequestQuery + ) => Promise>; + // convenience for injecting mock responses for actions index and responses + let havingActionsAndResponses: (actions: MockAction[], responses: any[]) => void; + + let havingErrors: () => void; + + beforeEach(() => { + const esClientMock = elasticsearchServiceMock.createScopedClusterClient(); + const routerMock = httpServiceMock.createRouter(); + endpointAppContextService = new EndpointAppContextService(); + endpointAppContextService.start(createMockEndpointAppContextServiceStartContract()); + + registerActionAuditLogRoutes(routerMock, { + logFactory: loggingSystemMock.create(), + service: endpointAppContextService, + config: () => Promise.resolve(createMockConfig()), + experimentalFeatures: parseExperimentalConfigValue(createMockConfig().enableExperimental), + }); + + getActivityLog = async (query?: any): Promise> => { + const req = httpServerMock.createKibanaRequest({ + params: { agent_id: mockID }, + query, + }); + const mockResponse = httpServerMock.createResponseFactory(); + const [, routeHandler]: [ + RouteConfig, + RequestHandler< + EndpointActionLogRequestParams, + EndpointActionLogRequestQuery, + unknown, + SecuritySolutionRequestHandlerContext + > + ] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith(ENDPOINT_ACTION_LOG_ROUTE) + )!; + await routeHandler( + createRouteHandlerContext(esClientMock, savedObjectsClientMock.create()), + req, + mockResponse + ); + + return mockResponse; + }; + + havingActionsAndResponses = (actions: MockAction[], responses: MockResponse[]) => { + const actionsData = actions.map((a) => ({ + _index: '.fleet-actions-7', + _source: a.build(), + })); + const responsesData = responses.map((r) => ({ + _index: '.ds-.fleet-actions-results-2021.06.09-000001', + _source: r.build(), + })); + const mockResult = mockAuditLog([...actionsData, ...responsesData]); + esClientMock.asCurrentUser.search = jest + .fn() + .mockImplementationOnce(() => Promise.resolve(mockResult)); + }; + + havingErrors = () => { + esClientMock.asCurrentUser.search = jest.fn().mockImplementationOnce(() => + Promise.resolve(() => { + throw new Error(); + }) + ); + }; + }); + + afterEach(() => { + endpointAppContextService.stop(); + }); + + it('should return an empty array when nothing in audit log', async () => { + havingActionsAndResponses([], []); + const response = await getActivityLog(); + expect(response.ok).toBeCalled(); + expect((response.ok.mock.calls[0][0]?.body as ActivityLog).data).toHaveLength(0); + }); + + it('should have actions and action responses', async () => { + havingActionsAndResponses( + [ + aMockAction().withAgent(mockID).withAction('isolate'), + aMockAction().withAgent(mockID).withAction('unisolate'), + aMockAction().withAgent(mockID).withAction('isolate'), + ], + [aMockResponse(actionID, mockID), aMockResponse(actionID, mockID)] + ); + const response = await getActivityLog(); + const responseBody = response.ok.mock.calls[0][0]?.body as ActivityLog; + + expect(response.ok).toBeCalled(); + expect(responseBody.data).toHaveLength(5); + expect(responseBody.data.filter((x: any) => x.type === 'response')).toHaveLength(2); + expect(responseBody.data.filter((x: any) => x.type === 'action')).toHaveLength(3); + }); + + it('should throw errors when no results for some agentID', async () => { + havingErrors(); + + try { + await getActivityLog(); + } catch (error) { + expect(error.message).toEqual(`Error fetching actions log for agent_id ${mockID}`); + } + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts index fdbb9608463e9..b0cea299af60d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/audit_log_handler.ts @@ -5,55 +5,34 @@ * 2.0. */ -import { TypeOf } from '@kbn/config-schema'; import { RequestHandler } from 'kibana/server'; -import { AGENT_ACTIONS_INDEX } from '../../../../../fleet/common'; -import { EndpointActionLogRequestSchema } from '../../../../common/endpoint/schema/actions'; - +import { + EndpointActionLogRequestParams, + EndpointActionLogRequestQuery, +} from '../../../../common/endpoint/schema/actions'; +import { getAuditLogResponse } from './service'; import { SecuritySolutionRequestHandlerContext } from '../../../types'; import { EndpointAppContext } from '../../types'; export const actionsLogRequestHandler = ( endpointContext: EndpointAppContext ): RequestHandler< - TypeOf, - unknown, + EndpointActionLogRequestParams, + EndpointActionLogRequestQuery, unknown, SecuritySolutionRequestHandlerContext > => { const logger = endpointContext.logFactory.get('audit_log'); + return async (context, req, res) => { - const esClient = context.core.elasticsearch.client.asCurrentUser; - let result; - try { - result = await esClient.search({ - index: AGENT_ACTIONS_INDEX, - body: { - query: { - match: { - agents: req.params.agent_id, - }, - }, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - }); - } catch (error) { - logger.error(error); - throw error; - } - if (result?.statusCode !== 200) { - logger.error(`Error fetching actions log for agent_id ${req.params.agent_id}`); - throw new Error(`Error fetching actions log for agent_id ${req.params.agent_id}`); - } + const { + params: { agent_id: elasticAgentId }, + query: { page, page_size: pageSize }, + } = req; + const body = await getAuditLogResponse({ elasticAgentId, page, pageSize, context, logger }); return res.ok({ - body: result.body.hits.hits.map((e) => e._source), + body, }); }; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/mocks.ts index 34f7d140a78de..f74ae07fdfac4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/mocks.ts @@ -18,6 +18,29 @@ import { ISOLATION_ACTIONS, } from '../../../../common/endpoint/types'; +export const mockAuditLog = (results: any = []): ApiResponse => { + return { + body: { + hits: { + total: results.length, + hits: results.map((a: any) => { + const _index = a._index; + delete a._index; + const _source = a; + return { + _index, + _source, + }; + }), + }, + }, + statusCode: 200, + headers: {}, + warnings: [], + meta: {} as any, + }; +}; + export const mockSearchResult = (results: any = []): ApiResponse => { return { body: { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/service.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/service.ts new file mode 100644 index 0000000000000..20b29694a1df1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/service.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import type { estypes } from '@elastic/elasticsearch'; +import { AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX } from '../../../../../fleet/common'; +import { SecuritySolutionRequestHandlerContext } from '../../../types'; + +export const getAuditLogESQuery = ({ + elasticAgentId, + from, + size, +}: { + elasticAgentId: string; + from: number; + size: number; +}): estypes.SearchRequest => { + return { + index: [AGENT_ACTIONS_INDEX, AGENT_ACTIONS_RESULTS_INDEX], + size, + from, + body: { + query: { + bool: { + should: [ + { terms: { agents: [elasticAgentId] } }, + { terms: { agent_id: [elasticAgentId] } }, + ], + }, + }, + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + }, + }; +}; + +export const getAuditLogResponse = async ({ + elasticAgentId, + page, + pageSize, + context, + logger, +}: { + elasticAgentId: string; + page: number; + pageSize: number; + context: SecuritySolutionRequestHandlerContext; + logger: Logger; +}): Promise<{ + total: number; + page: number; + pageSize: number; + data: Array<{ + type: 'action' | 'response'; + item: { + id: string; + data: unknown; + }; + }>; +}> => { + const size = pageSize; + const from = page <= 1 ? 0 : page * pageSize - pageSize + 1; + + const options = { + headers: { + 'X-elastic-product-origin': 'fleet', + }, + ignore: [404], + }; + const esClient = context.core.elasticsearch.client.asCurrentUser; + let result; + const params = getAuditLogESQuery({ + elasticAgentId, + from, + size, + }); + + try { + result = await esClient.search(params, options); + } catch (error) { + logger.error(error); + throw error; + } + if (result?.statusCode !== 200) { + logger.error(`Error fetching actions log for agent_id ${elasticAgentId}`); + throw new Error(`Error fetching actions log for agent_id ${elasticAgentId}`); + } + + return { + total: + typeof result.body.hits.total === 'number' + ? result.body.hits.total + : result.body.hits.total.value, + page, + pageSize, + data: result.body.hits.hits.map((e) => ({ + type: e._index.startsWith('.fleet-actions') ? 'action' : 'response', + item: { id: e._id, data: e._source }, + })), + }; +}; From e0a1a34fbc5a92cae76638514779a89cad853fa1 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 14 Jun 2021 15:57:03 +0200 Subject: [PATCH 02/91] [Uptime] Align synthetics report types (#101855) --- .../PageLoadDistribution/index.tsx | 2 +- .../components/filter_label.test.tsx | 8 +-- .../apm/service_latency_config.ts | 5 +- .../apm/service_throughput_config.ts | 57 --------------- .../configurations/constants/constants.ts | 34 +-------- .../constants/field_names/synthetics.ts | 8 +++ .../configurations/constants/labels.ts | 55 +++------------ .../configurations/default_configs.ts | 64 +++++++---------- .../configurations/lens_attributes.test.ts | 6 +- .../configurations/lens_attributes.ts | 19 +++-- .../logs/logs_frequency_config.ts | 41 ----------- .../metrics/cpu_usage_config.ts | 11 +-- .../metrics/memory_usage_config.ts | 11 +-- .../metrics/network_activity_config.ts | 11 +-- .../rum/core_web_vitals_config.ts | 5 +- ..._config.ts => data_distribution_config.ts} | 5 +- ...ends_config.ts => kpi_over_time_config.ts} | 5 +- ..._config.ts => data_distribution_config.ts} | 25 ++++--- .../synthetics/kpi_over_time_config.ts | 69 +++++++++++++++++++ .../synthetics/monitor_pings_config.ts | 50 -------------- .../exploratory_view.test.tsx | 2 +- .../exploratory_view/exploratory_view.tsx | 9 ++- .../exploratory_view/header/header.test.tsx | 2 +- .../hooks/use_lens_attributes.ts | 13 ++-- .../shared/exploratory_view/rtl_helpers.tsx | 2 +- .../columns/data_types_col.test.tsx | 2 +- .../series_builder/columns/data_types_col.tsx | 3 +- .../columns/operation_type_select.tsx | 6 ++ .../columns/report_breakdowns.test.tsx | 8 +-- .../columns/report_definition_col.test.tsx | 8 +-- .../columns/report_filters.test.tsx | 4 +- .../columns/report_types_col.test.tsx | 8 +-- .../series_builder/series_builder.tsx | 26 ++----- .../series_date_picker.test.tsx | 10 +-- .../series_editor/chart_edit_options.tsx | 2 +- .../series_editor/columns/breakdowns.test.tsx | 7 +- .../series_editor/columns/chart_options.tsx | 7 +- .../series_editor/columns/series_actions.tsx | 3 +- .../series_editor/selected_filters.test.tsx | 5 +- .../series_editor/series_editor.tsx | 30 ++++---- .../shared/exploratory_view/types.ts | 14 +--- .../common/charts/ping_histogram.tsx | 2 +- .../monitor_duration_container.tsx | 2 +- 43 files changed, 247 insertions(+), 419 deletions(-) delete mode 100644 x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_throughput_config.ts create mode 100644 x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/field_names/synthetics.ts delete mode 100644 x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/logs/logs_frequency_config.ts rename x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/{performance_dist_config.ts => data_distribution_config.ts} (94%) rename x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/{kpi_trends_config.ts => kpi_over_time_config.ts} (95%) rename x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/{monitor_duration_config.ts => data_distribution_config.ts} (61%) create mode 100644 x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/kpi_over_time_config.ts delete mode 100644 x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_pings_config.ts diff --git a/x-pack/plugins/apm/public/components/app/RumDashboard/PageLoadDistribution/index.tsx b/x-pack/plugins/apm/public/components/app/RumDashboard/PageLoadDistribution/index.tsx index a8435beca1e4a..adfb45303a4f3 100644 --- a/x-pack/plugins/apm/public/components/app/RumDashboard/PageLoadDistribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/RumDashboard/PageLoadDistribution/index.tsx @@ -89,7 +89,7 @@ export function PageLoadDistribution() { { [`${serviceName}-page-views`]: { dataType: 'ux', - reportType: 'pld', + reportType: 'dist', time: { from: rangeFrom!, to: rangeTo! }, reportDefinitions: { 'service.name': serviceName as string[], diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/components/filter_label.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/components/filter_label.test.tsx index c042ba9d0bcf8..7c772cb8dbdbc 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/components/filter_label.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/components/filter_label.test.tsx @@ -26,7 +26,7 @@ describe('FilterLabel', function () { value={'elastic-co'} label={'Web Application'} negate={false} - seriesId={'kpi-trends'} + seriesId={'kpi-over-time'} removeFilter={jest.fn()} /> ); @@ -49,7 +49,7 @@ describe('FilterLabel', function () { value={'elastic-co'} label={'Web Application'} negate={false} - seriesId={'kpi-trends'} + seriesId={'kpi-over-time'} removeFilter={removeFilter} /> ); @@ -71,7 +71,7 @@ describe('FilterLabel', function () { value={'elastic-co'} label={'Web Application'} negate={false} - seriesId={'kpi-trends'} + seriesId={'kpi-over-time'} removeFilter={removeFilter} /> ); @@ -96,7 +96,7 @@ describe('FilterLabel', function () { value={'elastic-co'} label={'Web Application'} negate={true} - seriesId={'kpi-trends'} + seriesId={'kpi-over-time'} removeFilter={jest.fn()} /> ); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_latency_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_latency_config.ts index 1c2627dac30e7..7c3abba3e5b05 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_latency_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_latency_config.ts @@ -10,10 +10,9 @@ import { FieldLabels } from '../constants'; import { buildPhraseFilter } from '../utils'; import { TRANSACTION_DURATION } from '../constants/elasticsearch_fieldnames'; -export function getServiceLatencyLensConfig({ seriesId, indexPattern }: ConfigProps): DataSeries { +export function getServiceLatencyLensConfig({ indexPattern }: ConfigProps): DataSeries { return { - id: seriesId, - reportType: 'service-latency', + reportType: 'kpi-over-time', defaultSeriesType: 'line', seriesTypes: ['line', 'bar'], xAxisColumn: { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_throughput_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_throughput_config.ts deleted file mode 100644 index 2de2cbdfd75a6..0000000000000 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/apm/service_throughput_config.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ConfigProps, DataSeries } from '../../types'; -import { FieldLabels } from '../constants/constants'; -import { buildPhraseFilter } from '../utils'; -import { TRANSACTION_DURATION } from '../constants/elasticsearch_fieldnames'; - -export function getServiceThroughputLensConfig({ - seriesId, - indexPattern, -}: ConfigProps): DataSeries { - return { - id: seriesId, - reportType: 'service-throughput', - defaultSeriesType: 'line', - seriesTypes: ['line', 'bar'], - xAxisColumn: { - sourceField: '@timestamp', - }, - yAxisColumns: [ - { - operationType: 'average', - sourceField: 'transaction.duration.us', - label: 'Throughput', - }, - ], - hasOperationType: true, - defaultFilters: [ - 'user_agent.name', - 'user_agent.os.name', - 'client.geo.country_name', - 'user_agent.device.name', - ], - breakdowns: [ - 'user_agent.name', - 'user_agent.os.name', - 'client.geo.country_name', - 'user_agent.device.name', - ], - filters: buildPhraseFilter('transaction.type', 'request', indexPattern), - labels: { ...FieldLabels, [TRANSACTION_DURATION]: 'Throughput' }, - reportDefinitions: [ - { - field: 'service.name', - required: true, - }, - { - field: 'service.environment', - }, - ], - }; -} diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/constants.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/constants.ts index e1142a071aab5..26459e676de08 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/constants.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/constants.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { AppDataType, ReportViewTypeId } from '../../types'; +import { ReportViewTypeId } from '../../types'; import { CLS_FIELD, FCP_FIELD, FID_FIELD, LCP_FIELD, TBT_FIELD } from './elasticsearch_fieldnames'; import { AGENT_HOST_LABEL, @@ -13,7 +13,6 @@ import { BROWSER_VERSION_LABEL, CLS_LABEL, CORE_WEB_VITALS_LABEL, - CPU_USAGE_LABEL, DEVICE_LABEL, ENVIRONMENT_LABEL, FCP_LABEL, @@ -23,25 +22,18 @@ import { KPI_LABEL, LCP_LABEL, LOCATION_LABEL, - LOGS_FREQUENCY_LABEL, - MEMORY_USAGE_LABEL, METRIC_LABEL, - MONITOR_DURATION_LABEL, MONITOR_ID_LABEL, MONITOR_NAME_LABEL, MONITOR_STATUS_LABEL, MONITOR_TYPE_LABEL, - NETWORK_ACTIVITY_LABEL, OBSERVER_LOCATION_LABEL, OS_LABEL, PERF_DIST_LABEL, PORT_LABEL, - SERVICE_LATENCY_LABEL, SERVICE_NAME_LABEL, - SERVICE_THROUGHPUT_LABEL, TAGS_LABEL, TBT_LABEL, - UPTIME_PINGS_LABEL, URL_LABEL, } from './labels'; @@ -83,33 +75,11 @@ export const FieldLabels: Record = { }; export const DataViewLabels: Record = { - pld: PERF_DIST_LABEL, - upd: MONITOR_DURATION_LABEL, - upp: UPTIME_PINGS_LABEL, - svl: SERVICE_LATENCY_LABEL, + dist: PERF_DIST_LABEL, kpi: KIP_OVER_TIME_LABEL, - tpt: SERVICE_THROUGHPUT_LABEL, - cpu: CPU_USAGE_LABEL, - logs: LOGS_FREQUENCY_LABEL, - mem: MEMORY_USAGE_LABEL, - nwk: NETWORK_ACTIVITY_LABEL, cwv: CORE_WEB_VITALS_LABEL, }; -export const ReportToDataTypeMap: Record = { - upd: 'synthetics', - upp: 'synthetics', - tpt: 'apm', - svl: 'apm', - kpi: 'ux', - pld: 'ux', - nwk: 'infra_metrics', - mem: 'infra_metrics', - logs: 'infra_logs', - cpu: 'infra_metrics', - cwv: 'ux', -}; - export const USE_BREAK_DOWN_COLUMN = 'USE_BREAK_DOWN_COLUMN'; export const FILTER_RECORDS = 'FILTER_RECORDS'; export const OPERATION_COLUMN = 'operation'; diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/field_names/synthetics.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/field_names/synthetics.ts new file mode 100644 index 0000000000000..edf8b7fb9d741 --- /dev/null +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/field_names/synthetics.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const MONITOR_DURATION_US = 'monitor.duration.us'; diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/labels.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/labels.ts index 92150a76319f8..b5816daa419df 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/labels.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/constants/labels.ts @@ -100,6 +100,10 @@ export const PAGES_LOADED_LABEL = i18n.translate( } ); +export const PINGS_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.pings', { + defaultMessage: 'Pings', +}); + export const MONITOR_ID_LABEL = i18n.translate( 'xpack.observability.expView.fieldLabels.monitorId', { @@ -165,42 +169,6 @@ export const PERF_DIST_LABEL = i18n.translate( } ); -export const MONITOR_DURATION_LABEL = i18n.translate( - 'xpack.observability.expView.fieldLabels.monitorDuration', - { - defaultMessage: 'Uptime monitor duration', - } -); - -export const UPTIME_PINGS_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.pings', { - defaultMessage: 'Uptime pings', -}); - -export const SERVICE_LATENCY_LABEL = i18n.translate( - 'xpack.observability.expView.fieldLabels.serviceLatency', - { - defaultMessage: 'APM Service latency', - } -); - -export const SERVICE_THROUGHPUT_LABEL = i18n.translate( - 'xpack.observability.expView.fieldLabels.serviceThroughput', - { - defaultMessage: 'APM Service throughput', - } -); - -export const CPU_USAGE_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.cpuUsage', { - defaultMessage: 'System CPU usage', -}); - -export const NETWORK_ACTIVITY_LABEL = i18n.translate( - 'xpack.observability.expView.fieldLabels.networkActivity', - { - defaultMessage: 'Network activity', - } -); - export const CORE_WEB_VITALS_LABEL = i18n.translate( 'xpack.observability.expView.fieldLabels.coreWebVitals', { @@ -215,13 +183,6 @@ export const MEMORY_USAGE_LABEL = i18n.translate( } ); -export const LOGS_FREQUENCY_LABEL = i18n.translate( - 'xpack.observability.expView.fieldLabels.logsFrequency', - { - defaultMessage: 'Logs frequency', - } -); - export const KIP_OVER_TIME_LABEL = i18n.translate( 'xpack.observability.expView.fieldLabels.kpiOverTime', { @@ -243,10 +204,10 @@ export const WEB_APPLICATION_LABEL = i18n.translate( } ); -export const UP_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.up', { - defaultMessage: 'Up', +export const UP_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.upPings', { + defaultMessage: 'Up Pings', }); -export const DOWN_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.down', { - defaultMessage: 'Down', +export const DOWN_LABEL = i18n.translate('xpack.observability.expView.fieldLabels.downPings', { + defaultMessage: 'Down Pings', }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/default_configs.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/default_configs.ts index 797ee0c2e0977..13a7900ef5764 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/default_configs.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/default_configs.ts @@ -5,51 +5,37 @@ * 2.0. */ -import { ReportViewTypes } from '../types'; -import { getPerformanceDistLensConfig } from './rum/performance_dist_config'; -import { getMonitorDurationConfig } from './synthetics/monitor_duration_config'; -import { getServiceLatencyLensConfig } from './apm/service_latency_config'; -import { getMonitorPingsConfig } from './synthetics/monitor_pings_config'; -import { getServiceThroughputLensConfig } from './apm/service_throughput_config'; -import { getKPITrendsLensConfig } from './rum/kpi_trends_config'; -import { getCPUUsageLensConfig } from './metrics/cpu_usage_config'; -import { getMemoryUsageLensConfig } from './metrics/memory_usage_config'; -import { getNetworkActivityLensConfig } from './metrics/network_activity_config'; -import { getLogsFrequencyLensConfig } from './logs/logs_frequency_config'; -import { IIndexPattern } from '../../../../../../../../src/plugins/data/common/index_patterns'; +import { AppDataType, ReportViewTypes } from '../types'; +import { getRumDistributionConfig } from './rum/data_distribution_config'; +import { getSyntheticsDistributionConfig } from './synthetics/data_distribution_config'; +import { getSyntheticsKPIConfig } from './synthetics/kpi_over_time_config'; +import { getKPITrendsLensConfig } from './rum/kpi_over_time_config'; +import { IndexPattern } from '../../../../../../../../src/plugins/data/common'; import { getCoreWebVitalsConfig } from './rum/core_web_vitals_config'; interface Props { reportType: keyof typeof ReportViewTypes; - seriesId: string; - indexPattern: IIndexPattern; + indexPattern: IndexPattern; + dataType: AppDataType; } -export const getDefaultConfigs = ({ reportType, seriesId, indexPattern }: Props) => { - switch (ReportViewTypes[reportType]) { - case 'page-load-dist': - return getPerformanceDistLensConfig({ seriesId, indexPattern }); - case 'kpi-trends': - return getKPITrendsLensConfig({ seriesId, indexPattern }); - case 'core-web-vitals': - return getCoreWebVitalsConfig({ seriesId, indexPattern }); - case 'uptime-duration': - return getMonitorDurationConfig({ seriesId, indexPattern }); - case 'uptime-pings': - return getMonitorPingsConfig({ seriesId, indexPattern }); - case 'service-latency': - return getServiceLatencyLensConfig({ seriesId, indexPattern }); - case 'service-throughput': - return getServiceThroughputLensConfig({ seriesId, indexPattern }); - case 'cpu-usage': - return getCPUUsageLensConfig({ seriesId }); - case 'memory-usage': - return getMemoryUsageLensConfig({ seriesId }); - case 'network-activity': - return getNetworkActivityLensConfig({ seriesId }); - case 'logs-frequency': - return getLogsFrequencyLensConfig({ seriesId }); +export const getDefaultConfigs = ({ reportType, dataType, indexPattern }: Props) => { + switch (dataType) { + case 'ux': + if (reportType === 'dist') { + return getRumDistributionConfig({ indexPattern }); + } + if (reportType === 'cwv') { + return getCoreWebVitalsConfig({ indexPattern }); + } + return getKPITrendsLensConfig({ indexPattern }); + case 'synthetics': + if (reportType === 'dist') { + return getSyntheticsDistributionConfig({ indexPattern }); + } + return getSyntheticsKPIConfig({ indexPattern }); + default: - return getKPITrendsLensConfig({ seriesId, indexPattern }); + return getKPITrendsLensConfig({ indexPattern }); } }; diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts index 6976b55921b09..8b21df64a3c91 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.test.ts @@ -15,9 +15,9 @@ describe('Lens Attribute', () => { mockAppIndexPattern(); const reportViewConfig = getDefaultConfigs({ - reportType: 'pld', + reportType: 'dist', + dataType: 'ux', indexPattern: mockIndexPattern, - seriesId: 'series-id', }); let lnsAttr: LensAttributes; @@ -73,6 +73,7 @@ describe('Lens Attribute', () => { readFromDocValues: true, }, fieldName: 'transaction.duration.us', + columnLabel: 'Page load time', }) ); }); @@ -95,6 +96,7 @@ describe('Lens Attribute', () => { readFromDocValues: true, }, fieldName: LCP_FIELD, + columnLabel: 'Largest contentful paint', }) ); }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts index 89fc9ca5fcc58..bc535e29ab435 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/lens_attributes.ts @@ -55,6 +55,7 @@ export const parseCustomFieldName = ( let fieldName = sourceField; let columnType; let columnFilters; + let columnLabel; const rdf = reportViewConfig.reportDefinitions ?? []; @@ -69,15 +70,17 @@ export const parseCustomFieldName = ( ); columnType = currField?.columnType; columnFilters = currField?.columnFilters; + columnLabel = currField?.label; } } else if (customField.options?.[0].field || customField.options?.[0].id) { fieldName = customField.options?.[0].field || customField.options?.[0].id; columnType = customField.options?.[0].columnType; columnFilters = customField.options?.[0].columnFilters; + columnLabel = customField.options?.[0].label; } } - return { fieldName, columnType, columnFilters }; + return { fieldName, columnType, columnFilters, columnLabel }; }; export class LensAttributes { @@ -260,12 +263,14 @@ export class LensAttributes { label?: string, colIndex?: number ) { - const { fieldMeta, columnType, fieldName, columnFilters } = this.getFieldMeta(sourceField); + const { fieldMeta, columnType, fieldName, columnFilters, columnLabel } = this.getFieldMeta( + sourceField + ); const { type: fieldType } = fieldMeta ?? {}; if (fieldName === 'Records' || columnType === FILTER_RECORDS) { return this.getRecordsColumn( - label, + columnLabel || label, colIndex !== undefined ? columnFilters?.[colIndex] : undefined ); } @@ -274,7 +279,7 @@ export class LensAttributes { return this.getDateHistogramColumn(fieldName); } if (fieldType === 'number') { - return this.getNumberColumn(fieldName, columnType, operationType, label); + return this.getNumberColumn(fieldName, columnType, operationType, columnLabel || label); } // FIXME review my approach again @@ -286,11 +291,13 @@ export class LensAttributes { } getFieldMeta(sourceField: string) { - const { fieldName, columnType, columnFilters } = this.getCustomFieldName(sourceField); + const { fieldName, columnType, columnFilters, columnLabel } = this.getCustomFieldName( + sourceField + ); const fieldMeta = this.indexPattern.getFieldByName(fieldName); - return { fieldMeta, fieldName, columnType, columnFilters }; + return { fieldMeta, fieldName, columnType, columnFilters, columnLabel }; } getMainYAxis() { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/logs/logs_frequency_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/logs/logs_frequency_config.ts deleted file mode 100644 index 97d915ede01a9..0000000000000 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/logs/logs_frequency_config.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DataSeries } from '../../types'; -import { FieldLabels } from '../constants'; - -interface Props { - seriesId: string; -} - -export function getLogsFrequencyLensConfig({ seriesId }: Props): DataSeries { - return { - id: seriesId, - reportType: 'logs-frequency', - defaultSeriesType: 'line', - seriesTypes: ['line', 'bar'], - xAxisColumn: { - sourceField: '@timestamp', - }, - yAxisColumns: [ - { - operationType: 'count', - }, - ], - hasOperationType: false, - defaultFilters: [], - breakdowns: ['agent.hostname'], - filters: [], - labels: { ...FieldLabels }, - reportDefinitions: [ - { - field: 'agent.hostname', - required: true, - }, - ], - }; -} diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/cpu_usage_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/cpu_usage_config.ts index 28b381bd12473..2d44e122af82d 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/cpu_usage_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/cpu_usage_config.ts @@ -5,17 +5,12 @@ * 2.0. */ -import { DataSeries } from '../../types'; +import { DataSeries, ConfigProps } from '../../types'; import { FieldLabels } from '../constants'; -interface Props { - seriesId: string; -} - -export function getCPUUsageLensConfig({ seriesId }: Props): DataSeries { +export function getCPUUsageLensConfig({}: ConfigProps): DataSeries { return { - id: seriesId, - reportType: 'cpu-usage', + reportType: 'kpi-over-time', defaultSeriesType: 'line', seriesTypes: ['line', 'bar'], xAxisColumn: { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/memory_usage_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/memory_usage_config.ts index 2bd0e4b032778..deaa551dce657 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/memory_usage_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/memory_usage_config.ts @@ -5,17 +5,12 @@ * 2.0. */ -import { DataSeries } from '../../types'; +import { DataSeries, ConfigProps } from '../../types'; import { FieldLabels } from '../constants'; -interface Props { - seriesId: string; -} - -export function getMemoryUsageLensConfig({ seriesId }: Props): DataSeries { +export function getMemoryUsageLensConfig({}: ConfigProps): DataSeries { return { - id: seriesId, - reportType: 'memory-usage', + reportType: 'kpi-over-time', defaultSeriesType: 'line', seriesTypes: ['line', 'bar'], xAxisColumn: { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/network_activity_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/network_activity_config.ts index 924701bc13490..d27cdba207d63 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/network_activity_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/metrics/network_activity_config.ts @@ -5,17 +5,12 @@ * 2.0. */ -import { DataSeries } from '../../types'; +import { DataSeries, ConfigProps } from '../../types'; import { FieldLabels } from '../constants'; -interface Props { - seriesId: string; -} - -export function getNetworkActivityLensConfig({ seriesId }: Props): DataSeries { +export function getNetworkActivityLensConfig({}: ConfigProps): DataSeries { return { - id: seriesId, - reportType: 'network-activity', + reportType: 'kpi-over-time', defaultSeriesType: 'line', seriesTypes: ['line', 'bar'], xAxisColumn: { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/core_web_vitals_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/core_web_vitals_config.ts index de9ea12be20cf..e34d8b0dcfdd0 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/core_web_vitals_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/core_web_vitals_config.ts @@ -27,13 +27,12 @@ import { SERVICE_ENVIRONMENT, } from '../constants/elasticsearch_fieldnames'; -export function getCoreWebVitalsConfig({ seriesId, indexPattern }: ConfigProps): DataSeries { +export function getCoreWebVitalsConfig({ indexPattern }: ConfigProps): DataSeries { const statusPallete = euiPaletteForStatus(3); return { - id: seriesId, defaultSeriesType: 'bar_horizontal_percentage_stacked', - reportType: 'kpi-trends', + reportType: 'core-web-vitals', seriesTypes: ['bar_horizontal_percentage_stacked'], xAxisColumn: { sourceField: USE_BREAK_DOWN_COLUMN, diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/performance_dist_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/data_distribution_config.ts similarity index 94% rename from x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/performance_dist_config.ts rename to x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/data_distribution_config.ts index 4a1521c834806..812f1b2e4cf33 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/performance_dist_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/data_distribution_config.ts @@ -39,10 +39,9 @@ import { WEB_APPLICATION_LABEL, } from '../constants/labels'; -export function getPerformanceDistLensConfig({ seriesId, indexPattern }: ConfigProps): DataSeries { +export function getRumDistributionConfig({ indexPattern }: ConfigProps): DataSeries { return { - id: seriesId ?? 'unique-key', - reportType: 'page-load-dist', + reportType: 'data-distribution', defaultSeriesType: 'line', seriesTypes: [], xAxisColumn: { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/kpi_trends_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/kpi_over_time_config.ts similarity index 95% rename from x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/kpi_trends_config.ts rename to x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/kpi_over_time_config.ts index f6c683caaa039..12d66c55c7d00 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/kpi_trends_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/rum/kpi_over_time_config.ts @@ -39,12 +39,11 @@ import { WEB_APPLICATION_LABEL, } from '../constants/labels'; -export function getKPITrendsLensConfig({ seriesId, indexPattern }: ConfigProps): DataSeries { +export function getKPITrendsLensConfig({ indexPattern }: ConfigProps): DataSeries { return { - id: seriesId, defaultSeriesType: 'bar_stacked', - reportType: 'kpi-trends', seriesTypes: [], + reportType: 'kpi-over-time', xAxisColumn: { sourceField: '@timestamp', }, diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_duration_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/data_distribution_config.ts similarity index 61% rename from x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_duration_config.ts rename to x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/data_distribution_config.ts index 5e8a43ccf2ef4..854f844db047d 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_duration_config.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/data_distribution_config.ts @@ -6,27 +6,25 @@ */ import { ConfigProps, DataSeries } from '../../types'; -import { FieldLabels } from '../constants'; +import { FieldLabels, RECORDS_FIELD } from '../constants'; import { buildExistsFilter } from '../utils'; -import { MONITORS_DURATION_LABEL } from '../constants/labels'; +import { MONITORS_DURATION_LABEL, PINGS_LABEL } from '../constants/labels'; -export function getMonitorDurationConfig({ seriesId, indexPattern }: ConfigProps): DataSeries { +export function getSyntheticsDistributionConfig({ indexPattern }: ConfigProps): DataSeries { return { - id: seriesId, - reportType: 'uptime-duration', + reportType: 'data-distribution', defaultSeriesType: 'line', seriesTypes: [], xAxisColumn: { - sourceField: '@timestamp', + sourceField: 'performance.metric', }, yAxisColumns: [ { - operationType: 'average', - sourceField: 'monitor.duration.us', - label: MONITORS_DURATION_LABEL, + sourceField: RECORDS_FIELD, + label: PINGS_LABEL, }, ], - hasOperationType: true, + hasOperationType: false, defaultFilters: ['monitor.type', 'observer.geo.name', 'tags'], breakdowns: [ 'observer.geo.name', @@ -44,6 +42,13 @@ export function getMonitorDurationConfig({ seriesId, indexPattern }: ConfigProps { field: 'url.full', }, + { + field: 'performance.metric', + custom: true, + options: [ + { label: 'Monitor duration', id: 'monitor.duration.us', field: 'monitor.duration.us' }, + ], + }, ], labels: { ...FieldLabels, 'monitor.duration.us': MONITORS_DURATION_LABEL }, }; diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/kpi_over_time_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/kpi_over_time_config.ts new file mode 100644 index 0000000000000..3e92845436363 --- /dev/null +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/kpi_over_time_config.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConfigProps, DataSeries } from '../../types'; +import { FieldLabels, OPERATION_COLUMN } from '../constants'; +import { buildExistsFilter } from '../utils'; +import { DOWN_LABEL, MONITORS_DURATION_LABEL, UP_LABEL } from '../constants/labels'; +import { MONITOR_DURATION_US } from '../constants/field_names/synthetics'; +const SUMMARY_UP = 'summary.up'; +const SUMMARY_DOWN = 'summary.down'; + +export function getSyntheticsKPIConfig({ indexPattern }: ConfigProps): DataSeries { + return { + reportType: 'kpi-over-time', + defaultSeriesType: 'bar_stacked', + seriesTypes: [], + xAxisColumn: { + sourceField: '@timestamp', + }, + yAxisColumns: [ + { + sourceField: 'business.kpi', + operationType: 'median', + }, + ], + hasOperationType: false, + defaultFilters: ['observer.geo.name', 'monitor.type', 'tags'], + breakdowns: ['observer.geo.name', 'monitor.type'], + filters: [...buildExistsFilter('summary.up', indexPattern)], + palette: { type: 'palette', name: 'status' }, + reportDefinitions: [ + { + field: 'monitor.name', + }, + { + field: 'url.full', + }, + { + field: 'business.kpi', + custom: true, + options: [ + { + label: MONITORS_DURATION_LABEL, + field: MONITOR_DURATION_US, + id: MONITOR_DURATION_US, + columnType: OPERATION_COLUMN, + }, + { + field: SUMMARY_UP, + id: SUMMARY_UP, + label: UP_LABEL, + columnType: OPERATION_COLUMN, + }, + { + field: SUMMARY_DOWN, + id: SUMMARY_DOWN, + label: DOWN_LABEL, + columnType: OPERATION_COLUMN, + }, + ], + }, + ], + labels: { ...FieldLabels }, + }; +} diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_pings_config.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_pings_config.ts deleted file mode 100644 index 697a940f666f7..0000000000000 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/configurations/synthetics/monitor_pings_config.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ConfigProps, DataSeries } from '../../types'; -import { FieldLabels } from '../constants'; -import { buildExistsFilter } from '../utils'; -import { DOWN_LABEL, UP_LABEL } from '../constants/labels'; - -export function getMonitorPingsConfig({ seriesId, indexPattern }: ConfigProps): DataSeries { - return { - id: seriesId, - reportType: 'uptime-pings', - defaultSeriesType: 'bar_stacked', - seriesTypes: [], - xAxisColumn: { - sourceField: '@timestamp', - }, - yAxisColumns: [ - { - operationType: 'sum', - sourceField: 'summary.up', - label: UP_LABEL, - }, - { - operationType: 'sum', - sourceField: 'summary.down', - label: DOWN_LABEL, - }, - ], - yTitle: 'Pings', - hasOperationType: false, - defaultFilters: ['observer.geo.name', 'monitor.type', 'tags'], - breakdowns: ['observer.geo.name', 'monitor.type'], - filters: [...buildExistsFilter('summary.up', indexPattern)], - palette: { type: 'palette', name: 'status' }, - reportDefinitions: [ - { - field: 'monitor.name', - }, - { - field: 'url.full', - }, - ], - labels: { ...FieldLabels }, - }; -} diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx index fc0062694e0a3..487ecdb2bafcc 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx @@ -52,7 +52,7 @@ describe('ExploratoryView', () => { data: { 'ux-series': { dataType: 'ux' as const, - reportType: 'pld' as const, + reportType: 'dist' as const, breakdown: 'user_agent .name', reportDefinitions: { 'service.name': ['elastic-co'] }, time: { from: 'now-15m', to: 'now' }, diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.tsx index 7958dca6e396e..329ed20ffed3d 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.tsx @@ -16,7 +16,6 @@ import { useLensAttributes } from './hooks/use_lens_attributes'; import { EmptyView } from './components/empty_view'; import { TypedLensByValueInput } from '../../../../../lens/public'; import { useAppIndexPatternContext } from './hooks/use_app_index_pattern'; -import { ReportToDataTypeMap } from './configurations/constants'; import { SeriesBuilder } from './series_builder/series_builder'; export function ExploratoryView({ @@ -61,10 +60,10 @@ export function ExploratoryView({ }; useEffect(() => { - if (series?.reportType || series?.dataType) { - loadIndexPattern({ dataType: series?.dataType ?? ReportToDataTypeMap[series?.reportType] }); + if (series?.dataType) { + loadIndexPattern({ dataType: series?.dataType }); } - }, [series?.reportType, series?.dataType, loadIndexPattern]); + }, [series?.dataType, loadIndexPattern]); useEffect(() => { setLensAttributes(lensAttributesT); @@ -91,7 +90,7 @@ export function ExploratoryView({ timeRange={series?.time} attributes={lensAttributes} onBrushEnd={({ range }) => { - if (series?.reportType !== 'pld') { + if (series?.reportType !== 'dist') { setSeries(seriesId, { ...series, time: { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.test.tsx index ca9f2c9e73eb8..1dedc4142f174 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.test.tsx @@ -26,7 +26,7 @@ describe('ExploratoryViewHeader', function () { data: { 'uptime-pings-histogram': { dataType: 'synthetics' as const, - reportType: 'upp' as const, + reportType: 'kpi' as const, breakdown: 'monitor.status', time: { from: 'now-15m', to: 'now' }, }, diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_lens_attributes.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_lens_attributes.ts index 4e9c360745b6b..1c85bc5089b2a 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_lens_attributes.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/hooks/use_lens_attributes.ts @@ -42,7 +42,8 @@ export const useLensAttributes = ({ }: Props): TypedLensByValueInput['attributes'] | null => { const { getSeries } = useSeriesStorage(); const series = getSeries(seriesId); - const { breakdown, seriesType, operationType, reportType, reportDefinitions = {} } = series ?? {}; + const { breakdown, seriesType, operationType, reportType, dataType, reportDefinitions = {} } = + series ?? {}; const { indexPattern } = useAppIndexPatternContext(); @@ -52,8 +53,8 @@ export const useLensAttributes = ({ } const dataViewConfig = getDefaultConfigs({ - seriesId, reportType, + dataType, indexPattern, }); @@ -78,12 +79,12 @@ export const useLensAttributes = ({ return lensAttributes.getJSON(); }, [ indexPattern, - breakdown, - seriesType, - operationType, reportType, reportDefinitions, - seriesId, + dataType, series.filters, + seriesType, + operationType, + breakdown, ]); }; diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/rtl_helpers.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/rtl_helpers.tsx index 9118e49a42dfb..ff766f7e6a1cf 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/rtl_helpers.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/rtl_helpers.tsx @@ -259,7 +259,7 @@ function mockSeriesStorageContext({ }) { const mockDataSeries = data || { 'performance-distribution': { - reportType: 'pld', + reportType: 'dist', dataType: 'ux', breakdown: breakdown || 'user_agent.name', time: { from: 'now-15m', to: 'now' }, diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.test.tsx index 51529a3b1ac17..e3c1666c533ef 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.test.tsx @@ -37,7 +37,7 @@ describe('DataTypesCol', function () { data: { [seriesId]: { dataType: 'synthetics' as const, - reportType: 'upp' as const, + reportType: 'kpi' as const, breakdown: 'monitor.status', time: { from: 'now-15m', to: 'now' }, }, diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.tsx index 08e7f4ddcd3d0..3fe88de518f75 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/data_types_col.tsx @@ -11,7 +11,6 @@ import styled from 'styled-components'; import { AppDataType } from '../../types'; import { useAppIndexPatternContext } from '../../hooks/use_app_index_pattern'; import { useSeriesStorage } from '../../hooks/use_series_storage'; -import { ReportToDataTypeMap } from '../../configurations/constants'; export const dataTypes: Array<{ id: AppDataType; label: string }> = [ { id: 'synthetics', label: 'Synthetic Monitoring' }, @@ -35,7 +34,7 @@ export function DataTypesCol({ seriesId }: { seriesId: string }) { } }; - const selectedDataType = series.dataType ?? ReportToDataTypeMap[series.reportType]; + const selectedDataType = series.dataType; return ( diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/operation_type_select.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/operation_type_select.tsx index fa273f6180935..fce1383f30f34 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/operation_type_select.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/operation_type_select.tsx @@ -48,6 +48,12 @@ export function OperationTypeSelect({ defaultMessage: 'Median', }), }, + { + value: 'sum' as OperationType, + inputDisplay: i18n.translate('xpack.observability.expView.operationType.sum', { + defaultMessage: 'Sum', + }), + }, { value: '75th' as OperationType, inputDisplay: i18n.translate('xpack.observability.expView.operationType.75thPercentile', { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_breakdowns.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_breakdowns.test.tsx index f576862f18e76..805186e877d57 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_breakdowns.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_breakdowns.test.tsx @@ -15,8 +15,8 @@ import { USER_AGENT_OS } from '../../configurations/constants/elasticsearch_fiel describe('Series Builder ReportBreakdowns', function () { const seriesId = 'test-series-id'; const dataViewSeries = getDefaultConfigs({ - seriesId, - reportType: 'pld', + reportType: 'dist', + dataType: 'ux', indexPattern: mockIndexPattern, }); @@ -45,7 +45,7 @@ describe('Series Builder ReportBreakdowns', function () { expect(setSeries).toHaveBeenCalledWith(seriesId, { breakdown: USER_AGENT_OS, dataType: 'ux', - reportType: 'pld', + reportType: 'dist', time: { from: 'now-15m', to: 'now' }, }); }); @@ -67,7 +67,7 @@ describe('Series Builder ReportBreakdowns', function () { expect(setSeries).toHaveBeenCalledWith(seriesId, { breakdown: undefined, dataType: 'ux', - reportType: 'pld', + reportType: 'dist', time: { from: 'now-15m', to: 'now' }, }); }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_definition_col.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_definition_col.test.tsx index fdf6633c0ddb5..8738235f0c54b 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_definition_col.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_definition_col.test.tsx @@ -22,16 +22,16 @@ describe('Series Builder ReportDefinitionCol', function () { const seriesId = 'test-series-id'; const dataViewSeries = getDefaultConfigs({ - seriesId, - reportType: 'pld', + reportType: 'dist', indexPattern: mockIndexPattern, + dataType: 'ux', }); const initSeries = { data: { [seriesId]: { dataType: 'ux' as const, - reportType: 'pld' as const, + reportType: 'dist' as const, time: { from: 'now-30d', to: 'now' }, reportDefinitions: { [SERVICE_NAME]: ['elastic-co'] }, }, @@ -81,7 +81,7 @@ describe('Series Builder ReportDefinitionCol', function () { expect(setSeries).toHaveBeenCalledWith(seriesId, { dataType: 'ux', reportDefinitions: {}, - reportType: 'pld', + reportType: 'dist', time: { from: 'now-30d', to: 'now' }, }); }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_filters.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_filters.test.tsx index dc2dc629cc121..7ca947fed0bc9 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_filters.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_filters.test.tsx @@ -15,9 +15,9 @@ describe('Series Builder ReportFilters', function () { const seriesId = 'test-series-id'; const dataViewSeries = getDefaultConfigs({ - seriesId, - reportType: 'pld', + reportType: 'dist', indexPattern: mockIndexPattern, + dataType: 'ux', }); it('should render properly', function () { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_types_col.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_types_col.test.tsx index c721a2fa2fe77..f36d64ca5bbbd 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_types_col.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/columns/report_types_col.test.tsx @@ -34,13 +34,13 @@ describe('ReportTypesCol', function () { ); - fireEvent.click(screen.getByText(/monitor duration/i)); + fireEvent.click(screen.getByText(/KPI over time/i)); expect(setSeries).toHaveBeenCalledWith(seriesId, { breakdown: 'user_agent.name', dataType: 'ux', reportDefinitions: {}, - reportType: 'upd', + reportType: 'kpi', time: { from: 'now-15m', to: 'now' }, }); expect(setSeries).toHaveBeenCalledTimes(1); @@ -51,7 +51,7 @@ describe('ReportTypesCol', function () { data: { [NEW_SERIES_KEY]: { dataType: 'synthetics' as const, - reportType: 'upp' as const, + reportType: 'kpi' as const, breakdown: 'monitor.status', time: { from: 'now-15m', to: 'now' }, }, @@ -64,7 +64,7 @@ describe('ReportTypesCol', function () { ); const button = screen.getByRole('button', { - name: /pings histogram/i, + name: /KPI over time/i, }); expect(button.classList).toContain('euiButton--fill'); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/series_builder.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/series_builder.tsx index 32f1fb7f7c43b..e24d246d60e58 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/series_builder.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_builder/series_builder.tsx @@ -21,29 +21,17 @@ import { getDefaultConfigs } from '../configurations/default_configs'; export const ReportTypes: Record> = { synthetics: [ - { id: 'upd', label: 'Monitor duration' }, - { id: 'upp', label: 'Pings histogram' }, + { id: 'kpi', label: 'KPI over time' }, + { id: 'dist', label: 'Performance distribution' }, ], ux: [ - { id: 'pld', label: 'Performance distribution' }, { id: 'kpi', label: 'KPI over time' }, + { id: 'dist', label: 'Performance distribution' }, { id: 'cwv', label: 'Core Web Vitals' }, ], - apm: [ - { id: 'svl', label: 'Latency' }, - { id: 'tpt', label: 'Throughput' }, - ], - infra_logs: [ - { - id: 'logs', - label: 'Logs Frequency', - }, - ], - infra_metrics: [ - { id: 'cpu', label: 'CPU usage' }, - { id: 'mem', label: 'Memory usage' }, - { id: 'nwk', label: 'Network activity' }, - ], + apm: [], + infra_logs: [], + infra_metrics: [], }; export function SeriesBuilder({ @@ -72,7 +60,7 @@ export function SeriesBuilder({ const getDataViewSeries = () => { return getDefaultConfigs({ - seriesId, + dataType, indexPattern, reportType: reportType!, }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_date_picker/series_date_picker.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_date_picker/series_date_picker.test.tsx index 0edc4330ef97a..2b46bb9a8cd62 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_date_picker/series_date_picker.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_date_picker/series_date_picker.test.tsx @@ -17,7 +17,7 @@ describe('SeriesDatePicker', function () { data: { 'uptime-pings-histogram': { dataType: 'synthetics' as const, - reportType: 'upp' as const, + reportType: 'dist' as const, breakdown: 'monitor.status', time: { from: 'now-30m', to: 'now' }, }, @@ -32,7 +32,7 @@ describe('SeriesDatePicker', function () { const initSeries = { data: { 'uptime-pings-histogram': { - reportType: 'upp' as const, + reportType: 'kpi' as const, dataType: 'synthetics' as const, breakdown: 'monitor.status', }, @@ -46,7 +46,7 @@ describe('SeriesDatePicker', function () { expect(setSeries1).toHaveBeenCalledWith('uptime-pings-histogram', { breakdown: 'monitor.status', dataType: 'synthetics' as const, - reportType: 'upp' as const, + reportType: 'kpi' as const, time: DEFAULT_TIME, }); }); @@ -56,7 +56,7 @@ describe('SeriesDatePicker', function () { data: { 'uptime-pings-histogram': { dataType: 'synthetics' as const, - reportType: 'upp' as const, + reportType: 'kpi' as const, breakdown: 'monitor.status', time: { from: 'now-30m', to: 'now' }, }, @@ -79,7 +79,7 @@ describe('SeriesDatePicker', function () { expect(setSeries).toHaveBeenCalledWith('series-id', { breakdown: 'monitor.status', dataType: 'synthetics', - reportType: 'upp', + reportType: 'kpi', time: { from: 'now/d', to: 'now/d' }, }); expect(setSeries).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/chart_edit_options.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/chart_edit_options.tsx index 4bef3e8f71821..a0d2fd86482a5 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/chart_edit_options.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/chart_edit_options.tsx @@ -23,7 +23,7 @@ export function ChartEditOptions({ series, seriesId, breakdowns }: Props) { - + ); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/breakdowns.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/breakdowns.test.tsx index 0ce9db73f92b1..1d552486921e1 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/breakdowns.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/breakdowns.test.tsx @@ -9,15 +9,14 @@ import React from 'react'; import { fireEvent, screen } from '@testing-library/react'; import { Breakdowns } from './breakdowns'; import { mockIndexPattern, render } from '../../rtl_helpers'; -import { NEW_SERIES_KEY } from '../../hooks/use_series_storage'; import { getDefaultConfigs } from '../../configurations/default_configs'; import { USER_AGENT_OS } from '../../configurations/constants/elasticsearch_fieldnames'; describe('Breakdowns', function () { const dataViewSeries = getDefaultConfigs({ - reportType: 'pld', + reportType: 'dist', indexPattern: mockIndexPattern, - seriesId: NEW_SERIES_KEY, + dataType: 'ux', }); it('should render properly', async function () { @@ -53,7 +52,7 @@ describe('Breakdowns', function () { expect(setSeries).toHaveBeenCalledWith('series-id', { breakdown: 'user_agent.name', dataType: 'ux', - reportType: 'pld', + reportType: 'dist', time: { from: 'now-15m', to: 'now' }, }); expect(setSeries).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/chart_options.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/chart_options.tsx index 975817a8417de..08664ac75eb8d 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/chart_options.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/chart_options.tsx @@ -13,17 +13,18 @@ import { SeriesChartTypesSelect } from '../../series_builder/columns/chart_types interface Props { series: DataSeries; + seriesId: string; } -export function ChartOptions({ series }: Props) { +export function ChartOptions({ series, seriesId }: Props) { return ( - + {series.hasOperationType && ( - + )} diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx index 5374fc33093a1..086a1d4341bbc 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/columns/series_actions.tsx @@ -10,7 +10,6 @@ import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { RemoveSeries } from './remove_series'; import { NEW_SERIES_KEY, useSeriesStorage } from '../../hooks/use_series_storage'; -import { ReportToDataTypeMap } from '../../configurations/constants'; interface Props { seriesId: string; @@ -21,7 +20,7 @@ export function SeriesActions({ seriesId }: Props) { const onEdit = () => { removeSeries(seriesId); - setSeries(NEW_SERIES_KEY, { ...series, dataType: ReportToDataTypeMap[series.reportType] }); + setSeries(NEW_SERIES_KEY, { ...series }); }; return ( diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/selected_filters.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/selected_filters.test.tsx index 2919714ac0cd4..8363b6b0eadfd 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/selected_filters.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/selected_filters.test.tsx @@ -10,16 +10,15 @@ import { screen, waitFor } from '@testing-library/react'; import { mockAppIndexPattern, mockIndexPattern, render } from '../rtl_helpers'; import { SelectedFilters } from './selected_filters'; import { getDefaultConfigs } from '../configurations/default_configs'; -import { NEW_SERIES_KEY } from '../hooks/use_series_storage'; import { USER_AGENT_NAME } from '../configurations/constants/elasticsearch_fieldnames'; describe('SelectedFilters', function () { mockAppIndexPattern(); const dataViewSeries = getDefaultConfigs({ - reportType: 'pld', + reportType: 'dist', indexPattern: mockIndexPattern, - seriesId: NEW_SERIES_KEY, + dataType: 'ux', }); it('should render properly', async function () { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/series_editor.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/series_editor.tsx index 6e513fcd2fec9..79218aa111f16 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/series_editor.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/series_editor/series_editor.tsx @@ -18,6 +18,11 @@ import { useAppIndexPatternContext } from '../hooks/use_app_index_pattern'; import { SeriesActions } from './columns/series_actions'; import { ChartEditOptions } from './chart_edit_options'; +interface EditItem { + seriesConfig: DataSeries; + id: string; +} + export function SeriesEditor() { const { allSeries, firstSeriesId } = useSeriesStorage(); @@ -43,8 +48,8 @@ export function SeriesEditor() { }), field: 'defaultFilters', width: '15%', - render: (defaultFilters: string[], series: DataSeries) => ( - + render: (defaultFilters: string[], { id, seriesConfig }: EditItem) => ( + ), }, { @@ -53,8 +58,8 @@ export function SeriesEditor() { }), field: 'breakdowns', width: '25%', - render: (val: string[], item: DataSeries) => ( - + render: (val: string[], item: EditItem) => ( + ), }, { @@ -69,7 +74,7 @@ export function SeriesEditor() { width: '20%', field: 'id', align: 'right' as const, - render: (val: string, item: DataSeries) => , + render: (val: string, item: EditItem) => , }, { name: i18n.translate('xpack.observability.expView.seriesEditor.actions', { @@ -78,7 +83,7 @@ export function SeriesEditor() { align: 'center' as const, width: '10%', field: 'id', - render: (val: string, item: DataSeries) => , + render: (val: string, item: EditItem) => , }, ] : []), @@ -86,20 +91,21 @@ export function SeriesEditor() { const allSeriesKeys = Object.keys(allSeries); - const items: DataSeries[] = []; + const items: EditItem[] = []; const { indexPattern } = useAppIndexPatternContext(); allSeriesKeys.forEach((seriesKey) => { const series = allSeries[seriesKey]; if (series.reportType && indexPattern) { - items.push( - getDefaultConfigs({ + items.push({ + id: seriesKey, + seriesConfig: getDefaultConfigs({ indexPattern, reportType: series.reportType, - seriesId: seriesKey, - }) - ); + dataType: series.dataType, + }), + }); } }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/types.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/types.ts index 87772532f410d..98605dfdb4ca3 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/types.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/types.ts @@ -20,17 +20,9 @@ import { IIndexPattern } from '../../../../../../../src/plugins/data/common/inde import { ExistsFilter } from '../../../../../../../src/plugins/data/common/es_query/filters'; export const ReportViewTypes = { - pld: 'page-load-dist', - kpi: 'kpi-trends', + dist: 'data-distribution', + kpi: 'kpi-over-time', cwv: 'core-web-vitals', - upd: 'uptime-duration', - upp: 'uptime-pings', - svl: 'service-latency', - tpt: 'service-throughput', - logs: 'logs-frequency', - cpu: 'cpu-usage', - mem: 'memory-usage', - nwk: 'network-activity', } as const; type ValueOf = T[keyof T]; @@ -60,7 +52,6 @@ export interface ReportDefinition { export interface DataSeries { reportType: ReportViewType; - id: string; xAxisColumn: Partial | Partial; yAxisColumns: Array>; @@ -100,7 +91,6 @@ export interface UrlFilter { } export interface ConfigProps { - seriesId: string; indexPattern: IIndexPattern; } diff --git a/x-pack/plugins/uptime/public/components/common/charts/ping_histogram.tsx b/x-pack/plugins/uptime/public/components/common/charts/ping_histogram.tsx index c864462c76915..35161561a23fe 100644 --- a/x-pack/plugins/uptime/public/components/common/charts/ping_histogram.tsx +++ b/x-pack/plugins/uptime/public/components/common/charts/ping_histogram.tsx @@ -191,7 +191,7 @@ export const PingHistogramComponent: React.FC = ({ { 'pings-over-time': { dataType: 'synthetics', - reportType: 'upp', + reportType: 'kpi', time: { from: dateRangeStart, to: dateRangeEnd }, ...(monitorId ? { filters: [{ field: 'monitor.id', values: [monitorId] }] } : {}), }, diff --git a/x-pack/plugins/uptime/public/components/monitor/monitor_duration/monitor_duration_container.tsx b/x-pack/plugins/uptime/public/components/monitor/monitor_duration/monitor_duration_container.tsx index 6cfe8f61a1843..377d7a8fa35d4 100644 --- a/x-pack/plugins/uptime/public/components/monitor/monitor_duration/monitor_duration_container.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/monitor_duration/monitor_duration_container.tsx @@ -56,7 +56,7 @@ export const MonitorDuration: React.FC = ({ monitorId }) => { const exploratoryViewLink = createExploratoryViewUrl( { [`monitor-duration`]: { - reportType: 'upd', + reportType: 'kpi', time: { from: dateRangeStart, to: dateRangeEnd }, reportDefinitions: { 'monitor.id': [monitorId] as string[], From dbe6bfd073f024477263d2adad91d1f98d69b8ea Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Mon, 14 Jun 2021 16:39:42 +0200 Subject: [PATCH 03/91] [Index Patterns] Cover field editor with a11y tests (#101888) --- test/accessibility/apps/management.ts | 22 ++++++++++++++ test/functional/page_objects/settings_page.ts | 29 ++++++++++++++----- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/test/accessibility/apps/management.ts b/test/accessibility/apps/management.ts index 692b140ade7ee..e71f6bb3ebfee 100644 --- a/test/accessibility/apps/management.ts +++ b/test/accessibility/apps/management.ts @@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); const a11y = getService('a11y'); + const testSubjects = getService('testSubjects'); describe('Management', () => { before(async () => { @@ -43,6 +44,27 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await a11y.testAppSnapshot(); }); + it('Index pattern field editor - initial view', async () => { + await PageObjects.settings.clickAddField(); + await a11y.testAppSnapshot(); + }); + + it('Index pattern field editor - all options shown', async () => { + await PageObjects.settings.setFieldName('test'); + await PageObjects.settings.setFieldType('Keyword'); + await PageObjects.settings.setFieldScript("emit('hello world')"); + await PageObjects.settings.toggleRow('formatRow'); + await PageObjects.settings.setFieldFormat('string'); + await PageObjects.settings.toggleRow('customLabelRow'); + await PageObjects.settings.setCustomLabel('custom label'); + await testSubjects.click('toggleAdvancedSetting'); + + await a11y.testAppSnapshot(); + + await testSubjects.click('euiFlyoutCloseButton'); + await PageObjects.settings.closeIndexPatternFieldEditor(); + }); + it('Open create index pattern wizard', async () => { await PageObjects.settings.clickKibanaIndexPatterns(); await PageObjects.settings.clickAddNewIndexPatternButton(); diff --git a/test/functional/page_objects/settings_page.ts b/test/functional/page_objects/settings_page.ts index 7d7da79b4a397..88951bb04c956 100644 --- a/test/functional/page_objects/settings_page.ts +++ b/test/functional/page_objects/settings_page.ts @@ -563,11 +563,8 @@ export class SettingsPageObject extends FtrService { async setFieldScript(script: string) { this.log.debug('set script = ' + script); - const formatRow = await this.testSubjects.find('valueRow'); - const formatRowToggle = (await formatRow.findAllByCssSelector('[data-test-subj="toggle"]'))[0]; - - await formatRowToggle.click(); - const getMonacoTextArea = async () => (await formatRow.findAllByCssSelector('textarea'))[0]; + const valueRow = await this.toggleRow('valueRow'); + const getMonacoTextArea = async () => (await valueRow.findAllByCssSelector('textarea'))[0]; this.retry.waitFor('monaco editor is ready', async () => !!(await getMonacoTextArea())); const monacoTextArea = await getMonacoTextArea(); await monacoTextArea.focus(); @@ -576,8 +573,8 @@ export class SettingsPageObject extends FtrService { async changeFieldScript(script: string) { this.log.debug('set script = ' + script); - const formatRow = await this.testSubjects.find('valueRow'); - const getMonacoTextArea = async () => (await formatRow.findAllByCssSelector('textarea'))[0]; + const valueRow = await this.testSubjects.find('valueRow'); + const getMonacoTextArea = async () => (await valueRow.findAllByCssSelector('textarea'))[0]; this.retry.waitFor('monaco editor is ready', async () => !!(await getMonacoTextArea())); const monacoTextArea = await getMonacoTextArea(); await monacoTextArea.focus(); @@ -622,6 +619,24 @@ export class SettingsPageObject extends FtrService { ); } + async toggleRow(rowTestSubj: string) { + this.log.debug('toggling tow = ' + rowTestSubj); + const row = await this.testSubjects.find(rowTestSubj); + const rowToggle = (await row.findAllByCssSelector('[data-test-subj="toggle"]'))[0]; + await rowToggle.click(); + return row; + } + + async setCustomLabel(label: string) { + this.log.debug('set custom label = ' + label); + await ( + await this.testSubjects.findDescendant( + 'input', + await this.testSubjects.find('customLabelRow') + ) + ).type(label); + } + async setScriptedFieldUrlType(type: string) { this.log.debug('set scripted field Url type = ' + type); await this.find.clickByCssSelector( From b430a985c2e392201cc4afd1bdb14e111320231e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 07:43:24 -0700 Subject: [PATCH 04/91] Update dependency @elastic/charts to v30.1.0 (master) (#101683) --- package.json | 2 +- .../common/charts/__snapshots__/donut_chart.test.tsx.snap | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index b4f9109503261..513352db3f81b 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "dependencies": { "@elastic/apm-rum": "^5.6.1", "@elastic/apm-rum-react": "^1.2.5", - "@elastic/charts": "30.0.0", + "@elastic/charts": "30.1.0", "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.13", "@elastic/ems-client": "7.13.0", diff --git a/x-pack/plugins/uptime/public/components/common/charts/__snapshots__/donut_chart.test.tsx.snap b/x-pack/plugins/uptime/public/components/common/charts/__snapshots__/donut_chart.test.tsx.snap index b689ca7ff56f0..1403e6c3e52b1 100644 --- a/x-pack/plugins/uptime/public/components/common/charts/__snapshots__/donut_chart.test.tsx.snap +++ b/x-pack/plugins/uptime/public/components/common/charts/__snapshots__/donut_chart.test.tsx.snap @@ -137,7 +137,7 @@ exports[`DonutChart component passes correct props without errors for valid prop "opacity": 1, }, "rectBorder": Object { - "strokeWidth": 0, + "strokeWidth": 1, "visible": false, }, }, diff --git a/yarn.lock b/yarn.lock index 9543f209d6849..0e2b953389c10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1359,10 +1359,10 @@ dependencies: object-hash "^1.3.0" -"@elastic/charts@30.0.0": - version "30.0.0" - resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-30.0.0.tgz#e19ad8b94928aa9bac5d7facc488fa69b683ff1e" - integrity sha512-r22T2dlW3drEmrIx6JNlOOzSp0JCkI/qbIfmvzdMBu8E8hITkJTaXJaLsNN4mz9EvR9jEM8XQQPQXOFKJhWixw== +"@elastic/charts@30.1.0": + version "30.1.0" + resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-30.1.0.tgz#eb9b3348c149ce13f74876738a9d2899b6b10067" + integrity sha512-aUfXRQYQopm+6O48tEO0v/w6fETYORGiSPBRtqlq5xPncZGhGnQbgXVNQsPngYqapnKpOupXAqzjopF+RJ4QWg== dependencies: "@popperjs/core" "^2.4.0" chroma-js "^2.1.0" From 8424a925332d5bd7d93148fbab72d1f518933485 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Mon, 14 Jun 2021 10:50:20 -0400 Subject: [PATCH 05/91] [ILM] Migrate to new page layout (#101927) --- .../__snapshots__/policy_table.test.tsx.snap | 98 +++-- .../edit_policy/edit_policy.container.tsx | 70 ++-- .../sections/edit_policy/edit_policy.tsx | 351 +++++++++--------- .../policy_table/policy_table.container.tsx | 70 ++-- .../sections/policy_table/policy_table.tsx | 101 +++-- 5 files changed, 339 insertions(+), 351 deletions(-) diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap b/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap index 8d839e196916b..556ac35d0565e 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap +++ b/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap @@ -46,71 +46,67 @@ Array [ `; exports[`policy table should show empty state when there are not any policies 1`] = ` -
+
+ - -
- -

- Create your first index lifecycle policy -

-
-
-

- An index lifecycle policy helps you manage your indices as they age. -

-
- + Create your first index lifecycle policy +
-
+ +
+
+ -
+ +
-
+ `; exports[`policy table should sort when linked indices header is clicked 1`] = ` diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx index fe82b20093072..07c2228863b81 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx @@ -7,7 +7,7 @@ import React, { useEffect } from 'react'; import { RouteComponentProps } from 'react-router-dom'; -import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner, EuiPageContent } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { MIN_SEARCHABLE_SNAPSHOT_LICENSE } from '../../../../common/constants'; @@ -52,43 +52,47 @@ export const EditPolicy: React.FunctionComponent} - body={ - - } - /> + + } + body={ + + } + /> + ); } if (error || !policies) { const { statusCode, message } = error ? error : { statusCode: '', message: '' }; return ( - - - - } - body={ -

- {message} ({statusCode}) -

- } - actions={ - - - - } - /> + + + + + } + body={ +

+ {message} ({statusCode}) +

+ } + actions={ + + + + } + /> + ); } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx index d7368249d76e8..172e8259b87af 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx @@ -19,15 +19,10 @@ import { EuiFlexItem, EuiFormRow, EuiHorizontalRule, - EuiPage, - EuiPageBody, - EuiPageContent, - EuiPageContentHeader, - EuiPageContentHeaderSection, EuiSpacer, EuiSwitch, EuiText, - EuiTitle, + EuiPageHeader, } from '@elastic/eui'; import { TextField, useForm, useFormData, useKibana } from '../../../shared_imports'; @@ -153,201 +148,199 @@ export const EditPolicy: React.FunctionComponent = ({ history }) => { }; return ( - - - - - - -

- {isNewPolicy - ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { - defaultMessage: 'Create policy', - }) - : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { - defaultMessage: 'Edit policy {originalPolicyName}', - values: { originalPolicyName }, - })} -

- - - - + <> + + {isNewPolicy + ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { + defaultMessage: 'Create policy', + }) + : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { + defaultMessage: 'Edit policy {originalPolicyName}', + values: { originalPolicyName }, + })} + + } + bottomBorder + rightSideItems={[ + + + , + ]} + /> + + + +
+ {isNewPolicy ? null : ( + + +

+ + + + .{' '} - - - - - {isNewPolicy ? null : ( - - -

- - - - .{' '} - -

- - - - - { - setSaveAsNew(e.target.checked); - }} - label={ - - - - } - /> - - - )} - - {saveAsNew || isNewPolicy ? ( - - path={policyNamePath} - config={{ - label: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.policyNameLabel', { - defaultMessage: 'Policy name', - }), - helpText: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.validPolicyNameMessage', - { - defaultMessage: - 'A policy name cannot start with an underscore and cannot contain a comma or a space.', - } - ), - validations: policyNameValidations, - }} - component={TextField} - componentProps={{ - fullWidth: false, - euiFieldProps: { - 'data-test-subj': 'policyNameField', - }, + /> +

+ + + + + { + setSaveAsNew(e.target.checked); }} + label={ + + + + } /> - ) : null} - - - - - - - -
- - - - - + + + )} + + {saveAsNew || isNewPolicy ? ( + + path={policyNamePath} + config={{ + label: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.policyNameLabel', { + defaultMessage: 'Policy name', + }), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.validPolicyNameMessage', + { + defaultMessage: + 'A policy name cannot start with an underscore and cannot contain a comma or a space.', + } + ), + validations: policyNameValidations, + }} + component={TextField} + componentProps={{ + fullWidth: false, + euiFieldProps: { + 'data-test-subj': 'policyNameField', + }, + }} + /> + ) : null} + + + + + + + +
+ + + + + + + + + {isAllowedByLicense && ( + <> - + + + )} - {isAllowedByLicense && ( - <> - - - - )} - - {/* We can't add the here as it breaks the layout + {/* We can't add the here as it breaks the layout and makes the connecting line go further that it needs to. There is an issue in EUI to fix this (https://github.com/elastic/eui/issues/4492) */} - -
+ +
- + - - - - - - - - {saveAsNew ? ( - - ) : ( - - )} - - - - - - - - - - + + + + - - {isShowingPolicyJsonFlyout ? ( + + {saveAsNew ? ( ) : ( )} + + + + + + + - {isShowingPolicyJsonFlyout ? ( - setIsShowingPolicyJsonFlyout(false)} - /> - ) : null} - -
- - + + + {isShowingPolicyJsonFlyout ? ( + + ) : ( + + )} + + + + + {isShowingPolicyJsonFlyout ? ( + setIsShowingPolicyJsonFlyout(false)} + /> + ) : null} + + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx index 7cf50de0ee999..deac2bb239d30 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx @@ -8,7 +8,7 @@ import React, { useEffect } from 'react'; import { ApplicationStart } from 'kibana/public'; import { RouteComponentProps } from 'react-router-dom'; -import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner, EuiPageContent } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { PolicyTable as PresentationComponent } from './policy_table'; import { useKibana } from '../../../shared_imports'; @@ -33,43 +33,47 @@ export const PolicyTable: React.FunctionComponent = if (isLoading) { return ( - } - body={ - - } - /> + + } + body={ + + } + /> + ); } if (error) { const { statusCode, message } = error ? error : { statusCode: '', message: '' }; return ( - - - - } - body={ -

- {message} ({statusCode}) -

- } - actions={ - - - - } - /> + + + + + } + body={ +

+ {message} ({statusCode}) +

+ } + actions={ + + + + } + /> + ); } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx index 3b0f815800ba3..ba89d6c895d93 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx @@ -16,9 +16,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, - EuiTitle, - EuiText, - EuiPageBody, + EuiPageHeader, EuiPageContent, } from '@elastic/eui'; import { ApplicationStart } from 'kibana/public'; @@ -119,67 +117,60 @@ export const PolicyTable: React.FunctionComponent = ({ ); } else { return ( - - - + + + + + } + body={ + +

- - } - body={ - -

- -

- - } - actions={createPolicyButton} - /> - - +

+ + } + actions={createPolicyButton} + /> + ); } return ( - - - {confirmModal} + <> + {confirmModal} - - - -

- -

- - - {createPolicyButton} - - - -

+ -

- + + } + description={ + + } + bottomBorder + rightSideItems={[createPolicyButton]} + /> - - {content} - - + + + {content} + ); }; From 277212df0b919b57d3dadd40bf059eb908b44fd0 Mon Sep 17 00:00:00 2001 From: Stacey Gammon Date: Mon, 14 Jun 2021 11:04:27 -0400 Subject: [PATCH 06/91] Update building_a_plugin.mdx (#101921) * Update building_a_plugin.mdx * put back the numbers on the same line * Add info about requiredBundles * Fix numbers again * Update dev_docs/tutorials/building_a_plugin.mdx Co-authored-by: Mikhail Shustov * Update dev_docs/tutorials/building_a_plugin.mdx Co-authored-by: Mikhail Shustov Co-authored-by: Mikhail Shustov Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- dev_docs/tutorials/building_a_plugin.mdx | 79 ++++++++++++++++++------ 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/dev_docs/tutorials/building_a_plugin.mdx b/dev_docs/tutorials/building_a_plugin.mdx index cee5a9a399de5..e751ce7d01b16 100644 --- a/dev_docs/tutorials/building_a_plugin.mdx +++ b/dev_docs/tutorials/building_a_plugin.mdx @@ -4,7 +4,7 @@ slug: /kibana-dev-docs/tutorials/build-a-plugin title: Kibana plugin tutorial summary: Anatomy of a Kibana plugin and how to build one date: 2021-02-05 -tags: ['kibana','onboarding', 'dev', 'tutorials'] +tags: ['kibana', 'onboarding', 'dev', 'tutorials'] --- Prereading material: @@ -14,7 +14,7 @@ Prereading material: ## The anatomy of a plugin Plugins are defined as classes and present themselves to Kibana through a simple wrapper function. A plugin can have browser-side code, server-side code, -or both. There is no architectural difference between a plugin in the browser and a plugin on the server. In both places, you describe your plugin similarly, +or both. There is no architectural difference between a plugin in the browser and a plugin on the server. In both places, you describe your plugin similarly, and you interact with Core and other plugins in the same way. The basic file structure of a Kibana plugin named demo that has both client-side and server-side code would be: @@ -33,7 +33,7 @@ plugins/ index.ts [6] ``` -### [1] kibana.json +### [1] kibana.json `kibana.json` is a static manifest file that is used to identify the plugin and to specify if this plugin has server-side code, browser-side code, or both: @@ -42,14 +42,33 @@ plugins/ "id": "demo", "version": "kibana", "server": true, - "ui": true + "ui": true, + "owner": { [1] + "name": "App Services", + "githubTeam": "kibana-app-services" + }, + "description": "This plugin extends Kibana by doing xyz, and allows other plugins to extend Kibana by offering abc functionality. It also exposes some helper utilities that do efg", [2] + "requiredPlugins": ["data"], [3] + "optionalPlugins": ["alerting"] [4] + "requiredBundles": ["anotherPlugin"] [5] } ``` +[1], [2]: Every internal plugin should fill in the owner and description properties. + +[3], [4]: Any plugin that you have a dependency on should be listed in `requiredPlugins` or `optionalPlugins`. Doing this will ensure that you have access to that plugin's start and setup contract inside your own plugin's start and setup lifecycle methods. If a plugin you optionally depend on is not installed or disabled, it will be undefined if you try to access it. If a plugin you require is not installed or disabled, kibana will fail to build. + +[5]: Don't worry too much about getting 5 right. The build optimizer will complain if any of these values are incorrect. + + + + You don't need to declare a dependency on a plugin if you only wish to access its types. + + ### [2] public/index.ts -`public/index.ts` is the entry point into the client-side code of this plugin. It must export a function named plugin, which will receive a standard set of - core capabilities as an argument. It should return an instance of its plugin class for Kibana to load. +`public/index.ts` is the entry point into the client-side code of this plugin. Everything exported from this file will be a part of the plugins . If the plugin only exists to export static utilities, consider using a package. Otherwise, this file must export a function named plugin, which will receive a standard set of +core capabilities as an argument. It should return an instance of its plugin class for Kibana to load. ``` import type { PluginInitializerContext } from 'kibana/server'; @@ -60,13 +79,32 @@ export function plugin(initializerContext: PluginInitializerContext) { } ``` + + +1. When possible, use + +``` +export type { AType } from '...'` +``` + +instead of + +``` +export { AType } from '...'`. +``` + +Using the non-`type` variation will increase the bundle size unnecessarily and may unwillingly provide access to the implementation of `AType` class. + +2. Don't use `export *` in these top level index.ts files + + + ### [3] public/plugin.ts `public/plugin.ts` is the client-side plugin definition itself. Technically speaking, it does not need to be a class or even a separate file from the entry - point, but all plugins at Elastic should be consistent in this way. +point, but all plugins at Elastic should be consistent in this way. - - ```ts +```ts import type { Plugin, PluginInitializerContext, CoreSetup, CoreStart } from 'kibana/server'; export class DemoPlugin implements Plugin { @@ -84,10 +122,9 @@ export class DemoPlugin implements Plugin { // called when plugin is torn down during Kibana's shutdown sequence } } - ``` - +``` -### [4] server/index.ts +### [4] server/index.ts `server/index.ts` is the entry-point into the server-side code of this plugin. It is identical in almost every way to the client-side entry-point: @@ -115,7 +152,7 @@ export class DemoPlugin implements Plugin { } ``` -Kibana does not impose any technical restrictions on how the the internals of a plugin are architected, though there are certain +Kibana does not impose any technical restrictions on how the the internals of a plugin are architected, though there are certain considerations related to how plugins integrate with core APIs and APIs exposed by other plugins that may greatly impact how they are built. ### [6] common/index.ts @@ -124,8 +161,8 @@ considerations related to how plugins integrate with core APIs and APIs exposed ## How plugin's interact with each other, and Core -The lifecycle-specific contracts exposed by core services are always passed as the first argument to the equivalent lifecycle function in a plugin. -For example, the core http service exposes a function createRouter to all plugin setup functions. To use this function to register an HTTP route handler, +The lifecycle-specific contracts exposed by core services are always passed as the first argument to the equivalent lifecycle function in a plugin. +For example, the core http service exposes a function createRouter to all plugin setup functions. To use this function to register an HTTP route handler, a plugin just accesses it off of the first argument: ```ts @@ -135,14 +172,16 @@ export class DemoPlugin { public setup(core: CoreSetup) { const router = core.http.createRouter(); // handler is called when '/path' resource is requested with `GET` method - router.get({ path: '/path', validate: false }, (context, req, res) => res.ok({ content: 'ok' })); + router.get({ path: '/path', validate: false }, (context, req, res) => + res.ok({ content: 'ok' }) + ); } } ``` Unlike core, capabilities exposed by plugins are not automatically injected into all plugins. Instead, if a plugin wishes to use the public interface provided by another plugin, it must first declare that plugin as a - dependency in it’s kibana.json manifest file. +dependency in it’s kibana.json manifest file. ** foobar plugin.ts: ** @@ -174,8 +213,8 @@ export class MyPlugin implements Plugin { } } ``` -[1] We highly encourage plugin authors to explicitly declare public interfaces for their plugins. +[1] We highly encourage plugin authors to explicitly declare public interfaces for their plugins. ** demo kibana.json** @@ -194,7 +233,7 @@ With that specified in the plugin manifest, the appropriate interfaces are then import type { CoreSetup, CoreStart } from 'kibana/server'; import type { FoobarPluginSetup, FoobarPluginStart } from '../../foobar/server'; -interface DemoSetupPlugins { [1] +interface DemoSetupPlugins { [1] foobar: FoobarPluginSetup; } @@ -218,7 +257,7 @@ export class DemoPlugin { public stop() {} } ``` - + [1] The interface for plugin’s dependencies must be manually composed. You can do this by importing the appropriate type from the plugin and constructing an interface where the property name is the plugin’s ID. [2] These manually constructed types should then be used to specify the type of the second argument to the plugin. From 0e7d4fed93b98d4b746f30141535a852c8b42f69 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Mon, 14 Jun 2021 17:14:02 +0200 Subject: [PATCH 07/91] Fix delayed status API updates in alerting and task_manager (#101778) --- test/functional/apps/bundles/index.js | 9 ++++++ .../server_integration/http/platform/cache.ts | 11 +++++++ .../alerting/server/health/get_state.test.ts | 20 +++++------- .../alerting/server/health/get_state.ts | 13 +++++++- x-pack/plugins/alerting/server/plugin.ts | 31 ++++++++++++------- x-pack/plugins/task_manager/server/plugin.ts | 14 ++++----- 6 files changed, 65 insertions(+), 33 deletions(-) diff --git a/test/functional/apps/bundles/index.js b/test/functional/apps/bundles/index.js index d13e74dd4eed9..577035a8c343c 100644 --- a/test/functional/apps/bundles/index.js +++ b/test/functional/apps/bundles/index.js @@ -18,6 +18,15 @@ export default function ({ getService }) { let buildNum; before(async () => { + // Wait for status to become green + let status; + const start = Date.now(); + do { + const resp = await supertest.get('/api/status'); + status = resp.status; + // Stop polling once status stabilizes OR once 40s has passed + } while (status !== 200 && Date.now() - start < 40_000); + const resp = await supertest.get('/api/status').expect(200); buildNum = resp.body.version.build_number; }); diff --git a/test/server_integration/http/platform/cache.ts b/test/server_integration/http/platform/cache.ts index 2c1aa90e963e2..a33f916cf1c4b 100644 --- a/test/server_integration/http/platform/cache.ts +++ b/test/server_integration/http/platform/cache.ts @@ -12,6 +12,17 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); describe('kibana server cache-control', () => { + before(async () => { + // Wait for status to become green + let status; + const start = Date.now(); + do { + const resp = await supertest.get('/api/status'); + status = resp.status; + // Stop polling once status stabilizes OR once 40s has passed + } while (status !== 200 && Date.now() - start < 40_000); + }); + it('properly marks responses as private, with directives to disable caching', async () => { await supertest .get('/api/status') diff --git a/x-pack/plugins/alerting/server/health/get_state.test.ts b/x-pack/plugins/alerting/server/health/get_state.test.ts index 96627e10fb3bd..2dddf81e3b766 100644 --- a/x-pack/plugins/alerting/server/health/get_state.test.ts +++ b/x-pack/plugins/alerting/server/health/get_state.test.ts @@ -58,7 +58,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { const mockTaskManager = taskManagerMock.createStart(); mockTaskManager.get.mockResolvedValue(getHealthCheckTask()); const pollInterval = 100; - const halfInterval = Math.floor(pollInterval / 2); getHealthStatusStream( mockTaskManager, @@ -77,16 +76,15 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { pollInterval ).subscribe(); - // shouldn't fire before poll interval passes + // should fire before poll interval passes // should fire once each poll interval - jest.advanceTimersByTime(halfInterval); - expect(mockTaskManager.get).toHaveBeenCalledTimes(0); - jest.advanceTimersByTime(halfInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(1); jest.advanceTimersByTime(pollInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(2); jest.advanceTimersByTime(pollInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(3); + jest.advanceTimersByTime(pollInterval); + expect(mockTaskManager.get).toHaveBeenCalledTimes(4); }); it('should retry on error', async () => { @@ -94,7 +92,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { mockTaskManager.get.mockRejectedValue(new Error('Failure')); const retryDelay = 10; const pollInterval = 100; - const halfInterval = Math.floor(pollInterval / 2); getHealthStatusStream( mockTaskManager, @@ -114,28 +111,27 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { retryDelay ).subscribe(); - jest.advanceTimersByTime(halfInterval); - expect(mockTaskManager.get).toHaveBeenCalledTimes(0); - jest.advanceTimersByTime(halfInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(1); + jest.advanceTimersByTime(pollInterval); + expect(mockTaskManager.get).toHaveBeenCalledTimes(2); // Retry on failure let numTimesCalled = 1; for (let i = 0; i < MAX_RETRY_ATTEMPTS; i++) { await tick(); jest.advanceTimersByTime(retryDelay); - expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled++ + 1); + expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled++ + 2); } // Once we've exceeded max retries, should not try again await tick(); jest.advanceTimersByTime(retryDelay); - expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled); + expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled + 1); // Once another poll interval passes, should call fn again await tick(); jest.advanceTimersByTime(pollInterval - MAX_RETRY_ATTEMPTS * retryDelay); - expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled + 1); + expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled + 2); }); it('should return healthy status when health status is "ok"', async () => { diff --git a/x-pack/plugins/alerting/server/health/get_state.ts b/x-pack/plugins/alerting/server/health/get_state.ts index 30099614ea42b..255037d7015a2 100644 --- a/x-pack/plugins/alerting/server/health/get_state.ts +++ b/x-pack/plugins/alerting/server/health/get_state.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import { defer, of, interval, Observable, throwError, timer } from 'rxjs'; -import { catchError, mergeMap, retryWhen, switchMap } from 'rxjs/operators'; +import { catchError, mergeMap, retryWhen, startWith, switchMap } from 'rxjs/operators'; import { Logger, SavedObjectsServiceStart, @@ -121,6 +121,17 @@ export const getHealthStatusStream = ( retryDelay?: number ): Observable> => interval(healthStatusInterval ?? HEALTH_STATUS_INTERVAL).pipe( + // Emit an initial check + startWith( + getHealthServiceStatusWithRetryAndErrorHandling( + taskManager, + logger, + savedObjects, + config, + retryDelay + ) + ), + // On each interval do a new check switchMap(() => getHealthServiceStatusWithRetryAndErrorHandling( taskManager, diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index 769243b8feaf6..41ae9f15d9af9 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -7,7 +7,7 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { first, map, share } from 'rxjs/operators'; -import { Observable } from 'rxjs'; +import { BehaviorSubject, Observable } from 'rxjs'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { combineLatest } from 'rxjs'; import { SecurityPluginSetup, SecurityPluginStart } from '../../security/server'; @@ -34,6 +34,7 @@ import { StatusServiceSetup, ServiceStatus, SavedObjectsBulkGetObject, + ServiceStatusLevels, } from '../../../../src/core/server'; import type { AlertingRequestHandlerContext } from './types'; import { defineRoutes } from './routes'; @@ -226,17 +227,23 @@ export class AlertingPlugin { this.config ); + const serviceStatus$ = new BehaviorSubject({ + level: ServiceStatusLevels.unavailable, + summary: 'Alerting is initializing', + }); + core.status.set(serviceStatus$); + core.getStartServices().then(async ([coreStart, startPlugins]) => { - core.status.set( - combineLatest([ - core.status.derivedStatus$, - getHealthStatusStream( - startPlugins.taskManager, - this.logger, - coreStart.savedObjects, - this.config - ), - ]).pipe( + combineLatest([ + core.status.derivedStatus$, + getHealthStatusStream( + startPlugins.taskManager, + this.logger, + coreStart.savedObjects, + this.config + ), + ]) + .pipe( map(([derivedStatus, healthStatus]) => { if (healthStatus.level > derivedStatus.level) { return healthStatus as ServiceStatus; @@ -246,7 +253,7 @@ export class AlertingPlugin { }), share() ) - ); + .subscribe(serviceStatus$); }); initializeAlertingHealth(this.logger, plugins.taskManager, core.getStartServices()); diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 51199da26ee7d..d3e251b751ef8 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -87,15 +87,13 @@ export class TaskManagerPlugin this.config! ); - core.getStartServices().then(async () => { - core.status.set( - combineLatest([core.status.derivedStatus$, serviceStatus$]).pipe( - map(([derivedStatus, serviceStatus]) => - serviceStatus.level > derivedStatus.level ? serviceStatus : derivedStatus - ) + core.status.set( + combineLatest([core.status.derivedStatus$, serviceStatus$]).pipe( + map(([derivedStatus, serviceStatus]) => + serviceStatus.level > derivedStatus.level ? serviceStatus : derivedStatus ) - ); - }); + ) + ); return { index: this.config.index, From 970e9a037bcb098ec84d9a41c5b1e680ecc05f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 14 Jun 2021 17:14:15 +0200 Subject: [PATCH 08/91] [APM] Add AWS and Azure icons for additional services (#101901) --- .../shared/span_icon/get_span_icon.ts | 17 ++++- .../shared/span_icon/span_icon.stories.tsx | 73 +++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx diff --git a/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts b/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts index bebfcba1a93b4..e2e1391a2f842 100644 --- a/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts +++ b/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts @@ -6,7 +6,6 @@ */ import { maybe } from '../../../../common/utils/maybe'; -import awsIcon from './icons/aws.svg'; import cassandraIcon from './icons/cassandra.svg'; import databaseIcon from './icons/database.svg'; import defaultIcon from './icons/default.svg'; @@ -33,12 +32,14 @@ const defaultTypeIcons: { [key: string]: string } = { resource: globeIcon, }; -const typeIcons: { [key: string]: { [key: string]: string } } = { +export const typeIcons: { [type: string]: { [subType: string]: string } } = { aws: { - servicename: awsIcon, + servicename: 'logoAWS', }, db: { cassandra: cassandraIcon, + cosmosdb: 'logoAzure', + dynamodb: 'logoAWS', elasticsearch: elasticsearchIcon, mongodb: mongodbIcon, mysql: mysqlIcon, @@ -51,8 +52,18 @@ const typeIcons: { [key: string]: { [key: string]: string } } = { websocket: websocketIcon, }, messaging: { + azurequeue: 'logoAzure', + azureservicebus: 'logoAzure', jms: javaIcon, kafka: kafkaIcon, + sns: 'logoAWS', + sqs: 'logoAWS', + }, + storage: { + azureblob: 'logoAzure', + azurefile: 'logoAzure', + azuretable: 'logoAzure', + s3: 'logoAWS', }, template: { handlebars: handlebarsIcon, diff --git a/x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx b/x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx new file mode 100644 index 0000000000000..951d3e61f1846 --- /dev/null +++ b/x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlexGrid, + EuiFlexItem, + EuiCopy, + EuiPanel, + EuiSpacer, + EuiCodeBlock, +} from '@elastic/eui'; +import React from 'react'; +import { storiesOf } from '@storybook/react'; +import { EuiThemeProvider } from '../../../../../../../src/plugins/kibana_react/common'; +import { SpanIcon } from './index'; +import { typeIcons } from './get_span_icon'; + +const types = Object.keys(typeIcons); + +storiesOf('shared/span_icon/span_icon', module) + .addDecorator((storyFn) => {storyFn()}) + .add( + 'Span icon', + () => { + return ( + <> + + {''} + + + + + {types.map((type) => { + const subTypes = Object.keys(typeIcons[type]); + return ( + <> + {subTypes.map((subType) => { + const id = `${type}.${subType}`; + return ( + + + {(copy) => ( + +  {' '} + {id} + + )} + + + ); + })} + + ); + })} + + + ); + }, + {} + ); From adda72edd28abf21b201477aaddc55b51e197596 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Mon, 14 Jun 2021 12:23:28 -0400 Subject: [PATCH 09/91] Document platform security plugins in kibana.json (#101965) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/plugins/security_oss/kibana.json | 5 +++++ src/plugins/spaces_oss/kibana.json | 5 +++++ x-pack/plugins/encrypted_saved_objects/kibana.json | 5 +++++ x-pack/plugins/security/kibana.json | 5 +++++ x-pack/plugins/spaces/kibana.json | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/src/plugins/security_oss/kibana.json b/src/plugins/security_oss/kibana.json index 70e37d586f1db..c93b5c3b60714 100644 --- a/src/plugins/security_oss/kibana.json +++ b/src/plugins/security_oss/kibana.json @@ -1,5 +1,10 @@ { "id": "securityOss", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin exposes a limited set of security functionality to OSS plugins.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["security"], diff --git a/src/plugins/spaces_oss/kibana.json b/src/plugins/spaces_oss/kibana.json index e048fb7ffb79c..10127634618f1 100644 --- a/src/plugins/spaces_oss/kibana.json +++ b/src/plugins/spaces_oss/kibana.json @@ -1,5 +1,10 @@ { "id": "spacesOss", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin exposes a limited set of spaces functionality to OSS plugins.", "version": "kibana", "server": false, "ui": true, diff --git a/x-pack/plugins/encrypted_saved_objects/kibana.json b/x-pack/plugins/encrypted_saved_objects/kibana.json index 74f797ba36a11..4812afe8c2072 100644 --- a/x-pack/plugins/encrypted_saved_objects/kibana.json +++ b/x-pack/plugins/encrypted_saved_objects/kibana.json @@ -1,5 +1,10 @@ { "id": "encryptedSavedObjects", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin provides encryption and decryption utilities for saved objects containing sensitive information.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "encryptedSavedObjects"], diff --git a/x-pack/plugins/security/kibana.json b/x-pack/plugins/security/kibana.json index f6e7b8bf46a39..a29c01b0f31cc 100644 --- a/x-pack/plugins/security/kibana.json +++ b/x-pack/plugins/security/kibana.json @@ -1,5 +1,10 @@ { "id": "security", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin provides authentication and authorization features, and exposes functionality to understand the capabilities of the currently authenticated user.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "security"], diff --git a/x-pack/plugins/spaces/kibana.json b/x-pack/plugins/spaces/kibana.json index e67931d4a3b8d..673055b24e8b6 100644 --- a/x-pack/plugins/spaces/kibana.json +++ b/x-pack/plugins/spaces/kibana.json @@ -1,5 +1,10 @@ { "id": "spaces", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin provides the Spaces feature, which allows saved objects to be organized into meaningful categories.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "spaces"], From 91b804f505da9614dc6b85cd64ae64f6d06bea0f Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 14 Jun 2021 12:33:46 -0400 Subject: [PATCH 10/91] [Fleet] Display NOTICE.txt from package if it exists (#101663) --- .../plugins/fleet/common/types/models/epm.ts | 8 +- .../applications/integrations/constants.tsx | 6 +- .../integrations/sections/epm/constants.tsx | 7 +- .../epm/screens/detail/overview/details.tsx | 57 +++++++++---- .../screens/detail/overview/notice_modal.tsx | 79 +++++++++++++++++++ .../server/services/epm/archive/index.ts | 7 ++ .../fleet/server/services/epm/packages/get.ts | 1 + .../server/services/epm/registry/index.ts | 12 +++ 8 files changed, 155 insertions(+), 22 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 0ef9f8b7ace36..f19684b0445e2 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -48,7 +48,12 @@ export type EpmPackageInstallStatus = 'installed' | 'installing'; export type DetailViewPanelName = 'overview' | 'policies' | 'settings' | 'custom'; export type ServiceName = 'kibana' | 'elasticsearch'; export type AgentAssetType = typeof agentAssetTypes; -export type AssetType = KibanaAssetType | ElasticsearchAssetType | ValueOf; +export type DocAssetType = 'doc' | 'notice'; +export type AssetType = + | KibanaAssetType + | ElasticsearchAssetType + | ValueOf + | DocAssetType; /* Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased) @@ -344,6 +349,7 @@ export interface EpmPackageAdditions { latestVersion: string; assets: AssetsGroupedByServiceByType; removable?: boolean; + notice?: string; } type Merge = Omit> & diff --git a/x-pack/plugins/fleet/public/applications/integrations/constants.tsx b/x-pack/plugins/fleet/public/applications/integrations/constants.tsx index 403a47f4b94b2..08197e18fec02 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/constants.tsx @@ -7,7 +7,7 @@ import type { IconType } from '@elastic/eui'; -import type { AssetType, ServiceName } from '../../types'; +import type { ServiceName } from '../../types'; import { ElasticsearchAssetType, KibanaAssetType } from '../../types'; export * from '../../constants'; @@ -20,8 +20,9 @@ export const DisplayedAssets: ServiceNameToAssetTypes = { kibana: Object.values(KibanaAssetType), elasticsearch: Object.values(ElasticsearchAssetType), }; +export type DisplayedAssetType = KibanaAssetType | ElasticsearchAssetType; -export const AssetTitleMap: Record = { +export const AssetTitleMap: Record = { dashboard: 'Dashboard', ilm_policy: 'ILM Policy', ingest_pipeline: 'Ingest Pipeline', @@ -31,7 +32,6 @@ export const AssetTitleMap: Record = { component_template: 'Component Template', search: 'Saved Search', visualization: 'Visualization', - input: 'Agent input', map: 'Map', data_stream_ilm_policy: 'Data Stream ILM Policy', lens: 'Lens', diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx index 6ddff968bd3f3..41db09b0538b9 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx @@ -7,7 +7,7 @@ import type { IconType } from '@elastic/eui'; -import type { AssetType, ServiceName } from '../../types'; +import type { ServiceName } from '../../types'; import { ElasticsearchAssetType, KibanaAssetType } from '../../types'; // only allow Kibana assets for the kibana key, ES asssets for elasticsearch, etc @@ -19,7 +19,9 @@ export const DisplayedAssets: ServiceNameToAssetTypes = { elasticsearch: Object.values(ElasticsearchAssetType), }; -export const AssetTitleMap: Record = { +export type DisplayedAssetType = ElasticsearchAssetType | KibanaAssetType; + +export const AssetTitleMap: Record = { dashboard: 'Dashboard', ilm_policy: 'ILM Policy', ingest_pipeline: 'Ingest Pipeline', @@ -29,7 +31,6 @@ export const AssetTitleMap: Record = { component_template: 'Component Template', search: 'Saved Search', visualization: 'Visualization', - input: 'Agent input', map: 'Map', data_stream_ilm_policy: 'Data Stream ILM Policy', lens: 'Lens', diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx index 487df17980345..0a601d2128bba 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { memo, useMemo } from 'react'; +import React, { memo, useCallback, useMemo, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, @@ -13,6 +13,8 @@ import { EuiTextColor, EuiDescriptionList, EuiNotificationBadge, + EuiLink, + EuiPortal, } from '@elastic/eui'; import type { EuiDescriptionListProps } from '@elastic/eui/src/components/description_list/description_list'; @@ -26,6 +28,8 @@ import { entries } from '../../../../../types'; import { useGetCategories } from '../../../../../hooks'; import { AssetTitleMap, DisplayedAssets, ServiceTitleMap } from '../../../constants'; +import { NoticeModal } from './notice_modal'; + interface Props { packageInfo: PackageInfo; } @@ -41,6 +45,11 @@ export const Details: React.FC = memo(({ packageInfo }) => { return []; }, [categoriesData, isLoadingCategories, packageInfo.categories]); + const [isNoticeModalOpen, setIsNoticeModalOpen] = useState(false); + const toggleNoticeModal = useCallback(() => { + setIsNoticeModalOpen(!isNoticeModalOpen); + }, [isNoticeModalOpen]); + const listItems = useMemo(() => { // Base details: version and categories const items: EuiDescriptionListProps['listItems'] = [ @@ -123,14 +132,23 @@ export const Details: React.FC = memo(({ packageInfo }) => { } // License details - if (packageInfo.license) { + if (packageInfo.license || packageInfo.notice) { items.push({ title: ( ), - description: packageInfo.license, + description: ( + <> +

{packageInfo.license}

+ {packageInfo.notice && ( +

+ NOTICE.txt +

+ )} + + ), }); } @@ -140,21 +158,30 @@ export const Details: React.FC = memo(({ packageInfo }) => { packageInfo.assets, packageInfo.data_streams, packageInfo.license, + packageInfo.notice, packageInfo.version, + toggleNoticeModal, ]); return ( - - - -

- -

- - - - - - + <> + + {isNoticeModalOpen && packageInfo.notice && ( + + )} + + + + +

+ +

+ + + + + + + ); }); diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx new file mode 100644 index 0000000000000..239bd133d3c91 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState } from 'react'; +import { + EuiCodeBlock, + EuiLoadingContent, + EuiModal, + EuiModalBody, + EuiModalHeader, + EuiModalFooter, + EuiModalHeaderTitle, + EuiButton, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { sendGetFileByPath, useStartServices } from '../../../../../hooks'; + +interface Props { + noticePath: string; + onClose: () => void; +} + +export const NoticeModal: React.FunctionComponent = ({ noticePath, onClose }) => { + const { notifications } = useStartServices(); + const [notice, setNotice] = useState(undefined); + + useEffect(() => { + async function fetchData() { + try { + const { data } = await sendGetFileByPath(noticePath); + setNotice(data || ''); + } catch (err) { + notifications.toasts.addError(err, { + title: i18n.translate('xpack.fleet.epm.errorLoadingNotice', { + defaultMessage: 'Error loading NOTICE.txt', + }), + }); + } + } + fetchData(); + }, [noticePath, notifications]); + return ( + + + +

NOTICE.txt

+ + + + + {notice ? ( + notice + ) : ( + // Simulate a long notice while loading + <> +

+ +

+

+ +

+ + )} + + + + + + + + + ); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 809684df0592c..b08ec815a394d 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -114,6 +114,13 @@ export function getPathParts(path: string): AssetParts { [pkgkey, service, type, file] = path.replace(`data_stream/${dataset}/`, '').split('/'); } + // To support the NOTICE asset at the root level + if (service === 'NOTICE.txt') { + file = service; + type = 'notice'; + service = ''; + } + // This is to cover for the fields.yml files inside the "fields" directory if (file === undefined) { file = type; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index e4e4f9c55fd2b..404431816d10c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -126,6 +126,7 @@ export async function getPackageInfo(options: { title: packageInfo.title || nameAsTitle(packageInfo.name), assets: Registry.groupPathsByService(paths || []), removable: !isRequiredPackage(pkgName), + notice: Registry.getNoticePath(paths || []), }; const updated = { ...packageInfo, ...additions }; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 5ee7e198555c5..011a0e74e8c18 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -255,3 +255,15 @@ export function groupPathsByService(paths: string[]): AssetsGroupedByServiceByTy elasticsearch: assets.elasticsearch, }; } + +export function getNoticePath(paths: string[]): string | undefined { + for (const path of paths) { + const parts = getPathParts(path.replace(/^\/package\//, '')); + if (parts.type === 'notice') { + const { pkgName, pkgVersion } = splitPkgKey(parts.pkgkey); + return `/package/${pkgName}/${pkgVersion}/${parts.file}`; + } + } + + return undefined; +} From 571524005ee9913f50eaac73a8689c0e33439e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Mon, 14 Jun 2021 18:34:52 +0200 Subject: [PATCH 11/91] [APM] Change impact indicator size (#102060) --- .../shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap | 2 +- x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap b/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap index 8e89939f585aa..87b5b68e26026 100644 --- a/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap +++ b/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap @@ -4,7 +4,7 @@ exports[`ImpactBar component should render with default values 1`] = ` `; diff --git a/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx b/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx index 92f488b8ba0ee..87b3c669e993c 100644 --- a/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx @@ -18,7 +18,7 @@ export interface ImpactBarProps extends Record { export function ImpactBar({ value, - size = 'l', + size = 'm', max = 100, color = 'primary', ...rest From 666bce392383a57f983617bc25b901a556ec20b0 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 14 Jun 2021 18:44:29 +0200 Subject: [PATCH 12/91] [APM] Enforce span creation/naming for ES searches (#101856) --- packages/kbn-apm-utils/src/index.ts | 51 ++- .../chart_preview/get_transaction_duration.ts | 120 +++--- .../get_transaction_error_count.ts | 72 ++-- .../get_transaction_error_rate.ts | 5 +- ...et_correlations_for_failed_transactions.ts | 207 +++++----- .../errors/get_overall_error_timeseries.ts | 58 +-- .../get_correlations_for_slow_transactions.ts | 89 ++--- .../latency/get_duration_for_percentile.ts | 41 +- .../latency/get_latency_distribution.ts | 116 +++--- .../correlations/latency/get_max_latency.ts | 61 ++- .../get_overall_latency_distribution.ts | 5 +- .../lib/environments/get_all_environments.ts | 88 +++-- .../lib/environments/get_environments.ts | 83 ++-- .../__snapshots__/get_buckets.test.ts.snap | 1 + .../errors/distribution/get_buckets.test.ts | 2 +- .../lib/errors/distribution/get_buckets.ts | 90 ++--- .../lib/errors/get_error_group_sample.ts | 77 ++-- .../apm/server/lib/errors/get_error_groups.ts | 139 ++++--- .../helpers/aggregated_transactions/index.ts | 18 +- .../create_es_client/call_async_with_debug.ts | 21 +- .../create_apm_event_client/index.test.ts | 2 +- .../create_apm_event_client/index.ts | 19 +- .../create_internal_es_client/index.ts | 44 ++- .../server/lib/helpers/setup_request.test.ts | 32 +- .../java/gc/fetch_and_transform_gc_metrics.ts | 4 +- .../by_agent/java/gc/get_gc_rate_chart.ts | 22 +- .../by_agent/java/gc/get_gc_time_chart.ts | 22 +- .../by_agent/java/heap_memory/index.ts | 34 +- .../by_agent/java/non_heap_memory/index.ts | 38 +- .../by_agent/java/thread_count/index.ts | 30 +- .../lib/metrics/by_agent/shared/cpu/index.ts | 32 +- .../metrics/by_agent/shared/memory/index.ts | 70 ++-- .../metrics/fetch_and_transform_metrics.ts | 4 +- .../get_service_count.ts | 50 +-- .../get_transactions_per_minute.ts | 113 +++--- .../lib/observability_overview/has_data.ts | 48 +-- .../lib/rum_client/get_client_metrics.ts | 2 +- .../server/lib/rum_client/get_js_errors.ts | 2 +- .../lib/rum_client/get_long_task_metrics.ts | 2 +- .../rum_client/get_page_load_distribution.ts | 7 +- .../lib/rum_client/get_page_view_trends.ts | 2 +- .../lib/rum_client/get_pl_dist_breakdown.ts | 5 +- .../server/lib/rum_client/get_rum_services.ts | 2 +- .../server/lib/rum_client/get_url_search.ts | 2 +- .../lib/rum_client/get_visitor_breakdown.ts | 2 +- .../lib/rum_client/get_web_core_vitals.ts | 2 +- .../apm/server/lib/rum_client/has_rum_data.ts | 2 +- .../ui_filters/local_ui_filters/index.ts | 55 +-- .../fetch_service_paths_from_trace_ids.ts | 120 +++--- .../server/lib/service_map/get_service_map.ts | 101 ++--- .../get_service_map_service_node_info.ts | 119 +++--- .../lib/service_map/get_trace_sample_ids.ts | 195 +++++----- .../apm/server/lib/service_nodes/index.ts | 101 +++-- .../__snapshots__/queries.test.ts.snap | 1 + .../get_derived_service_annotations.ts | 142 ++++--- .../lib/services/get_service_agent_name.ts | 68 ++-- .../get_destination_map.ts | 137 ++++--- .../get_service_dependencies/get_metrics.ts | 94 ++--- ...service_error_group_detailed_statistics.ts | 111 +++--- ...get_service_error_group_main_statistics.ts | 45 ++- .../get_service_error_groups/index.ts | 76 ++-- .../get_service_instance_metadata_details.ts | 56 +-- ...vice_instances_system_metric_statistics.ts | 259 +++++++------ ...ervice_instances_transaction_statistics.ts | 221 ++++++----- .../services/get_service_metadata_details.ts | 172 ++++----- .../services/get_service_metadata_icons.ts | 92 ++--- .../lib/services/get_service_node_metadata.ts | 69 ++-- ...e_transaction_group_detailed_statistics.ts | 191 +++++----- .../get_service_transaction_groups.ts | 86 ++--- .../services/get_service_transaction_types.ts | 68 ++-- .../get_services/get_legacy_data_status.ts | 44 +-- .../get_service_transaction_stats.ts | 142 +++---- .../get_services_from_metric_documents.ts | 36 +- .../get_services/has_historical_agent_data.ts | 35 +- .../apm/server/lib/services/get_throughput.ts | 25 +- .../get_service_profiling_statistics.ts | 125 +++---- .../get_service_profiling_timeline.ts | 50 +-- .../apm/server/lib/services/queries.test.ts | 2 +- .../create_or_update_configuration.ts | 43 +-- .../delete_configuration.ts | 17 +- .../find_exact_configuration.ts | 56 ++- .../get_agent_name_by_service.ts | 54 +-- .../get_existing_environments_for_service.ts | 54 +-- .../agent_configuration/get_service_names.ts | 66 ++-- .../list_configurations.ts | 6 +- .../mark_applied_by_agent.ts | 5 +- .../search_configurations.ts | 98 +++-- .../create_or_update_custom_link.test.ts | 52 +-- .../create_or_update_custom_link.ts | 31 +- .../custom_link/delete_custom_link.ts | 17 +- .../settings/custom_link/get_transaction.ts | 56 +-- .../settings/custom_link/list_custom_links.ts | 78 ++-- .../apm/server/lib/traces/get_trace_items.ts | 166 ++++---- .../lib/transaction_groups/get_error_rate.ts | 132 +++---- .../get_transaction_group_stats.ts | 187 +++++----- .../lib/transaction_groups/queries.test.ts | 4 +- .../lib/transactions/breakdown/index.ts | 353 +++++++++--------- .../distribution/get_buckets/index.ts | 128 ++++--- .../distribution/get_distribution_max.ts | 70 ++-- .../transactions/get_latency_charts/index.ts | 69 ++-- .../get_throughput_charts/index.ts | 41 +- .../lib/transactions/get_transaction/index.ts | 39 +- .../get_transaction_by_trace/index.ts | 59 +-- .../plugins/apm/server/utils/test_helpers.tsx | 2 +- .../plugins/apm/server/utils/with_apm_span.ts | 2 +- 105 files changed, 3410 insertions(+), 3451 deletions(-) diff --git a/packages/kbn-apm-utils/src/index.ts b/packages/kbn-apm-utils/src/index.ts index 384b6683199e5..09a6989091f60 100644 --- a/packages/kbn-apm-utils/src/index.ts +++ b/packages/kbn-apm-utils/src/index.ts @@ -14,6 +14,7 @@ export interface SpanOptions { type?: string; subtype?: string; labels?: Record; + intercept?: boolean; } type Span = Exclude; @@ -36,23 +37,27 @@ export async function withSpan( ): Promise { const options = parseSpanOptions(optionsOrName); - const { name, type, subtype, labels } = options; + const { name, type, subtype, labels, intercept } = options; if (!agent.isStarted()) { return cb(); } + let createdSpan: Span | undefined; + // When a span starts, it's marked as the active span in its context. // When it ends, it's not untracked, which means that if a span // starts directly after this one ends, the newly started span is a // child of this span, even though it should be a sibling. // To mitigate this, we queue a microtask by awaiting a promise. - await Promise.resolve(); + if (!intercept) { + await Promise.resolve(); - const span = agent.startSpan(name); + createdSpan = agent.startSpan(name) ?? undefined; - if (!span) { - return cb(); + if (!createdSpan) { + return cb(); + } } // If a span is created in the same context as the span that we just @@ -61,33 +66,51 @@ export async function withSpan( // mitigate this we create a new context. return runInNewContext(() => { + const promise = cb(createdSpan); + + let span: Span | undefined = createdSpan; + + if (intercept) { + span = agent.currentSpan ?? undefined; + } + + if (!span) { + return promise; + } + + const targetedSpan = span; + + if (name) { + targetedSpan.name = name; + } + // @ts-ignore if (type) { - span.type = type; + targetedSpan.type = type; } if (subtype) { - span.subtype = subtype; + targetedSpan.subtype = subtype; } if (labels) { - span.addLabels(labels); + targetedSpan.addLabels(labels); } - return cb(span) + return promise .then((res) => { - if (!span.outcome || span.outcome === 'unknown') { - span.outcome = 'success'; + if (!targetedSpan.outcome || targetedSpan.outcome === 'unknown') { + targetedSpan.outcome = 'success'; } return res; }) .catch((err) => { - if (!span.outcome || span.outcome === 'unknown') { - span.outcome = 'failure'; + if (!targetedSpan.outcome || targetedSpan.outcome === 'unknown') { + targetedSpan.outcome = 'failure'; } throw err; }) .finally(() => { - span.end(); + targetedSpan.end(); }); }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts index 091982598d6a3..6ce175fcb8362 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts @@ -15,82 +15,82 @@ import { import { ProcessorEvent } from '../../../../common/processor_event'; import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; import { AlertParams } from '../../../routes/alerts/chart_preview'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -export function getTransactionDurationChartPreview({ +export async function getTransactionDurationChartPreview({ alertParams, setup, }: { alertParams: AlertParams; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_transaction_duration_chart_preview', async () => { - const { apmEventClient, start, end } = setup; - const { - aggregationType, - environment, - serviceName, - transactionType, - } = alertParams; + const { apmEventClient, start, end } = setup; + const { + aggregationType, + environment, + serviceName, + transactionType, + } = alertParams; - const query = { - bool: { - filter: [ - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), - ...(transactionType - ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] - : []), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ] as QueryDslQueryContainer[], - }, - }; + const query = { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), + ...(transactionType + ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] + : []), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ] as QueryDslQueryContainer[], + }, + }; - const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); + const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); - const aggs = { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - }, - aggs: { - agg: - aggregationType === 'avg' - ? { avg: { field: TRANSACTION_DURATION } } - : { - percentiles: { - field: TRANSACTION_DURATION, - percents: [aggregationType === '95th' ? 95 : 99], - }, + const aggs = { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + }, + aggs: { + agg: + aggregationType === 'avg' + ? { avg: { field: TRANSACTION_DURATION } } + : { + percentiles: { + field: TRANSACTION_DURATION, + percents: [aggregationType === '95th' ? 95 : 99], }, - }, + }, }, - }; - const params = { - apm: { events: [ProcessorEvent.transaction] }, - body: { size: 0, query, aggs }, - }; - const resp = await apmEventClient.search(params); + }, + }; + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { size: 0, query, aggs }, + }; + const resp = await apmEventClient.search( + 'get_transaction_duration_chart_preview', + params + ); - if (!resp.aggregations) { - return []; - } + if (!resp.aggregations) { + return []; + } - return resp.aggregations.timeseries.buckets.map((bucket) => { - const percentilesKey = aggregationType === '95th' ? '95.0' : '99.0'; - const x = bucket.key; - const y = - aggregationType === 'avg' - ? (bucket.agg as { value: number | null }).value - : (bucket.agg as { values: Record }).values[ - percentilesKey - ]; + return resp.aggregations.timeseries.buckets.map((bucket) => { + const percentilesKey = aggregationType === '95th' ? '95.0' : '99.0'; + const x = bucket.key; + const y = + aggregationType === 'avg' + ? (bucket.agg as { value: number | null }).value + : (bucket.agg as { values: Record }).values[ + percentilesKey + ]; - return { x, y }; - }); + return { x, y }; }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts index 2cf1317dc44b0..3d64c63cb2041 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts @@ -9,58 +9,58 @@ import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; import { AlertParams } from '../../../routes/alerts/chart_preview'; import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -export function getTransactionErrorCountChartPreview({ +export async function getTransactionErrorCountChartPreview({ setup, alertParams, }: { setup: Setup & SetupTimeRange; alertParams: AlertParams; }) { - return withApmSpan('get_transaction_error_count_chart_preview', async () => { - const { apmEventClient, start, end } = setup; - const { serviceName, environment } = alertParams; + const { apmEventClient, start, end } = setup; + const { serviceName, environment } = alertParams; - const query = { - bool: { - filter: [ - ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ], - }, - }; + const query = { + bool: { + filter: [ + ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ], + }, + }; - const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); + const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); - const aggs = { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - }, + const aggs = { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, }, - }; + }, + }; - const params = { - apm: { events: [ProcessorEvent.error] }, - body: { size: 0, query, aggs }, - }; + const params = { + apm: { events: [ProcessorEvent.error] }, + body: { size: 0, query, aggs }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_transaction_error_count_chart_preview', + params + ); - if (!resp.aggregations) { - return []; - } + if (!resp.aggregations) { + return []; + } - return resp.aggregations.timeseries.buckets.map((bucket) => { - return { - x: bucket.key, - y: bucket.doc_count, - }; - }); + return resp.aggregations.timeseries.buckets.map((bucket) => { + return { + x: bucket.key, + y: bucket.doc_count, + }; }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts index f0c8d23e0e8fa..0a6a25ad9c533 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts @@ -64,7 +64,10 @@ export async function getTransactionErrorRateChartPreview({ body: { size: 0, query, aggs }, }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_transaction_error_rate_chart_preview', + params + ); if (!resp.aggregations) { return []; diff --git a/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts b/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts index 8ee469c9a93c7..11e9f99ddb356 100644 --- a/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts @@ -21,68 +21,68 @@ import { getTimeseriesAggregation, getTransactionErrorRateTimeSeries, } from '../../helpers/transaction_error_rate'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; interface Options extends CorrelationsOptions { fieldNames: string[]; } export async function getCorrelationsForFailedTransactions(options: Options) { - return withApmSpan('get_correlations_for_failed_transactions', async () => { - const { fieldNames, setup } = options; - const { apmEventClient } = setup; - const filters = getCorrelationsFilters(options); - - const params = { - apm: { events: [ProcessorEvent.transaction] }, - track_total_hits: true, - body: { - size: 0, - query: { - bool: { filter: filters }, - }, - aggs: { - failed_transactions: { - filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, - - // significant term aggs - aggs: fieldNames.reduce((acc, fieldName) => { - return { - ...acc, - [fieldName]: { - significant_terms: { - size: 10, - field: fieldName, - background_filter: { - bool: { - filter: filters, - must_not: { - term: { [EVENT_OUTCOME]: EventOutcome.failure }, - }, + const { fieldNames, setup } = options; + const { apmEventClient } = setup; + const filters = getCorrelationsFilters(options); + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + track_total_hits: true, + body: { + size: 0, + query: { + bool: { filter: filters }, + }, + aggs: { + failed_transactions: { + filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, + + // significant term aggs + aggs: fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { + bool: { + filter: filters, + must_not: { + term: { [EVENT_OUTCOME]: EventOutcome.failure }, }, }, }, }, - }; - }, {} as Record), - }, + }, + }; + }, {} as Record), }, }, - }; - - const response = await apmEventClient.search(params); - if (!response.aggregations) { - return { significantTerms: [] }; - } - - const sigTermAggs = omit( - response.aggregations?.failed_transactions, - 'doc_count' - ); - - const topSigTerms = processSignificantTermAggs({ sigTermAggs }); - return getErrorRateTimeSeries({ setup, filters, topSigTerms }); - }); + }, + }; + + const response = await apmEventClient.search( + 'get_correlations_for_failed_transactions', + params + ); + if (!response.aggregations) { + return { significantTerms: [] }; + } + + const sigTermAggs = omit( + response.aggregations?.failed_transactions, + 'doc_count' + ); + + const topSigTerms = processSignificantTermAggs({ sigTermAggs }); + return getErrorRateTimeSeries({ setup, filters, topSigTerms }); } export async function getErrorRateTimeSeries({ @@ -94,58 +94,59 @@ export async function getErrorRateTimeSeries({ filters: ESFilter[]; topSigTerms: TopSigTerm[]; }) { - return withApmSpan('get_error_rate_timeseries', async () => { - const { start, end, apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); - - if (isEmpty(topSigTerms)) { - return { significantTerms: [] }; + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); + + if (isEmpty(topSigTerms)) { + return { significantTerms: [] }; + } + + const timeseriesAgg = getTimeseriesAggregation(start, end, intervalString); + + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { timeseries: timeseriesAgg }, + }; + return acc; + }, + {} as { + [key: string]: { + filter: AggregationOptionsByType['filter']; + aggs: { timeseries: typeof timeseriesAgg }; + }; } - - const timeseriesAgg = getTimeseriesAggregation(start, end, intervalString); - - const perTermAggs = topSigTerms.reduce( - (acc, term, index) => { - acc[`term_${index}`] = { - filter: { term: { [term.fieldName]: term.fieldValue } }, - aggs: { timeseries: timeseriesAgg }, - }; - return acc; - }, - {} as { - [key: string]: { - filter: AggregationOptionsByType['filter']; - aggs: { timeseries: typeof timeseriesAgg }; - }; - } - ); - - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: filters } }, - aggs: perTermAggs, - }, - }; - - const response = await apmEventClient.search(params); - const { aggregations } = response; - - if (!aggregations) { - return { significantTerms: [] }; - } - - return { - significantTerms: topSigTerms.map((topSig, index) => { - const agg = aggregations[`term_${index}`]!; - - return { - ...topSig, - timeseries: getTransactionErrorRateTimeSeries(agg.timeseries.buckets), - }; - }), - }; - }); + ); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: perTermAggs, + }, + }; + + const response = await apmEventClient.search( + 'get_error_rate_timeseries', + params + ); + const { aggregations } = response; + + if (!aggregations) { + return { significantTerms: [] }; + } + + return { + significantTerms: topSigTerms.map((topSig, index) => { + const agg = aggregations[`term_${index}`]!; + + return { + ...topSig, + timeseries: getTransactionErrorRateTimeSeries(agg.timeseries.buckets), + }; + }), + }; } diff --git a/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts b/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts index 9387e64a51e01..f3477273806b6 100644 --- a/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts +++ b/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts @@ -11,41 +11,41 @@ import { getTimeseriesAggregation, getTransactionErrorRateTimeSeries, } from '../../helpers/transaction_error_rate'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; export async function getOverallErrorTimeseries(options: CorrelationsOptions) { - return withApmSpan('get_error_rate_timeseries', async () => { - const { setup } = options; - const filters = getCorrelationsFilters(options); - const { start, end, apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); + const { setup } = options; + const filters = getCorrelationsFilters(options); + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: filters } }, - aggs: { - timeseries: getTimeseriesAggregation(start, end, intervalString), - }, + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: { + timeseries: getTimeseriesAggregation(start, end, intervalString), }, - }; + }, + }; - const response = await apmEventClient.search(params); - const { aggregations } = response; + const response = await apmEventClient.search( + 'get_error_rate_timeseries', + params + ); + const { aggregations } = response; - if (!aggregations) { - return { overall: null }; - } + if (!aggregations) { + return { overall: null }; + } - return { - overall: { - timeseries: getTransactionErrorRateTimeSeries( - aggregations.timeseries.buckets - ), - }, - }; - }); + return { + overall: { + timeseries: getTransactionErrorRateTimeSeries( + aggregations.timeseries.buckets + ), + }, + }; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts index 0f93d1411a001..c37b3e3ab8242 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts @@ -41,60 +41,63 @@ export async function getCorrelationsForSlowTransactions(options: Options) { return { significantTerms: [] }; } - const response = await withApmSpan('get_significant_terms', () => { - const params = { - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { - bool: { - // foreground filters - filter: filters, - must: { - function_score: { - query: { - range: { - [TRANSACTION_DURATION]: { gte: durationForPercentile }, - }, + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + // foreground filters + filter: filters, + must: { + function_score: { + query: { + range: { + [TRANSACTION_DURATION]: { gte: durationForPercentile }, }, - script_score: { - script: { - source: `Math.log(2 + doc['${TRANSACTION_DURATION}'].value)`, - }, + }, + script_score: { + script: { + source: `Math.log(2 + doc['${TRANSACTION_DURATION}'].value)`, }, }, }, }, }, - aggs: fieldNames.reduce((acc, fieldName) => { - return { - ...acc, - [fieldName]: { - significant_terms: { - size: 10, - field: fieldName, - background_filter: { - bool: { - filter: [ - ...filters, - { - range: { - [TRANSACTION_DURATION]: { - lt: durationForPercentile, - }, + }, + aggs: fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { + bool: { + filter: [ + ...filters, + { + range: { + [TRANSACTION_DURATION]: { + lt: durationForPercentile, }, }, - ], - }, + }, + ], }, }, }, - }; - }, {} as Record), - }, - }; - return apmEventClient.search(params); - }); + }, + }; + }, {} as Record), + }, + }; + + const response = await apmEventClient.search( + 'get_significant_terms', + params + ); + if (!response.aggregations) { return { significantTerms: [] }; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts index 43c261743861d..a686980700d83 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts @@ -8,7 +8,6 @@ import { ESFilter } from '../../../../../../../typings/elasticsearch'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; export async function getDurationForPercentile({ @@ -20,31 +19,27 @@ export async function getDurationForPercentile({ filters: ESFilter[]; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_duration_for_percentiles', async () => { - const { apmEventClient } = setup; - const res = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], + const { apmEventClient } = setup; + const res = await apmEventClient.search('get_duration_for_percentiles', { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + size: 0, + query: { + bool: { filter: filters }, }, - body: { - size: 0, - query: { - bool: { filter: filters }, - }, - aggs: { - percentile: { - percentiles: { - field: TRANSACTION_DURATION, - percents: [durationPercentile], - }, + aggs: { + percentile: { + percentiles: { + field: TRANSACTION_DURATION, + percents: [durationPercentile], }, }, }, - }); - - const duration = Object.values( - res.aggregations?.percentile.values || {} - )[0]; - return duration || 0; + }, }); + + const duration = Object.values(res.aggregations?.percentile.values || {})[0]; + return duration || 0; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts index 6d42b26b22e42..be1bb631378cf 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts @@ -10,7 +10,7 @@ import { ESFilter } from '../../../../../../../typings/elasticsearch'; import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { TopSigTerm } from '../process_significant_term_aggs'; -import { withApmSpan } from '../../../utils/with_apm_span'; + import { getDistributionAggregation, trimBuckets, @@ -29,70 +29,70 @@ export async function getLatencyDistribution({ maxLatency: number; distributionInterval: number; }) { - return withApmSpan('get_latency_distribution', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const distributionAgg = getDistributionAggregation( - maxLatency, - distributionInterval - ); + const distributionAgg = getDistributionAggregation( + maxLatency, + distributionInterval + ); - const perTermAggs = topSigTerms.reduce( - (acc, term, index) => { - acc[`term_${index}`] = { - filter: { term: { [term.fieldName]: term.fieldValue } }, - aggs: { - distribution: distributionAgg, - }, + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { + distribution: distributionAgg, + }, + }; + return acc; + }, + {} as Record< + string, + { + filter: AggregationOptionsByType['filter']; + aggs: { + distribution: typeof distributionAgg; }; - return acc; - }, - {} as Record< - string, - { - filter: AggregationOptionsByType['filter']; - aggs: { - distribution: typeof distributionAgg; - }; - } - > - ); + } + > + ); - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: filters } }, - aggs: perTermAggs, - }, - }; + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: perTermAggs, + }, + }; - const response = await withApmSpan('get_terms_distribution', () => - apmEventClient.search(params) - ); - type Agg = NonNullable; + const response = await apmEventClient.search( + 'get_latency_distribution', + params + ); - if (!response.aggregations) { - return []; - } + type Agg = NonNullable; - return topSigTerms.map((topSig, index) => { - // ignore the typescript error since existence of response.aggregations is already checked: - // @ts-expect-error - const agg = response.aggregations[`term_${index}`] as Agg[string]; - const total = agg.distribution.doc_count; - const buckets = trimBuckets( - agg.distribution.dist_filtered_by_latency.buckets - ); + if (!response.aggregations) { + return []; + } - return { - ...topSig, - distribution: buckets.map((bucket) => ({ - x: bucket.key, - y: (bucket.doc_count / total) * 100, - })), - }; - }); + return topSigTerms.map((topSig, index) => { + // ignore the typescript error since existence of response.aggregations is already checked: + // @ts-expect-error + const agg = response.aggregations[`term_${index}`] as Agg[string]; + const total = agg.distribution.doc_count; + const buckets = trimBuckets( + agg.distribution.dist_filtered_by_latency.buckets + ); + + return { + ...topSig, + distribution: buckets.map((bucket) => ({ + x: bucket.key, + y: (bucket.doc_count / total) * 100, + })), + }; }); } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts index 8b415bf0d80a7..f2762086614b4 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts @@ -8,7 +8,6 @@ import { ESFilter } from '../../../../../../../typings/elasticsearch'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { TopSigTerm } from '../process_significant_term_aggs'; @@ -21,41 +20,39 @@ export async function getMaxLatency({ filters: ESFilter[]; topSigTerms?: TopSigTerm[]; }) { - return withApmSpan('get_max_latency', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { - bool: { - filter: filters, + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + filter: filters, - ...(topSigTerms.length - ? { - // only include docs containing the significant terms - should: topSigTerms.map((term) => ({ - term: { [term.fieldName]: term.fieldValue }, - })), - minimum_should_match: 1, - } - : null), - }, + ...(topSigTerms.length + ? { + // only include docs containing the significant terms + should: topSigTerms.map((term) => ({ + term: { [term.fieldName]: term.fieldValue }, + })), + minimum_should_match: 1, + } + : null), }, - aggs: { - // TODO: add support for metrics - // max_latency: { max: { field: TRANSACTION_DURATION } }, - max_latency: { - percentiles: { field: TRANSACTION_DURATION, percents: [99] }, - }, + }, + aggs: { + // TODO: add support for metrics + // max_latency: { max: { field: TRANSACTION_DURATION } }, + max_latency: { + percentiles: { field: TRANSACTION_DURATION, percents: [99] }, }, }, - }; + }, + }; - const response = await apmEventClient.search(params); - // return response.aggregations?.max_latency.value; - return Object.values(response.aggregations?.max_latency.values ?? {})[0]; - }); + const response = await apmEventClient.search('get_max_latency', params); + // return response.aggregations?.max_latency.value; + return Object.values(response.aggregations?.max_latency.values ?? {})[0]; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts index c5d4def51ea54..b0e0f22c70366 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts @@ -71,8 +71,9 @@ export async function getOverallLatencyDistribution( }, }; - const response = await withApmSpan('get_terms_distribution', () => - apmEventClient.search(params) + const response = await apmEventClient.search( + 'get_terms_distribution', + params ); if (!response.aggregations) { diff --git a/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts index 1bf01c24776fb..f6a1987974853 100644 --- a/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts +++ b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts @@ -13,7 +13,6 @@ import { } from '../../../common/elasticsearch_fieldnames'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; /** * This is used for getting *all* environments, and does not filter by range. @@ -30,59 +29,56 @@ export async function getAllEnvironments({ searchAggregatedTransactions: boolean; includeMissing?: boolean; }) { - const spanName = serviceName + const operationName = serviceName ? 'get_all_environments_for_service' : 'get_all_environments_for_all_services'; - return withApmSpan(spanName, async () => { - const { apmEventClient, config } = setup; - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - // omit filter for service.name if "All" option is selected - const serviceNameFilter = serviceName - ? [{ term: { [SERVICE_NAME]: serviceName } }] - : []; + const { apmEventClient, config } = setup; + const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - // use timeout + min_doc_count to return as early as possible - // if filter is not defined to prevent timeouts - ...(!serviceName ? { timeout: '1ms' } : {}), - size: 0, - query: { - bool: { - filter: [...serviceNameFilter], - }, + // omit filter for service.name if "All" option is selected + const serviceNameFilter = serviceName + ? [{ term: { [SERVICE_NAME]: serviceName } }] + : []; + + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + // use timeout + min_doc_count to return as early as possible + // if filter is not defined to prevent timeouts + ...(!serviceName ? { timeout: '1ms' } : {}), + size: 0, + query: { + bool: { + filter: [...serviceNameFilter], }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, - ...(!serviceName ? { min_doc_count: 0 } : {}), - missing: includeMissing - ? ENVIRONMENT_NOT_DEFINED.value - : undefined, - }, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + size: maxServiceEnvironments, + ...(!serviceName ? { min_doc_count: 0 } : {}), + missing: includeMissing ? ENVIRONMENT_NOT_DEFINED.value : undefined, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search(operationName, params); - const environments = - resp.aggregations?.environments.buckets.map( - (bucket) => bucket.key as string - ) || []; - return environments; - }); + const environments = + resp.aggregations?.environments.buckets.map( + (bucket) => bucket.key as string + ) || []; + return environments; } diff --git a/x-pack/plugins/apm/server/lib/environments/get_environments.ts b/x-pack/plugins/apm/server/lib/environments/get_environments.ts index 509e4cdcd67ac..c0b267f180010 100644 --- a/x-pack/plugins/apm/server/lib/environments/get_environments.ts +++ b/x-pack/plugins/apm/server/lib/environments/get_environments.ts @@ -12,7 +12,6 @@ import { import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { ProcessorEvent } from '../../../common/processor_event'; import { rangeQuery } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; @@ -29,60 +28,58 @@ export async function getEnvironments({ serviceName?: string; searchAggregatedTransactions: boolean; }) { - const spanName = serviceName + const operationName = serviceName ? 'get_environments_for_service' : 'get_environments'; - return withApmSpan(spanName, async () => { - const { start, end, apmEventClient, config } = setup; + const { start, end, apmEventClient, config } = setup; - const filter = rangeQuery(start, end); + const filter = rangeQuery(start, end); - if (serviceName) { - filter.push({ - term: { [SERVICE_NAME]: serviceName }, - }); - } + if (serviceName) { + filter.push({ + term: { [SERVICE_NAME]: serviceName }, + }); + } - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; + const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.metric, - ProcessorEvent.error, - ], - }, - body: { - size: 0, - query: { - bool: { - filter, - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.metric, + ProcessorEvent.error, + ], + }, + body: { + size: 0, + query: { + bool: { + filter, }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - missing: ENVIRONMENT_NOT_DEFINED.value, - size: maxServiceEnvironments, - }, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + missing: ENVIRONMENT_NOT_DEFINED.value, + size: maxServiceEnvironments, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - const aggs = resp.aggregations; - const environmentsBuckets = aggs?.environments.buckets || []; + const resp = await apmEventClient.search(operationName, params); + const aggs = resp.aggregations; + const environmentsBuckets = aggs?.environments.buckets || []; - const environments = environmentsBuckets.map( - (environmentBucket) => environmentBucket.key as string - ); + const environments = environmentsBuckets.map( + (environmentBucket) => environmentBucket.key as string + ); - return environments; - }); + return environments; } diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap b/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap index 43fe4dfe752e6..2c0330f17320d 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap @@ -3,6 +3,7 @@ exports[`get buckets should make the correct query 1`] = ` Array [ Array [ + "get_error_distribution_buckets", Object { "apm": Object { "events": Array [ diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts index b1260d653f3de..712343d445d44 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts +++ b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts @@ -65,7 +65,7 @@ describe('get buckets', () => { }); it('should limit query results to error documents', () => { - const query = clientSpy.mock.calls[0][0]; + const query = clientSpy.mock.calls[0][1]; expect(query.apm.events).toEqual([ProcessorEvent.error]); }); }); diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts index 462c9bcdc4310..a51464764f2b4 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts +++ b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts @@ -16,7 +16,6 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; export async function getBuckets({ @@ -34,58 +33,59 @@ export async function getBuckets({ bucketSize: number; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_error_distribution_buckets', async () => { - const { start, end, apmEventClient } = setup; - const filter: ESFilter[] = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; + const { start, end, apmEventClient } = setup; + const filter: ESFilter[] = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; - if (groupId) { - filter.push({ term: { [ERROR_GROUP_ID]: groupId } }); - } + if (groupId) { + filter.push({ term: { [ERROR_GROUP_ID]: groupId } }); + } - const params = { - apm: { - events: [ProcessorEvent.error], - }, - body: { - size: 0, - query: { - bool: { - filter, - }, + const params = { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter, }, - aggs: { - distribution: { - histogram: { - field: '@timestamp', - min_doc_count: 0, - interval: bucketSize, - extended_bounds: { - min: start, - max: end, - }, + }, + aggs: { + distribution: { + histogram: { + field: '@timestamp', + min_doc_count: 0, + interval: bucketSize, + extended_bounds: { + min: start, + max: end, }, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_error_distribution_buckets', + params + ); - const buckets = (resp.aggregations?.distribution.buckets || []).map( - (bucket) => ({ - key: bucket.key, - count: bucket.doc_count, - }) - ); + const buckets = (resp.aggregations?.distribution.buckets || []).map( + (bucket) => ({ + key: bucket.key, + count: bucket.doc_count, + }) + ); - return { - noHits: resp.hits.total.value === 0, - buckets: resp.hits.total.value > 0 ? buckets : [], - }; - }); + return { + noHits: resp.hits.total.value === 0, + buckets: resp.hits.total.value > 0 ? buckets : [], + }; } diff --git a/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts b/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts index 57fb486180993..a915a4fb03305 100644 --- a/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts +++ b/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts @@ -17,11 +17,10 @@ import { rangeQuery, kqlQuery, } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getTransaction } from '../transactions/get_transaction'; -export function getErrorGroupSample({ +export async function getErrorGroupSample({ environment, kuery, serviceName, @@ -34,48 +33,46 @@ export function getErrorGroupSample({ groupId: string; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_error_group_sample', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - apm: { - events: [ProcessorEvent.error as const], - }, - body: { - size: 1, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [ERROR_GROUP_ID]: groupId } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], - should: [{ term: { [TRANSACTION_SAMPLED]: true } }], - }, + const params = { + apm: { + events: [ProcessorEvent.error as const], + }, + body: { + size: 1, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [ERROR_GROUP_ID]: groupId } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + should: [{ term: { [TRANSACTION_SAMPLED]: true } }], }, - sort: asMutableArray([ - { _score: 'desc' }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top - { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error - ] as const), }, - }; + sort: asMutableArray([ + { _score: 'desc' }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top + { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error + ] as const), + }, + }; - const resp = await apmEventClient.search(params); - const error = resp.hits.hits[0]?._source; - const transactionId = error?.transaction?.id; - const traceId = error?.trace?.id; + const resp = await apmEventClient.search('get_error_group_sample', params); + const error = resp.hits.hits[0]?._source; + const transactionId = error?.transaction?.id; + const traceId = error?.trace?.id; - let transaction; - if (transactionId && traceId) { - transaction = await getTransaction({ transactionId, traceId, setup }); - } + let transaction; + if (transactionId && traceId) { + transaction = await getTransaction({ transactionId, traceId, setup }); + } - return { - transaction, - error, - occurrencesCount: resp.hits.total.value, - }; - }); + return { + transaction, + error, + occurrencesCount: resp.hits.total.value, + }; } diff --git a/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts b/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts index f5b22e5349756..d705a2eb5a00c 100644 --- a/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts +++ b/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts @@ -15,11 +15,10 @@ import { } from '../../../common/elasticsearch_fieldnames'; import { getErrorGroupsProjection } from '../../projections/errors'; import { mergeProjection } from '../../projections/util/merge_projection'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getErrorName } from '../helpers/get_error_name'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -export function getErrorGroups({ +export async function getErrorGroups({ environment, kuery, serviceName, @@ -34,87 +33,83 @@ export function getErrorGroups({ sortDirection?: 'asc' | 'desc'; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_error_groups', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - // sort buckets by last occurrence of error - const sortByLatestOccurrence = sortField === 'latestOccurrenceAt'; + // sort buckets by last occurrence of error + const sortByLatestOccurrence = sortField === 'latestOccurrenceAt'; - const projection = getErrorGroupsProjection({ - environment, - kuery, - setup, - serviceName, - }); + const projection = getErrorGroupsProjection({ + environment, + kuery, + setup, + serviceName, + }); - const order = sortByLatestOccurrence - ? { - max_timestamp: sortDirection, - } - : { _count: sortDirection }; + const order = sortByLatestOccurrence + ? { + max_timestamp: sortDirection, + } + : { _count: sortDirection }; - const params = mergeProjection(projection, { - body: { - size: 0, - aggs: { - error_groups: { - terms: { - ...projection.body.aggs.error_groups.terms, - size: 500, - order, - }, - aggs: { - sample: { - top_hits: { - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], - sort: [{ '@timestamp': 'desc' as const }], - size: 1, - }, + const params = mergeProjection(projection, { + body: { + size: 0, + aggs: { + error_groups: { + terms: { + ...projection.body.aggs.error_groups.terms, + size: 500, + order, + }, + aggs: { + sample: { + top_hits: { + _source: [ + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ERROR_CULPRIT, + ERROR_GROUP_ID, + '@timestamp', + ], + sort: [{ '@timestamp': 'desc' as const }], + size: 1, }, - ...(sortByLatestOccurrence - ? { - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - } - : {}), }, + ...(sortByLatestOccurrence + ? { + max_timestamp: { + max: { + field: '@timestamp', + }, + }, + } + : {}), }, }, }, - }); + }, + }); - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search('get_error_groups', params); - // aggregations can be undefined when no matching indices are found. - // this is an exception rather than the rule so the ES type does not account for this. - const hits = (resp.aggregations?.error_groups.buckets || []).map( - (bucket) => { - const source = bucket.sample.hits.hits[0]._source; - const message = getErrorName(source); + // aggregations can be undefined when no matching indices are found. + // this is an exception rather than the rule so the ES type does not account for this. + const hits = (resp.aggregations?.error_groups.buckets || []).map((bucket) => { + const source = bucket.sample.hits.hits[0]._source; + const message = getErrorName(source); - return { - message, - occurrenceCount: bucket.doc_count, - culprit: source.error.culprit, - groupId: source.error.grouping_key, - latestOccurrenceAt: source['@timestamp'], - handled: source.error.exception?.[0].handled, - type: source.error.exception?.[0].type, - }; - } - ); - - return hits; + return { + message, + occurrenceCount: bucket.doc_count, + culprit: source.error.culprit, + groupId: source.error.grouping_key, + latestOccurrenceAt: source['@timestamp'], + handled: source.error.exception?.[0].handled, + type: source.error.exception?.[0].type, + }; }); + + return hits; } diff --git a/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts b/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts index 394cf6b988f12..8bfb137c1689c 100644 --- a/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts @@ -14,7 +14,6 @@ import { } from '../../../../common/elasticsearch_fieldnames'; import { APMConfig } from '../../..'; import { APMEventClient } from '../create_es_client/create_apm_event_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function getHasAggregatedTransactions({ start, @@ -25,8 +24,9 @@ export async function getHasAggregatedTransactions({ end?: number; apmEventClient: APMEventClient; }) { - return withApmSpan('get_has_aggregated_transactions', async () => { - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_has_aggregated_transactions', + { apm: { events: [ProcessorEvent.metric], }, @@ -41,14 +41,14 @@ export async function getHasAggregatedTransactions({ }, }, terminateAfter: 1, - }); - - if (response.hits.total.value > 0) { - return true; } + ); + + if (response.hits.total.value > 0) { + return true; + } - return false; - }); + return false; } export async function getSearchAggregatedTransactions({ diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts index 989297544c78f..39018e26f371c 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts @@ -81,17 +81,26 @@ export async function callAsyncWithDebug({ return res; } -export const getDebugBody = ( - params: Record, - requestType: string -) => { +export const getDebugBody = ({ + params, + requestType, + operationName, +}: { + params: Record; + requestType: string; + operationName: string; +}) => { + const operationLine = `${operationName}\n`; + if (requestType === 'search') { - return `GET ${params.index}/_search\n${formatObj(params.body)}`; + return `${operationLine}GET ${params.index}/_search\n${formatObj( + params.body + )}`; } return `${chalk.bold('ES operation:')} ${requestType}\n${chalk.bold( 'ES query:' - )}\n${formatObj(params)}`; + )}\n${operationLine}${formatObj(params)}`; }; export const getDebugTitle = (request: KibanaRequest) => diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts index addd7391d782d..8e82a189d75f3 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts @@ -47,7 +47,7 @@ describe('createApmEventClient', () => { }, }); - await eventClient.search({ + await eventClient.search('foo', { apm: { events: [], }, diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index b8a14253a229a..916a6981f286a 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -6,6 +6,7 @@ */ import { ValuesType } from 'utility-types'; +import { withApmSpan } from '../../../../utils/with_apm_span'; import { Profile } from '../../../../../typings/es_schemas/ui/profile'; import { ElasticsearchClient, @@ -34,6 +35,7 @@ import { unpackProcessorEvents } from './unpack_processor_events'; export type APMEventESSearchRequest = Omit & { apm: { events: ProcessorEvent[]; + includeLegacyData?: boolean; }; }; @@ -78,11 +80,13 @@ export function createApmEventClient({ }) { return { async search( - params: TParams, - { includeLegacyData = false } = {} + operationName: string, + params: TParams ): Promise> { const withProcessorEventFilter = unpackProcessorEvents(params, indices); + const { includeLegacyData = false } = params.apm; + const withPossibleLegacyDataFilter = !includeLegacyData ? addFilterToExcludeLegacyData(withProcessorEventFilter) : withProcessorEventFilter; @@ -98,15 +102,18 @@ export function createApmEventClient({ return callAsyncWithDebug({ cb: () => { - const searchPromise = cancelEsRequestOnAbort( - esClient.search(searchParams), - request + const searchPromise = withApmSpan(operationName, () => + cancelEsRequestOnAbort(esClient.search(searchParams), request) ); return unwrapEsResponse(searchPromise); }, getDebugMessage: () => ({ - body: getDebugBody(searchParams, requestType), + body: getDebugBody({ + params: searchParams, + requestType, + operationName, + }), title: getDebugTitle(request), }), isCalledWithInternalUser: false, diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts index 1544538de74a6..e6b61a709ae35 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts @@ -31,20 +31,23 @@ export function createInternalESClient({ }: Pick & { debug: boolean }) { const { asInternalUser } = context.core.elasticsearch.client; - function callEs({ - cb, - requestType, - params, - }: { - requestType: string; - cb: () => TransportRequestPromise; - params: Record; - }) { + function callEs( + operationName: string, + { + cb, + requestType, + params, + }: { + requestType: string; + cb: () => TransportRequestPromise; + params: Record; + } + ) { return callAsyncWithDebug({ cb: () => unwrapEsResponse(cancelEsRequestOnAbort(cb(), request)), getDebugMessage: () => ({ title: getDebugTitle(request), - body: getDebugBody(params, requestType), + body: getDebugBody({ params, requestType, operationName }), }), debug, isCalledWithInternalUser: true, @@ -59,30 +62,37 @@ export function createInternalESClient({ TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest >( + operationName: string, params: TSearchRequest ): Promise> => { - return callEs({ + return callEs(operationName, { requestType: 'search', cb: () => asInternalUser.search(params), params, }); }, - index: (params: APMIndexDocumentParams) => { - return callEs({ + index: (operationName: string, params: APMIndexDocumentParams) => { + return callEs(operationName, { requestType: 'index', cb: () => asInternalUser.index(params), params, }); }, - delete: (params: estypes.DeleteRequest): Promise<{ result: string }> => { - return callEs({ + delete: ( + operationName: string, + params: estypes.DeleteRequest + ): Promise<{ result: string }> => { + return callEs(operationName, { requestType: 'delete', cb: () => asInternalUser.delete(params), params, }); }, - indicesCreate: (params: estypes.IndicesCreateRequest) => { - return callEs({ + indicesCreate: ( + operationName: string, + params: estypes.IndicesCreateRequest + ) => { + return callEs(operationName, { requestType: 'indices.create', cb: () => asInternalUser.indices.create(params), params, diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts index c0ff0cab88f47..66b3c91fc6f2d 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts @@ -109,7 +109,7 @@ describe('setupRequest', () => { it('calls callWithRequest', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [ProcessorEvent.transaction] }, body: { foo: 'bar' }, }); @@ -137,7 +137,7 @@ describe('setupRequest', () => { it('calls callWithInternalUser', async () => { const mockResources = getMockResources(); const { internalClient } = await setupRequest(mockResources); - await internalClient.search({ + await internalClient.search('foo', { index: ['apm-*'], body: { foo: 'bar' }, } as any); @@ -156,7 +156,7 @@ describe('setupRequest', () => { it('adds a range filter for `observer.version_major` to the existing filter', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [ProcessorEvent.transaction], }, @@ -183,19 +183,15 @@ describe('setupRequest', () => { it('does not add a range filter for `observer.version_major` if includeLegacyData=true', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search( - { - apm: { - events: [ProcessorEvent.error], - }, - body: { - query: { bool: { filter: [{ term: { field: 'someTerm' } }] } }, - }, - }, - { + await apmEventClient.search('foo', { + apm: { + events: [ProcessorEvent.error], includeLegacyData: true, - } - ); + }, + body: { + query: { bool: { filter: [{ term: { field: 'someTerm' } }] } }, + }, + }); const params = mockResources.context.core.elasticsearch.client.asCurrentUser.search .mock.calls[0][0]; @@ -221,7 +217,7 @@ describe('without a bool filter', () => { it('adds a range filter for `observer.version_major`', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [ProcessorEvent.error], }, @@ -251,7 +247,7 @@ describe('with includeFrozen=false', () => { const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [], }, @@ -273,7 +269,7 @@ describe('with includeFrozen=true', () => { const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [] }, }); diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts index 3b3ef8b9c4bcf..d1040b49dcd8b 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts @@ -30,6 +30,7 @@ export async function fetchAndTransformGcMetrics({ serviceNodeName, chartBase, fieldName, + operationName, }: { environment?: string; kuery?: string; @@ -38,6 +39,7 @@ export async function fetchAndTransformGcMetrics({ serviceNodeName?: string; chartBase: ChartBase; fieldName: typeof METRIC_JAVA_GC_COUNT | typeof METRIC_JAVA_GC_TIME; + operationName: string; }) { const { start, end, apmEventClient, config } = setup; @@ -108,7 +110,7 @@ export async function fetchAndTransformGcMetrics({ }, }); - const response = await apmEventClient.search(params); + const response = await apmEventClient.search(operationName, params); const { aggregations } = response; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts index 388331f3bbf17..3ec40d5171694 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_GC_COUNT } from '../../../../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../../../../helpers/setup_request'; import { fetchAndTransformGcMetrics } from './fetch_and_transform_gc_metrics'; @@ -45,17 +44,16 @@ function getGcRateChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_gc_rate_charts', () => - fetchAndTransformGcMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - fieldName: METRIC_JAVA_GC_COUNT, - }) - ); + return fetchAndTransformGcMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + fieldName: METRIC_JAVA_GC_COUNT, + operationName: 'get_gc_rate_charts', + }); } export { getGcRateChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts index e6f80190d1daa..8e4416d94fb90 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_GC_TIME } from '../../../../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../../../../helpers/setup_request'; import { fetchAndTransformGcMetrics } from './fetch_and_transform_gc_metrics'; @@ -45,17 +44,16 @@ function getGcTimeChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_gc_time_charts', () => - fetchAndTransformGcMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - fieldName: METRIC_JAVA_GC_TIME, - }) - ); + return fetchAndTransformGcMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + fieldName: METRIC_JAVA_GC_TIME, + operationName: 'get_gc_time_charts', + }); } export { getGcTimeChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts index 7630827a3cb38..6a23213e94537 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_HEAP_MEMORY_MAX, METRIC_JAVA_HEAP_MEMORY_COMMITTED, @@ -65,22 +64,21 @@ export function getHeapMemoryChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_heap_memory_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - heapMemoryMax: { avg: { field: METRIC_JAVA_HEAP_MEMORY_MAX } }, - heapMemoryCommitted: { - avg: { field: METRIC_JAVA_HEAP_MEMORY_COMMITTED }, - }, - heapMemoryUsed: { avg: { field: METRIC_JAVA_HEAP_MEMORY_USED } }, + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + heapMemoryMax: { avg: { field: METRIC_JAVA_HEAP_MEMORY_MAX } }, + heapMemoryCommitted: { + avg: { field: METRIC_JAVA_HEAP_MEMORY_COMMITTED }, }, - additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], - }) - ); + heapMemoryUsed: { avg: { field: METRIC_JAVA_HEAP_MEMORY_USED } }, + }, + additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], + operationName: 'get_heap_memory_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts index cd11e5e5383b6..1ceb42b7db479 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_NON_HEAP_MEMORY_MAX, METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED, @@ -62,24 +61,23 @@ export async function getNonHeapMemoryChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_non_heap_memory_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - nonHeapMemoryMax: { avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_MAX } }, - nonHeapMemoryCommitted: { - avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED }, - }, - nonHeapMemoryUsed: { - avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_USED }, - }, + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + nonHeapMemoryMax: { avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_MAX } }, + nonHeapMemoryCommitted: { + avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED }, }, - additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], - }) - ); + nonHeapMemoryUsed: { + avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_USED }, + }, + }, + additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], + operationName: 'get_non_heap_memory_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts index 8d4c079197d19..700c5e08d4def 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_THREAD_COUNT, AGENT_NAME, @@ -54,19 +53,18 @@ export async function getThreadCountChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_thread_count_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - threadCount: { avg: { field: METRIC_JAVA_THREAD_COUNT } }, - threadCountMax: { max: { field: METRIC_JAVA_THREAD_COUNT } }, - }, - additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], - }) - ); + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + threadCount: { avg: { field: METRIC_JAVA_THREAD_COUNT } }, + threadCountMax: { max: { field: METRIC_JAVA_THREAD_COUNT } }, + }, + additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], + operationName: 'get_thread_count_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts index 37bef191ae876..a568d58bdd438 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_SYSTEM_CPU_PERCENT, METRIC_PROCESS_CPU_PERCENT, @@ -66,20 +65,19 @@ export function getCPUChartData({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_cpu_metric_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - systemCPUAverage: { avg: { field: METRIC_SYSTEM_CPU_PERCENT } }, - systemCPUMax: { max: { field: METRIC_SYSTEM_CPU_PERCENT } }, - processCPUAverage: { avg: { field: METRIC_PROCESS_CPU_PERCENT } }, - processCPUMax: { max: { field: METRIC_PROCESS_CPU_PERCENT } }, - }, - }) - ); + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + systemCPUAverage: { avg: { field: METRIC_SYSTEM_CPU_PERCENT } }, + systemCPUMax: { max: { field: METRIC_SYSTEM_CPU_PERCENT } }, + processCPUAverage: { avg: { field: METRIC_PROCESS_CPU_PERCENT } }, + processCPUMax: { max: { field: METRIC_PROCESS_CPU_PERCENT } }, + }, + operationName: 'get_cpu_metric_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts index 0ec2f2c2fcfb2..1f7860d567b03 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts @@ -84,45 +84,41 @@ export async function getMemoryChartData({ serviceNodeName?: string; }) { return withApmSpan('get_memory_metrics_charts', async () => { - const cgroupResponse = await withApmSpan( - 'get_cgroup_memory_metrics_charts', - () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - memoryUsedAvg: { avg: { script: percentCgroupMemoryUsedScript } }, - memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } }, - }, - additionalFilters: [ - { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, - ], - }) - ); + const cgroupResponse = await fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + memoryUsedAvg: { avg: { script: percentCgroupMemoryUsedScript } }, + memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } }, + }, + additionalFilters: [ + { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, + ], + operationName: 'get_cgroup_memory_metrics_charts', + }); if (cgroupResponse.noHits) { - return await withApmSpan('get_system_memory_metrics_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - memoryUsedAvg: { avg: { script: percentSystemMemoryUsedScript } }, - memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } }, - }, - additionalFilters: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, - ], - }) - ); + return await fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + memoryUsedAvg: { avg: { script: percentSystemMemoryUsedScript } }, + memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } }, + }, + additionalFilters: [ + { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, + { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ], + operationName: 'get_system_memory_metrics_charts', + }); } return cgroupResponse; diff --git a/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts b/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts index 30234447821ec..df9e33e6f4b40 100644 --- a/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts +++ b/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts @@ -56,6 +56,7 @@ export async function fetchAndTransformMetrics({ chartBase, aggs, additionalFilters = [], + operationName, }: { environment?: string; kuery?: string; @@ -65,6 +66,7 @@ export async function fetchAndTransformMetrics({ chartBase: ChartBase; aggs: T; additionalFilters?: Filter[]; + operationName: string; }) { const { start, end, apmEventClient, config } = setup; @@ -98,7 +100,7 @@ export async function fetchAndTransformMetrics({ }, }); - const response = await apmEventClient.search(params); + const response = await apmEventClient.search(operationName, params); return transformDataToMetricsChart(response, chartBase); } diff --git a/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts b/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts index 2ccbe318862f1..086516371387e 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts @@ -10,40 +10,40 @@ import { rangeQuery } from '../../../server/utils/queries'; import { SERVICE_NAME } from '../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceCount({ +export async function getServiceCount({ setup, searchAggregatedTransactions, }: { setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('observability_overview_get_service_count', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 0, - query: { - bool: { - filter: rangeQuery(start, end), - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 0, + query: { + bool: { + filter: rangeQuery(start, end), }, - aggs: { serviceCount: { cardinality: { field: SERVICE_NAME } } }, }, - }; + aggs: { serviceCount: { cardinality: { field: SERVICE_NAME } } }, + }, + }; - const { aggregations } = await apmEventClient.search(params); - return aggregations?.serviceCount.value || 0; - }); + const { aggregations } = await apmEventClient.search( + 'observability_overview_get_service_count', + params + ); + return aggregations?.serviceCount.value || 0; } diff --git a/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts b/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts index da8ac7c50b594..016cb50566da0 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts @@ -14,9 +14,8 @@ import { rangeQuery } from '../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { calculateThroughput } from '../helpers/calculate_throughput'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getTransactionsPerMinute({ +export async function getTransactionsPerMinute({ setup, bucketSize, searchAggregatedTransactions, @@ -25,71 +24,69 @@ export function getTransactionsPerMinute({ bucketSize: string; searchAggregatedTransactions: boolean; }) { - return withApmSpan( - 'observability_overview_get_transactions_per_minute', - async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const { aggregations } = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], + const { aggregations } = await apmEventClient.search( + 'observability_overview_get_transactions_per_minute', + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: rangeQuery(start, end), + }, }, - body: { - size: 0, - query: { - bool: { - filter: rangeQuery(start, end), + aggs: { + transactionType: { + terms: { + field: TRANSACTION_TYPE, }, - }, - aggs: { - transactionType: { - terms: { - field: TRANSACTION_TYPE, - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: bucketSize, - min_doc_count: 0, - }, - aggs: { - throughput: { rate: { unit: 'minute' as const } }, - }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: bucketSize, + min_doc_count: 0, + }, + aggs: { + throughput: { rate: { unit: 'minute' as const } }, }, }, }, }, }, - }); + }, + } + ); - if (!aggregations || !aggregations.transactionType.buckets) { - return { value: undefined, timeseries: [] }; - } + if (!aggregations || !aggregations.transactionType.buckets) { + return { value: undefined, timeseries: [] }; + } - const topTransactionTypeBucket = - aggregations.transactionType.buckets.find( - ({ key: transactionType }) => - transactionType === TRANSACTION_REQUEST || - transactionType === TRANSACTION_PAGE_LOAD - ) || aggregations.transactionType.buckets[0]; + const topTransactionTypeBucket = + aggregations.transactionType.buckets.find( + ({ key: transactionType }) => + transactionType === TRANSACTION_REQUEST || + transactionType === TRANSACTION_PAGE_LOAD + ) || aggregations.transactionType.buckets[0]; - return { - value: calculateThroughput({ - start, - end, - value: topTransactionTypeBucket?.doc_count || 0, - }), - timeseries: - topTransactionTypeBucket?.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.throughput.value, - })) || [], - }; - } - ); + return { + value: calculateThroughput({ + start, + end, + value: topTransactionTypeBucket?.doc_count || 0, + }), + timeseries: + topTransactionTypeBucket?.timeseries.buckets.map((bucket) => ({ + x: bucket.key, + y: bucket.throughput.value, + })) || [], + }; } diff --git a/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts b/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts index bbe13874d7d3b..5c1a33e750e12 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts @@ -6,31 +6,31 @@ */ import { ProcessorEvent } from '../../../common/processor_event'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup } from '../helpers/setup_request'; -export function getHasData({ setup }: { setup: Setup }) { - return withApmSpan('observability_overview_has_apm_data', async () => { - const { apmEventClient } = setup; - try { - const params = { - apm: { - events: [ - ProcessorEvent.transaction, - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - terminateAfter: 1, - body: { - size: 0, - }, - }; +export async function getHasData({ setup }: { setup: Setup }) { + const { apmEventClient } = setup; + try { + const params = { + apm: { + events: [ + ProcessorEvent.transaction, + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + terminateAfter: 1, + body: { + size: 0, + }, + }; - const response = await apmEventClient.search(params); - return response.hits.total.value > 0; - } catch (e) { - return false; - } - }); + const response = await apmEventClient.search( + 'observability_overview_has_apm_data', + params + ); + return response.hits.total.value > 0; + } catch (e) { + return false; + } } diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts b/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts index ea1a65020e0cc..e56f234c0633e 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts @@ -63,7 +63,7 @@ export async function getClientMetrics({ }); const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_client_metrics', params); const { hasFetchStartField: { backEnd, totalPageLoadDuration }, } = response.aggregations!; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts b/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts index d65399f91bef8..6f734a214501d 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts @@ -94,7 +94,7 @@ export async function getJSErrors({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_js_errors', params); const { totalErrorGroups, totalErrorPages, errors } = response.aggregations ?? {}; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts b/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts index c873b9b4aed82..c4c6f613172d1 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts @@ -64,7 +64,7 @@ export async function getLongTaskMetrics({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_long_task_metrics', params); const pkey = percentile.toFixed(1); diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts b/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts index 0f1d7146f8459..73d634e3134d1 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts @@ -117,7 +117,7 @@ export async function getPageLoadDistribution({ const { aggregations, hits: { total }, - } = await apmEventClient.search(params); + } = await apmEventClient.search('get_page_load_distribution', params); if (total.value === 0) { return null; @@ -210,7 +210,10 @@ const getPercentilesDistribution = async ({ const { apmEventClient } = setup; - const { aggregations } = await apmEventClient.search(params); + const { aggregations } = await apmEventClient.search( + 'get_page_load_distribution', + params + ); return aggregations?.loadDistribution.values ?? []; }; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts b/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts index 13046e6c5a873..41af2ae166aaf 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts @@ -69,7 +69,7 @@ export async function getPageViewTrends({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_page_view_trends', params); const { topBreakdowns } = response.aggregations ?? {}; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts b/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts index 6a6caab953733..e63d834307a5f 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts @@ -92,7 +92,10 @@ export const getPageLoadDistBreakdown = async ({ const { apmEventClient } = setup; - const { aggregations } = await apmEventClient.search(params); + const { aggregations } = await apmEventClient.search( + 'get_page_load_dist_breakdown', + params + ); const pageDistBreakdowns = aggregations?.breakdowns.buckets; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts b/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts index ffe9225f1ab99..a2e6b55738d3a 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts @@ -38,7 +38,7 @@ export async function getRumServices({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_rum_services', params); const result = response.aggregations?.services.buckets ?? []; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts b/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts index 7cf9066fb4d4d..ae65cdbd121ea 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts @@ -56,7 +56,7 @@ export async function getUrlSearch({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_url_search', params); const { urls, totalUrls } = response.aggregations ?? {}; const pkey = percentile.toFixed(1); diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts b/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts index 247b808896e41..9c7a64d7c6481 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts @@ -51,7 +51,7 @@ export async function getVisitorBreakdown({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_visitor_breakdown', params); const { browsers, os } = response.aggregations!; const totalItems = response.hits.total.value; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts b/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts index 9bde701df5672..bbb301e22aa8d 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts @@ -103,7 +103,7 @@ export async function getWebCoreVitals({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_web_core_vitals', params); const { lcp, cls, diff --git a/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts b/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts index 87136fc0538a6..fc5da4ec1d0fa 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts @@ -51,7 +51,7 @@ export async function hasRumData({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('has_rum_data', params); return { indices: setup.indices['apm_oss.transactionIndices']!, hasData: response.hits.total.value > 0, diff --git a/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts b/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts index 8fdeb77171862..e0e9bb2ca002f 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts @@ -38,37 +38,38 @@ export function getLocalUIFilters({ delete projectionWithoutAggs.body.aggs; return Promise.all( - localFilterNames.map(async (name) => - withApmSpan('get_ui_filter_options_for_field', async () => { - const query = getLocalFilterQuery({ - uiFilters, - projection, - localUIFilterName: name, - }); + localFilterNames.map(async (name) => { + const query = getLocalFilterQuery({ + uiFilters, + projection, + localUIFilterName: name, + }); - const response = await apmEventClient.search(query); + const response = await apmEventClient.search( + 'get_ui_filter_options_for_field', + query + ); - const filter = localUIFilters[name]; + const filter = localUIFilters[name]; - const buckets = response?.aggregations?.by_terms?.buckets ?? []; + const buckets = response?.aggregations?.by_terms?.buckets ?? []; - return { - ...filter, - options: orderBy( - buckets.map((bucket) => { - return { - name: bucket.key as string, - count: bucket.bucket_count - ? bucket.bucket_count.value - : bucket.doc_count, - }; - }), - 'count', - 'desc' - ), - }; - }) - ) + return { + ...filter, + options: orderBy( + buckets.map((bucket) => { + return { + name: bucket.key as string, + count: bucket.bucket_count + ? bucket.bucket_count.value + : bucket.doc_count, + }; + }), + 'count', + 'desc' + ), + }; + }) ); }); } diff --git a/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts b/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts index 6047b97651e6a..6ecfe425dc8c5 100644 --- a/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts +++ b/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts @@ -14,44 +14,42 @@ import { ServiceConnectionNode, } from '../../../common/service_map'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -import { withApmSpan } from '../../utils/with_apm_span'; export async function fetchServicePathsFromTraceIds( setup: Setup & SetupTimeRange, traceIds: string[] ) { - return withApmSpan('get_service_paths_from_trace_ids', async () => { - const { apmEventClient } = setup; - - // make sure there's a range so ES can skip shards - const dayInMs = 24 * 60 * 60 * 1000; - const start = setup.start - dayInMs; - const end = setup.end + dayInMs; - - const serviceMapParams = { - apm: { - events: [ProcessorEvent.span, ProcessorEvent.transaction], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { - terms: { - [TRACE_ID]: traceIds, - }, + const { apmEventClient } = setup; + + // make sure there's a range so ES can skip shards + const dayInMs = 24 * 60 * 60 * 1000; + const start = setup.start - dayInMs; + const end = setup.end + dayInMs; + + const serviceMapParams = { + apm: { + events: [ProcessorEvent.span, ProcessorEvent.transaction], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { + terms: { + [TRACE_ID]: traceIds, }, - ...rangeQuery(start, end), - ], - }, + }, + ...rangeQuery(start, end), + ], }, - aggs: { - service_map: { - scripted_metric: { - init_script: { - lang: 'painless', - source: `state.eventsById = new HashMap(); + }, + aggs: { + service_map: { + scripted_metric: { + init_script: { + lang: 'painless', + source: `state.eventsById = new HashMap(); String[] fieldsToCopy = new String[] { 'parent.id', @@ -65,10 +63,10 @@ export async function fetchServicePathsFromTraceIds( 'agent.name' }; state.fieldsToCopy = fieldsToCopy;`, - }, - map_script: { - lang: 'painless', - source: `def id; + }, + map_script: { + lang: 'painless', + source: `def id; if (!doc['span.id'].empty) { id = doc['span.id'].value; } else { @@ -85,14 +83,14 @@ export async function fetchServicePathsFromTraceIds( } state.eventsById[id] = copy`, - }, - combine_script: { - lang: 'painless', - source: `return state.eventsById;`, - }, - reduce_script: { - lang: 'painless', - source: ` + }, + combine_script: { + lang: 'painless', + source: `return state.eventsById;`, + }, + reduce_script: { + lang: 'painless', + source: ` def getDestination ( def event ) { def destination = new HashMap(); destination['span.destination.service.resource'] = event['span.destination.service.resource']; @@ -208,29 +206,29 @@ export async function fetchServicePathsFromTraceIds( response.discoveredServices = discoveredServices; return response;`, - }, }, }, } as const, }, - }; - - const serviceMapFromTraceIdsScriptResponse = await apmEventClient.search( - serviceMapParams - ); - - return serviceMapFromTraceIdsScriptResponse as { - aggregations?: { - service_map: { - value: { - paths: ConnectionNode[][]; - discoveredServices: Array<{ - from: ExternalConnectionNode; - to: ServiceConnectionNode; - }>; - }; + }, + }; + + const serviceMapFromTraceIdsScriptResponse = await apmEventClient.search( + 'get_service_paths_from_trace_ids', + serviceMapParams + ); + + return serviceMapFromTraceIdsScriptResponse as { + aggregations?: { + service_map: { + value: { + paths: ConnectionNode[][]; + discoveredServices: Array<{ + from: ExternalConnectionNode; + to: ServiceConnectionNode; + }>; }; }; }; - }); + }; } diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts index e5b0b72b8784a..6d50023d3fd0e 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts @@ -87,69 +87,70 @@ async function getConnectionData({ } async function getServicesData(options: IEnvOptions) { - return withApmSpan('get_service_stats_for_service_map', async () => { - const { environment, setup, searchAggregatedTransactions } = options; + const { environment, setup, searchAggregatedTransactions } = options; - const projection = getServicesProjection({ - setup, - searchAggregatedTransactions, - }); + const projection = getServicesProjection({ + setup, + searchAggregatedTransactions, + }); - let filter = [ - ...projection.body.query.bool.filter, - ...environmentQuery(environment), - ]; + let filter = [ + ...projection.body.query.bool.filter, + ...environmentQuery(environment), + ]; - if (options.serviceName) { - filter = filter.concat({ - term: { - [SERVICE_NAME]: options.serviceName, + if (options.serviceName) { + filter = filter.concat({ + term: { + [SERVICE_NAME]: options.serviceName, + }, + }); + } + + const params = mergeProjection(projection, { + body: { + size: 0, + query: { + bool: { + ...projection.body.query.bool, + filter, }, - }); - } - - const params = mergeProjection(projection, { - body: { - size: 0, - query: { - bool: { - ...projection.body.query.bool, - filter, + }, + aggs: { + services: { + terms: { + field: projection.body.aggs.services.terms.field, + size: 500, }, - }, - aggs: { - services: { - terms: { - field: projection.body.aggs.services.terms.field, - size: 500, - }, - aggs: { - agent_name: { - terms: { - field: AGENT_NAME, - }, + aggs: { + agent_name: { + terms: { + field: AGENT_NAME, }, }, }, }, }, - }); + }, + }); - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search( + 'get_service_stats_for_service_map', + params + ); - return ( - response.aggregations?.services.buckets.map((bucket) => { - return { - [SERVICE_NAME]: bucket.key as string, - [AGENT_NAME]: - (bucket.agent_name.buckets[0]?.key as string | undefined) || '', - [SERVICE_ENVIRONMENT]: options.environment || null, - }; - }) || [] - ); - }); + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + [SERVICE_NAME]: bucket.key as string, + [AGENT_NAME]: + (bucket.agent_name.buckets[0]?.key as string | undefined) || '', + [SERVICE_ENVIRONMENT]: options.environment || null, + }; + }) || [] + ); } export type ConnectionsResponse = PromiseReturnType; diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts index 9850c36c573dd..2709fb640d8ce 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts @@ -120,7 +120,7 @@ async function getErrorStats({ }); } -function getTransactionStats({ +async function getTransactionStats({ setup, filter, minutes, @@ -129,68 +129,70 @@ function getTransactionStats({ avgTransactionDuration: number | null; avgRequestsPerMinute: number | null; }> { - return withApmSpan('get_transaction_stats_for_service_map_node', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - ...filter, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - { - terms: { - [TRANSACTION_TYPE]: [ - TRANSACTION_REQUEST, - TRANSACTION_PAGE_LOAD, - ], - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...filter, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + { + terms: { + [TRANSACTION_TYPE]: [ + TRANSACTION_REQUEST, + TRANSACTION_PAGE_LOAD, + ], }, - ], - }, - }, - track_total_hits: true, - aggs: { - duration: { - avg: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), }, + ], + }, + }, + track_total_hits: true, + aggs: { + duration: { + avg: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, }, }, - }; - const response = await apmEventClient.search(params); + }, + }; + const response = await apmEventClient.search( + 'get_transaction_stats_for_service_map_node', + params + ); - const totalRequests = response.hits.total.value; + const totalRequests = response.hits.total.value; - return { - avgTransactionDuration: response.aggregations?.duration.value ?? null, - avgRequestsPerMinute: totalRequests > 0 ? totalRequests / minutes : null, - }; - }); + return { + avgTransactionDuration: response.aggregations?.duration.value ?? null, + avgRequestsPerMinute: totalRequests > 0 ? totalRequests / minutes : null, + }; } -function getCpuStats({ +async function getCpuStats({ setup, filter, }: TaskParameters): Promise<{ avgCpuUsage: number | null }> { - return withApmSpan('get_avg_cpu_usage_for_service_map_node', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_avg_cpu_usage_for_service_map_node', + { apm: { events: [ProcessorEvent.metric], }, @@ -206,10 +208,10 @@ function getCpuStats({ }, aggs: { avgCpuUsage: { avg: { field: METRIC_SYSTEM_CPU_PERCENT } } }, }, - }); + } + ); - return { avgCpuUsage: response.aggregations?.avgCpuUsage.value ?? null }; - }); + return { avgCpuUsage: response.aggregations?.avgCpuUsage.value ?? null }; } function getMemoryStats({ @@ -219,7 +221,7 @@ function getMemoryStats({ return withApmSpan('get_memory_stats_for_service_map_node', async () => { const { apmEventClient } = setup; - const getAvgMemoryUsage = ({ + const getAvgMemoryUsage = async ({ additionalFilters, script, }: { @@ -228,8 +230,9 @@ function getMemoryStats({ | typeof percentCgroupMemoryUsedScript | typeof percentSystemMemoryUsedScript; }) => { - return withApmSpan('get_avg_memory_for_service_map_node', async () => { - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_avg_memory_for_service_map_node', + { apm: { events: [ProcessorEvent.metric], }, @@ -244,9 +247,9 @@ function getMemoryStats({ avgMemoryUsage: { avg: { script } }, }, }, - }); - return response.aggregations?.avgMemoryUsage.value ?? null; - }); + } + ); + return response.aggregations?.avgMemoryUsage.value ?? null; }; let avgMemoryUsage = await getAvgMemoryUsage({ diff --git a/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts b/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts index fa04b963388b2..7894a95cf4d7e 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts @@ -18,12 +18,11 @@ import { import { ProcessorEvent } from '../../../common/processor_event'; import { SERVICE_MAP_TIMEOUT_ERROR } from '../../../common/service_map'; import { environmentQuery, rangeQuery } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; const MAX_TRACES_TO_INSPECT = 1000; -export function getTraceSampleIds({ +export async function getTraceSampleIds({ serviceName, environment, setup, @@ -32,90 +31,88 @@ export function getTraceSampleIds({ environment?: string; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_trace_sample_ids', async () => { - const { start, end, apmEventClient, config } = setup; + const { start, end, apmEventClient, config } = setup; - const query = { - bool: { - filter: [ - { - exists: { - field: SPAN_DESTINATION_SERVICE_RESOURCE, - }, + const query = { + bool: { + filter: [ + { + exists: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, }, - ...rangeQuery(start, end), - ] as ESFilter[], - }, - } as { bool: { filter: ESFilter[]; must_not?: ESFilter[] | ESFilter } }; + }, + ...rangeQuery(start, end), + ] as ESFilter[], + }, + } as { bool: { filter: ESFilter[]; must_not?: ESFilter[] | ESFilter } }; - if (serviceName) { - query.bool.filter.push({ term: { [SERVICE_NAME]: serviceName } }); - } + if (serviceName) { + query.bool.filter.push({ term: { [SERVICE_NAME]: serviceName } }); + } - query.bool.filter.push(...environmentQuery(environment)); + query.bool.filter.push(...environmentQuery(environment)); - const fingerprintBucketSize = serviceName - ? config['xpack.apm.serviceMapFingerprintBucketSize'] - : config['xpack.apm.serviceMapFingerprintGlobalBucketSize']; + const fingerprintBucketSize = serviceName + ? config['xpack.apm.serviceMapFingerprintBucketSize'] + : config['xpack.apm.serviceMapFingerprintGlobalBucketSize']; - const traceIdBucketSize = serviceName - ? config['xpack.apm.serviceMapTraceIdBucketSize'] - : config['xpack.apm.serviceMapTraceIdGlobalBucketSize']; + const traceIdBucketSize = serviceName + ? config['xpack.apm.serviceMapTraceIdBucketSize'] + : config['xpack.apm.serviceMapTraceIdGlobalBucketSize']; - const samplerShardSize = traceIdBucketSize * 10; + const samplerShardSize = traceIdBucketSize * 10; - const params = { - apm: { - events: [ProcessorEvent.span], - }, - body: { - size: 0, - query, - aggs: { - connections: { - composite: { - sources: asMutableArray([ - { - [SPAN_DESTINATION_SERVICE_RESOURCE]: { - terms: { - field: SPAN_DESTINATION_SERVICE_RESOURCE, - }, + const params = { + apm: { + events: [ProcessorEvent.span], + }, + body: { + size: 0, + query, + aggs: { + connections: { + composite: { + sources: asMutableArray([ + { + [SPAN_DESTINATION_SERVICE_RESOURCE]: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, }, }, - { - [SERVICE_NAME]: { - terms: { - field: SERVICE_NAME, - }, + }, + { + [SERVICE_NAME]: { + terms: { + field: SERVICE_NAME, }, }, - { - [SERVICE_ENVIRONMENT]: { - terms: { - field: SERVICE_ENVIRONMENT, - missing_bucket: true, - }, + }, + { + [SERVICE_ENVIRONMENT]: { + terms: { + field: SERVICE_ENVIRONMENT, + missing_bucket: true, }, }, - ] as const), - size: fingerprintBucketSize, - }, - aggs: { - sample: { - sampler: { - shard_size: samplerShardSize, - }, - aggs: { - trace_ids: { - terms: { - field: TRACE_ID, - size: traceIdBucketSize, - execution_hint: 'map' as const, - // remove bias towards large traces by sorting on trace.id - // which will be random-esque - order: { - _key: 'desc' as const, - }, + }, + ] as const), + size: fingerprintBucketSize, + }, + aggs: { + sample: { + sampler: { + shard_size: samplerShardSize, + }, + aggs: { + trace_ids: { + terms: { + field: TRACE_ID, + size: traceIdBucketSize, + execution_hint: 'map' as const, + // remove bias towards large traces by sorting on trace.id + // which will be random-esque + order: { + _key: 'desc' as const, }, }, }, @@ -124,34 +121,36 @@ export function getTraceSampleIds({ }, }, }, - }; + }, + }; - try { - const tracesSampleResponse = await apmEventClient.search(params); - // make sure at least one trace per composite/connection bucket - // is queried - const traceIdsWithPriority = - tracesSampleResponse.aggregations?.connections.buckets.flatMap( - (bucket) => - bucket.sample.trace_ids.buckets.map((sampleDocBucket, index) => ({ - traceId: sampleDocBucket.key as string, - priority: index, - })) - ) || []; + try { + const tracesSampleResponse = await apmEventClient.search( + 'get_trace_sample_ids', + params + ); + // make sure at least one trace per composite/connection bucket + // is queried + const traceIdsWithPriority = + tracesSampleResponse.aggregations?.connections.buckets.flatMap((bucket) => + bucket.sample.trace_ids.buckets.map((sampleDocBucket, index) => ({ + traceId: sampleDocBucket.key as string, + priority: index, + })) + ) || []; - const traceIds = take( - uniq( - sortBy(traceIdsWithPriority, 'priority').map(({ traceId }) => traceId) - ), - MAX_TRACES_TO_INSPECT - ); + const traceIds = take( + uniq( + sortBy(traceIdsWithPriority, 'priority').map(({ traceId }) => traceId) + ), + MAX_TRACES_TO_INSPECT + ); - return { traceIds }; - } catch (error) { - if ('displayName' in error && error.displayName === 'RequestTimeout') { - throw Boom.internal(SERVICE_MAP_TIMEOUT_ERROR); - } - throw error; + return { traceIds }; + } catch (error) { + if ('displayName' in error && error.displayName === 'RequestTimeout') { + throw Boom.internal(SERVICE_MAP_TIMEOUT_ERROR); } - }); + throw error; + } } diff --git a/x-pack/plugins/apm/server/lib/service_nodes/index.ts b/x-pack/plugins/apm/server/lib/service_nodes/index.ts index 07b7e532d8055..97c553f344205 100644 --- a/x-pack/plugins/apm/server/lib/service_nodes/index.ts +++ b/x-pack/plugins/apm/server/lib/service_nodes/index.ts @@ -14,10 +14,9 @@ import { import { SERVICE_NODE_NAME_MISSING } from '../../../common/service_nodes'; import { getServiceNodesProjection } from '../../projections/service_nodes'; import { mergeProjection } from '../../projections/util/merge_projection'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -const getServiceNodes = ({ +const getServiceNodes = async ({ kuery, setup, serviceName, @@ -26,69 +25,67 @@ const getServiceNodes = ({ setup: Setup & SetupTimeRange; serviceName: string; }) => { - return withApmSpan('get_service_nodes', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const projection = getServiceNodesProjection({ kuery, setup, serviceName }); + const projection = getServiceNodesProjection({ kuery, setup, serviceName }); - const params = mergeProjection(projection, { - body: { - aggs: { - nodes: { - terms: { - ...projection.body.aggs.nodes.terms, - size: 10000, - missing: SERVICE_NODE_NAME_MISSING, - }, - aggs: { - cpu: { - avg: { - field: METRIC_PROCESS_CPU_PERCENT, - }, + const params = mergeProjection(projection, { + body: { + aggs: { + nodes: { + terms: { + ...projection.body.aggs.nodes.terms, + size: 10000, + missing: SERVICE_NODE_NAME_MISSING, + }, + aggs: { + cpu: { + avg: { + field: METRIC_PROCESS_CPU_PERCENT, }, - heapMemory: { - avg: { - field: METRIC_JAVA_HEAP_MEMORY_USED, - }, + }, + heapMemory: { + avg: { + field: METRIC_JAVA_HEAP_MEMORY_USED, }, - nonHeapMemory: { - avg: { - field: METRIC_JAVA_NON_HEAP_MEMORY_USED, - }, + }, + nonHeapMemory: { + avg: { + field: METRIC_JAVA_NON_HEAP_MEMORY_USED, }, - threadCount: { - max: { - field: METRIC_JAVA_THREAD_COUNT, - }, + }, + threadCount: { + max: { + field: METRIC_JAVA_THREAD_COUNT, }, }, }, }, }, - }); + }, + }); - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_service_nodes', params); - if (!response.aggregations) { - return []; - } + if (!response.aggregations) { + return []; + } - return response.aggregations.nodes.buckets - .map((bucket) => ({ - name: bucket.key as string, - cpu: bucket.cpu.value, - heapMemory: bucket.heapMemory.value, - nonHeapMemory: bucket.nonHeapMemory.value, - threadCount: bucket.threadCount.value, - })) - .filter( - (item) => - item.cpu !== null || - item.heapMemory !== null || - item.nonHeapMemory !== null || - item.threadCount != null - ); - }); + return response.aggregations.nodes.buckets + .map((bucket) => ({ + name: bucket.key as string, + cpu: bucket.cpu.value, + heapMemory: bucket.heapMemory.value, + nonHeapMemory: bucket.nonHeapMemory.value, + threadCount: bucket.threadCount.value, + })) + .filter( + (item) => + item.cpu !== null || + item.heapMemory !== null || + item.nonHeapMemory !== null || + item.threadCount != null + ); }; export { getServiceNodes }; diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index 9d05369aca840..2f653e2c4df1d 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -22,6 +22,7 @@ Object { "events": Array [ "transaction", ], + "includeLegacyData": true, }, "body": Object { "query": Object { diff --git a/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts index 611f9b18a0b1a..202b5075d2ea7 100644 --- a/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts @@ -13,7 +13,6 @@ import { SERVICE_VERSION, } from '../../../../common/elasticsearch_fieldnames'; import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, @@ -31,20 +30,52 @@ export async function getDerivedServiceAnnotations({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_derived_service_annotations', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const filter: ESFilter[] = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ...environmentQuery(environment), - ]; + const filter: ESFilter[] = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...environmentQuery(environment), + ]; - const versions = - ( - await apmEventClient.search({ + const versions = + ( + await apmEventClient.search('get_derived_service_annotations', { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [...filter, ...rangeQuery(start, end)], + }, + }, + aggs: { + versions: { + terms: { + field: SERVICE_VERSION, + }, + }, + }, + }, + }) + ).aggregations?.versions.buckets.map((bucket) => bucket.key) ?? []; + + if (versions.length <= 1) { + return []; + } + const annotations = await Promise.all( + versions.map(async (version) => { + const response = await apmEventClient.search( + 'get_first_seen_of_version', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -53,73 +84,40 @@ export async function getDerivedServiceAnnotations({ ], }, body: { - size: 0, + size: 1, query: { bool: { - filter: [...filter, ...rangeQuery(start, end)], + filter: [...filter, { term: { [SERVICE_VERSION]: version } }], }, }, - aggs: { - versions: { - terms: { - field: SERVICE_VERSION, - }, - }, + sort: { + '@timestamp': 'asc', }, }, - }) - ).aggregations?.versions.buckets.map((bucket) => bucket.key) ?? []; + } + ); - if (versions.length <= 1) { - return []; - } - const annotations = await Promise.all( - versions.map(async (version) => { - return withApmSpan('get_first_seen_of_version', async () => { - const response = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 1, - query: { - bool: { - filter: [...filter, { term: { [SERVICE_VERSION]: version } }], - }, - }, - sort: { - '@timestamp': 'asc', - }, - }, - }); - - const firstSeen = new Date( - response.hits.hits[0]._source['@timestamp'] - ).getTime(); + const firstSeen = new Date( + response.hits.hits[0]._source['@timestamp'] + ).getTime(); - if (!isFiniteNumber(firstSeen)) { - throw new Error( - 'First seen for version was unexpectedly undefined or null.' - ); - } + if (!isFiniteNumber(firstSeen)) { + throw new Error( + 'First seen for version was unexpectedly undefined or null.' + ); + } - if (firstSeen < start || firstSeen > end) { - return null; - } + if (firstSeen < start || firstSeen > end) { + return null; + } - return { - type: AnnotationType.VERSION, - id: version, - '@timestamp': firstSeen, - text: version, - }; - }); - }) - ); - return annotations.filter(Boolean) as Annotation[]; - }); + return { + type: AnnotationType.VERSION, + id: version, + '@timestamp': firstSeen, + text: version, + }; + }) + ); + return annotations.filter(Boolean) as Annotation[]; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts b/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts index a81c0b2fc2c44..82147d7c94236 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts @@ -13,9 +13,8 @@ import { import { rangeQuery } from '../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceAgentName({ +export async function getServiceAgentName({ serviceName, setup, searchAggregatedTransactions, @@ -24,42 +23,41 @@ export function getServiceAgentName({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_service_agent_name', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ - ProcessorEvent.error, - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.metric, - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ], - }, + const params = { + terminateAfter: 1, + apm: { + events: [ + ProcessorEvent.error, + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.metric, + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ], }, - aggs: { - agents: { - terms: { field: AGENT_NAME, size: 1 }, - }, + }, + aggs: { + agents: { + terms: { field: AGENT_NAME, size: 1 }, }, }, - }; + }, + }; - const { aggregations } = await apmEventClient.search(params); - const agentName = aggregations?.agents.buckets[0]?.key as - | string - | undefined; - return { agentName }; - }); + const { aggregations } = await apmEventClient.search( + 'get_service_agent_name', + params + ); + const agentName = aggregations?.agents.buckets[0]?.key as string | undefined; + return { agentName }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts index db491012c986b..4993484f5b240 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts @@ -38,56 +38,54 @@ export const getDestinationMap = ({ return withApmSpan('get_service_destination_map', async () => { const { start, end, apmEventClient } = setup; - const response = await withApmSpan('get_exit_span_samples', async () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.span], + const response = await apmEventClient.search('get_exit_span_samples', { + apm: { + events: [ProcessorEvent.span], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { exists: { field: SPAN_DESTINATION_SERVICE_RESOURCE } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { exists: { field: SPAN_DESTINATION_SERVICE_RESOURCE } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ], + aggs: { + connections: { + composite: { + size: 1000, + sources: asMutableArray([ + { + [SPAN_DESTINATION_SERVICE_RESOURCE]: { + terms: { field: SPAN_DESTINATION_SERVICE_RESOURCE }, + }, + }, + // make sure we get samples for both successful + // and failed calls + { [EVENT_OUTCOME]: { terms: { field: EVENT_OUTCOME } } }, + ] as const), }, - }, - aggs: { - connections: { - composite: { - size: 1000, - sources: asMutableArray([ - { - [SPAN_DESTINATION_SERVICE_RESOURCE]: { - terms: { field: SPAN_DESTINATION_SERVICE_RESOURCE }, + aggs: { + sample: { + top_hits: { + size: 1, + _source: [SPAN_TYPE, SPAN_SUBTYPE, SPAN_ID], + sort: [ + { + '@timestamp': 'desc' as const, }, - }, - // make sure we get samples for both successful - // and failed calls - { [EVENT_OUTCOME]: { terms: { field: EVENT_OUTCOME } } }, - ] as const), - }, - aggs: { - sample: { - top_hits: { - size: 1, - _source: [SPAN_TYPE, SPAN_SUBTYPE, SPAN_ID], - sort: [ - { - '@timestamp': 'desc' as const, - }, - ], - }, + ], }, }, }, }, }, - }) - ); + }, + }); const outgoingConnections = response.aggregations?.connections.buckets.map((bucket) => { @@ -103,38 +101,37 @@ export const getDestinationMap = ({ }; }) ?? []; - const transactionResponse = await withApmSpan( + const transactionResponse = await apmEventClient.search( 'get_transactions_for_exit_spans', - () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - query: { - bool: { - filter: [ - { - terms: { - [PARENT_ID]: outgoingConnections.map( - (connection) => connection[SPAN_ID] - ), - }, + { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + query: { + bool: { + filter: [ + { + terms: { + [PARENT_ID]: outgoingConnections.map( + (connection) => connection[SPAN_ID] + ), }, - ...rangeQuery(start, end), - ], - }, + }, + ...rangeQuery(start, end), + ], }, - size: outgoingConnections.length, - docvalue_fields: asMutableArray([ - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - PARENT_ID, - ] as const), - _source: false, }, - }) + size: outgoingConnections.length, + docvalue_fields: asMutableArray([ + SERVICE_NAME, + SERVICE_ENVIRONMENT, + AGENT_NAME, + PARENT_ID, + ] as const), + _source: false, + }, + } ); const incomingConnections = transactionResponse.hits.hits.map((hit) => ({ diff --git a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts index c8642c6272b5f..1d815dd7180e3 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts @@ -18,9 +18,8 @@ import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { EventOutcome } from '../../../../common/event_outcome'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export const getMetrics = ({ +export const getMetrics = async ({ setup, serviceName, environment, @@ -31,10 +30,11 @@ export const getMetrics = ({ environment?: string; numBuckets: number; }) => { - return withApmSpan('get_service_destination_metrics', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_destination_metrics', + { apm: { events: [ProcessorEvent.metric], }, @@ -46,7 +46,9 @@ export const getMetrics = ({ filter: [ { term: { [SERVICE_NAME]: serviceName } }, { - exists: { field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT }, + exists: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, + }, }, ...rangeQuery(start, end), ...environmentQuery(environment), @@ -99,47 +101,47 @@ export const getMetrics = ({ }, }, }, - }); + } + ); - return ( - response.aggregations?.connections.buckets.map((bucket) => ({ - span: { - destination: { - service: { - resource: String(bucket.key), - }, + return ( + response.aggregations?.connections.buckets.map((bucket) => ({ + span: { + destination: { + service: { + resource: String(bucket.key), }, }, - value: { - count: sum( - bucket.timeseries.buckets.map( - (dateBucket) => dateBucket.count.value ?? 0 - ) - ), - latency_sum: sum( - bucket.timeseries.buckets.map( - (dateBucket) => dateBucket.latency_sum.value ?? 0 - ) - ), - error_count: sum( - bucket.timeseries.buckets.flatMap( - (dateBucket) => - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0 - ) - ), - }, - timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - count: dateBucket.count.value ?? 0, - latency_sum: dateBucket.latency_sum.value ?? 0, - error_count: - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0, - })), - })) ?? [] - ); - }); + }, + value: { + count: sum( + bucket.timeseries.buckets.map( + (dateBucket) => dateBucket.count.value ?? 0 + ) + ), + latency_sum: sum( + bucket.timeseries.buckets.map( + (dateBucket) => dateBucket.latency_sum.value ?? 0 + ) + ), + error_count: sum( + bucket.timeseries.buckets.flatMap( + (dateBucket) => + dateBucket[EVENT_OUTCOME].buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.count.value ?? 0 + ) + ), + }, + timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + count: dateBucket.count.value ?? 0, + latency_sum: dateBucket.latency_sum.value ?? 0, + error_count: + dateBucket[EVENT_OUTCOME].buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.count.value ?? 0, + })), + })) ?? [] + ); }; diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts index e45864de2fc1e..bd69bfc53db71 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts @@ -18,7 +18,6 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; @@ -43,75 +42,71 @@ export async function getServiceErrorGroupDetailedStatistics({ start: number; end: number; }): Promise> { - return withApmSpan( - 'get_service_error_group_detailed_statistics', - async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets }); + const { intervalString } = getBucketSize({ start, end, numBuckets }); - const timeseriesResponse = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.error], + const timeseriesResponse = await apmEventClient.search( + 'get_service_error_group_detailed_statistics', + { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { [ERROR_GROUP_ID]: groupIds } }, + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { terms: { [ERROR_GROUP_ID]: groupIds } }, - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], + aggs: { + error_groups: { + terms: { + field: ERROR_GROUP_ID, + size: 500, }, - }, - aggs: { - error_groups: { - terms: { - field: ERROR_GROUP_ID, - size: 500, - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, - }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, }, }, }, }, }, }, - }); - - if (!timeseriesResponse.aggregations) { - return []; - } - - return timeseriesResponse.aggregations.error_groups.buckets.map( - (bucket) => { - const groupId = bucket.key as string; - return { - groupId, - timeseries: bucket.timeseries.buckets.map((timeseriesBucket) => { - return { - x: timeseriesBucket.key, - y: timeseriesBucket.doc_count, - }; - }), - }; - } - ); + }, } ); + + if (!timeseriesResponse.aggregations) { + return []; + } + + return timeseriesResponse.aggregations.error_groups.buckets.map((bucket) => { + const groupId = bucket.key as string; + return { + groupId, + timeseries: bucket.timeseries.buckets.map((timeseriesBucket) => { + return { + x: timeseriesBucket.key, + y: timeseriesBucket.doc_count, + }; + }), + }; + }); } export async function getServiceErrorGroupPeriods({ diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts index 361c92244aee0..8168c0d5549aa 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts @@ -19,11 +19,10 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getErrorName } from '../../helpers/get_error_name'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -export function getServiceErrorGroupMainStatistics({ +export async function getServiceErrorGroupMainStatistics({ kuery, serviceName, setup, @@ -36,10 +35,11 @@ export function getServiceErrorGroupMainStatistics({ transactionType: string; environment?: string; }) { - return withApmSpan('get_service_error_group_main_statistics', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_error_group_main_statistics', + { apm: { events: [ProcessorEvent.error], }, @@ -79,24 +79,23 @@ export function getServiceErrorGroupMainStatistics({ }, }, }, - }); + } + ); - const errorGroups = - response.aggregations?.error_groups.buckets.map((bucket) => ({ - group_id: bucket.key as string, - name: - getErrorName(bucket.sample.hits.hits[0]._source) ?? - NOT_AVAILABLE_LABEL, - last_seen: new Date( - bucket.sample.hits.hits[0]?._source['@timestamp'] - ).getTime(), - occurrences: bucket.doc_count, - })) ?? []; + const errorGroups = + response.aggregations?.error_groups.buckets.map((bucket) => ({ + group_id: bucket.key as string, + name: + getErrorName(bucket.sample.hits.hits[0]._source) ?? NOT_AVAILABLE_LABEL, + last_seen: new Date( + bucket.sample.hits.hits[0]?._source['@timestamp'] + ).getTime(), + occurrences: bucket.doc_count, + })) ?? []; - return { - is_aggregation_accurate: - (response.aggregations?.error_groups.sum_other_doc_count ?? 0) === 0, - error_groups: errorGroups, - }; - }); + return { + is_aggregation_accurate: + (response.aggregations?.error_groups.sum_other_doc_count ?? 0) === 0, + error_groups: errorGroups, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts index 7729822df30ca..b720c56464c30 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts @@ -59,8 +59,9 @@ export async function getServiceErrorGroups({ const { intervalString } = getBucketSize({ start, end, numBuckets }); - const response = await withApmSpan('get_top_service_error_groups', () => - apmEventClient.search({ + const response = await apmEventClient.search( + 'get_top_service_error_groups', + { apm: { events: [ProcessorEvent.error], }, @@ -104,7 +105,7 @@ export async function getServiceErrorGroups({ }, }, }, - }) + } ); const errorGroups = @@ -139,50 +140,49 @@ export async function getServiceErrorGroups({ (group) => group.group_id ); - const timeseriesResponse = await withApmSpan( + const timeseriesResponse = await apmEventClient.search( 'get_service_error_groups_timeseries', - async () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.error], + { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { [ERROR_GROUP_ID]: sortedErrorGroupIds } }, + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { terms: { [ERROR_GROUP_ID]: sortedErrorGroupIds } }, - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], + aggs: { + error_groups: { + terms: { + field: ERROR_GROUP_ID, + size, }, - }, - aggs: { - error_groups: { - terms: { - field: ERROR_GROUP_ID, - size, - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, - }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, }, }, }, }, }, }, - }) + }, + } ); return { diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts b/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts index 25935bcc37dff..bdf9530a9c0c7 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts @@ -11,7 +11,6 @@ import { TRANSACTION_TYPE, } from '../../../common/elasticsearch_fieldnames'; import { environmentQuery, kqlQuery, rangeQuery } from '../../utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; @@ -37,18 +36,19 @@ export async function getServiceInstanceMetadataDetails({ environment?: string; kuery?: string; }) { - return withApmSpan('get_service_instance_metadata_details', async () => { - const { start, end, apmEventClient } = setup; - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [SERVICE_NODE_NAME]: serviceNodeName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; + const { start, end, apmEventClient } = setup; + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [SERVICE_NODE_NAME]: serviceNodeName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_instance_metadata_details', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -61,24 +61,24 @@ export async function getServiceInstanceMetadataDetails({ size: 1, query: { bool: { filter } }, }, - }); + } + ); - const sample = response.hits.hits[0]?._source; + const sample = response.hits.hits[0]?._source; - if (!sample) { - return {}; - } + if (!sample) { + return {}; + } - const { agent, service, container, kubernetes, host, cloud } = sample; + const { agent, service, container, kubernetes, host, cloud } = sample; - return { - '@timestamp': sample['@timestamp'], - agent, - service, - container, - kubernetes, - host, - cloud, - }; - }); + return { + '@timestamp': sample['@timestamp'], + agent, + service, + container, + kubernetes, + host, + cloud, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts index 1a33e9810dd5e..526ae19143f13 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts @@ -24,7 +24,6 @@ import { percentCgroupMemoryUsedScript, percentSystemMemoryUsedScript, } from '../../metrics/by_agent/shared/memory'; -import { withApmSpan } from '../../../utils/with_apm_span'; interface ServiceInstanceSystemMetricPrimaryStatistics { serviceNodeName: string; @@ -67,142 +66,140 @@ export async function getServiceInstancesSystemMetricStatistics< size?: number; isComparisonSearch: T; }): Promise>> { - return withApmSpan( - 'get_service_instances_system_metric_statistics', - async () => { - const { apmEventClient } = setup; - - const { intervalString } = getBucketSize({ start, end, numBuckets }); - - const systemMemoryFilter = { - bool: { - filter: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, - ], - }, - }; - - const cgroupMemoryFilter = { - exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES }, - }; - - const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } }; - - function withTimeseries( - agg: TParams - ) { - return { - ...(isComparisonSearch - ? { - avg: { avg: agg }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, - }, - }, - aggs: { avg: { avg: agg } }, + const { apmEventClient } = setup; + + const { intervalString } = getBucketSize({ start, end, numBuckets }); + + const systemMemoryFilter = { + bool: { + filter: [ + { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, + { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ], + }, + }; + + const cgroupMemoryFilter = { + exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES }, + }; + + const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } }; + + function withTimeseries( + agg: TParams + ) { + return { + ...(isComparisonSearch + ? { + avg: { avg: agg }, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, }, - } - : { avg: { avg: agg } }), - }; - } - - const subAggs = { - memory_usage_cgroup: { - filter: cgroupMemoryFilter, - aggs: withTimeseries({ script: percentCgroupMemoryUsedScript }), - }, - memory_usage_system: { - filter: systemMemoryFilter, - aggs: withTimeseries({ script: percentSystemMemoryUsedScript }), - }, - cpu_usage: { - filter: cpuUsageFilter, - aggs: withTimeseries({ field: METRIC_PROCESS_CPU_PERCENT }), - }, - }; - - const response = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.metric], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ...(isComparisonSearch && serviceNodeIds - ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] - : []), - ], - should: [cgroupMemoryFilter, systemMemoryFilter, cpuUsageFilter], - minimum_should_match: 1, + }, + aggs: { avg: { avg: agg } }, }, + } + : { avg: { avg: agg } }), + }; + } + + const subAggs = { + memory_usage_cgroup: { + filter: cgroupMemoryFilter, + aggs: withTimeseries({ script: percentCgroupMemoryUsedScript }), + }, + memory_usage_system: { + filter: systemMemoryFilter, + aggs: withTimeseries({ script: percentSystemMemoryUsedScript }), + }, + cpu_usage: { + filter: cpuUsageFilter, + aggs: withTimeseries({ field: METRIC_PROCESS_CPU_PERCENT }), + }, + }; + + const response = await apmEventClient.search( + 'get_service_instances_system_metric_statistics', + { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...(isComparisonSearch && serviceNodeIds + ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] + : []), + ], + should: [cgroupMemoryFilter, systemMemoryFilter, cpuUsageFilter], + minimum_should_match: 1, }, - aggs: { - [SERVICE_NODE_NAME]: { - terms: { - field: SERVICE_NODE_NAME, - missing: SERVICE_NODE_NAME_MISSING, - ...(size ? { size } : {}), - ...(isComparisonSearch ? { include: serviceNodeIds } : {}), - }, - aggs: subAggs, + }, + aggs: { + [SERVICE_NODE_NAME]: { + terms: { + field: SERVICE_NODE_NAME, + missing: SERVICE_NODE_NAME_MISSING, + ...(size ? { size } : {}), + ...(isComparisonSearch ? { include: serviceNodeIds } : {}), }, + aggs: subAggs, }, }, - }); - - return ( - (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( - (serviceNodeBucket) => { - const serviceNodeName = String(serviceNodeBucket.key); - const hasCGroupData = - serviceNodeBucket.memory_usage_cgroup.avg.value !== null; - - const memoryMetricsKey = hasCGroupData - ? 'memory_usage_cgroup' - : 'memory_usage_system'; - - const cpuUsage = - // Timeseries is available when isComparisonSearch is true - 'timeseries' in serviceNodeBucket.cpu_usage - ? serviceNodeBucket.cpu_usage.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.avg.value, - }) - ) - : serviceNodeBucket.cpu_usage.avg.value; - - const memoryUsageValue = serviceNodeBucket[memoryMetricsKey]; - const memoryUsage = - // Timeseries is available when isComparisonSearch is true - 'timeseries' in memoryUsageValue - ? memoryUsageValue.timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.avg.value, - })) - : serviceNodeBucket[memoryMetricsKey].avg.value; - - return { - serviceNodeName, - cpuUsage, - memoryUsage, - }; - } - ) as Array>) || [] - ); + }, } ); + + return ( + (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( + (serviceNodeBucket) => { + const serviceNodeName = String(serviceNodeBucket.key); + const hasCGroupData = + serviceNodeBucket.memory_usage_cgroup.avg.value !== null; + + const memoryMetricsKey = hasCGroupData + ? 'memory_usage_cgroup' + : 'memory_usage_system'; + + const cpuUsage = + // Timeseries is available when isComparisonSearch is true + 'timeseries' in serviceNodeBucket.cpu_usage + ? serviceNodeBucket.cpu_usage.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.avg.value, + }) + ) + : serviceNodeBucket.cpu_usage.avg.value; + + const memoryUsageValue = serviceNodeBucket[memoryMetricsKey]; + const memoryUsage = + // Timeseries is available when isComparisonSearch is true + 'timeseries' in memoryUsageValue + ? memoryUsageValue.timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.avg.value, + })) + : serviceNodeBucket[memoryMetricsKey].avg.value; + + return { + serviceNodeName, + cpuUsage, + memoryUsage, + }; + } + ) as Array>) || [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts index ad54a231b52ef..7d9dca9b2a706 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts @@ -26,7 +26,6 @@ import { getLatencyValue, } from '../../helpers/latency_aggregation_type'; import { Setup } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; interface ServiceInstanceTransactionPrimaryStatistics { serviceNodeName: string; @@ -77,126 +76,124 @@ export async function getServiceInstancesTransactionStatistics< size?: number; numBuckets?: number; }): Promise>> { - return withApmSpan( - 'get_service_instances_transaction_statistics', - async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const { intervalString, bucketSize } = getBucketSize({ - start, - end, - numBuckets, - }); + const { intervalString, bucketSize } = getBucketSize({ + start, + end, + numBuckets, + }); - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); + const field = getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ); - const subAggs = { - ...getLatencyAggregation(latencyAggregationType, field), - failures: { - filter: { - term: { - [EVENT_OUTCOME]: EventOutcome.failure, - }, - }, + const subAggs = { + ...getLatencyAggregation(latencyAggregationType, field), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, }, - }; + }, + }, + }; - const query = { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ...(isComparisonSearch && serviceNodeIds - ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] - : []), - ], - }, - }; + const query = { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...(isComparisonSearch && serviceNodeIds + ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] + : []), + ], + }, + }; - const aggs = { - [SERVICE_NODE_NAME]: { - terms: { - field: SERVICE_NODE_NAME, - missing: SERVICE_NODE_NAME_MISSING, - ...(size ? { size } : {}), - ...(isComparisonSearch ? { include: serviceNodeIds } : {}), - }, - aggs: isComparisonSearch - ? { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { min: start, max: end }, - }, - aggs: subAggs, - }, - } - : subAggs, - }, - }; + const aggs = { + [SERVICE_NODE_NAME]: { + terms: { + field: SERVICE_NODE_NAME, + missing: SERVICE_NODE_NAME_MISSING, + ...(size ? { size } : {}), + ...(isComparisonSearch ? { include: serviceNodeIds } : {}), + }, + aggs: isComparisonSearch + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: subAggs, + }, + } + : subAggs, + }, + }; - const response = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { size: 0, query, aggs }, - }); + const response = await apmEventClient.search( + 'get_service_instances_transaction_statistics', + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { size: 0, query, aggs }, + } + ); - const bucketSizeInMinutes = bucketSize / 60; + const bucketSizeInMinutes = bucketSize / 60; - return ( - (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( - (serviceNodeBucket) => { - const { doc_count: count, key } = serviceNodeBucket; - const serviceNodeName = String(key); + return ( + (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( + (serviceNodeBucket) => { + const { doc_count: count, key } = serviceNodeBucket; + const serviceNodeName = String(key); - // Timeseries is returned when isComparisonSearch is true - if ('timeseries' in serviceNodeBucket) { - const { timeseries } = serviceNodeBucket; - return { - serviceNodeName, - errorRate: timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.failures.doc_count / dateBucket.doc_count, - })), - throughput: timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.doc_count / bucketSizeInMinutes, - })), - latency: timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: getLatencyValue({ - aggregation: dateBucket.latency, - latencyAggregationType, - }), - })), - }; - } else { - const { failures, latency } = serviceNodeBucket; - return { - serviceNodeName, - errorRate: failures.doc_count / count, - latency: getLatencyValue({ - aggregation: latency, - latencyAggregationType, - }), - throughput: calculateThroughput({ start, end, value: count }), - }; - } - } - ) as Array>) || [] - ); - } + // Timeseries is returned when isComparisonSearch is true + if ('timeseries' in serviceNodeBucket) { + const { timeseries } = serviceNodeBucket; + return { + serviceNodeName, + errorRate: timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.failures.doc_count / dateBucket.doc_count, + })), + throughput: timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.doc_count / bucketSizeInMinutes, + })), + latency: timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: getLatencyValue({ + aggregation: dateBucket.latency, + latencyAggregationType, + }), + })), + }; + } else { + const { failures, latency } = serviceNodeBucket; + return { + serviceNodeName, + errorRate: failures.doc_count / count, + latency: getLatencyValue({ + aggregation: latency, + latencyAggregationType, + }), + throughput: calculateThroughput({ start, end, value: count }), + }; + } + } + ) as Array>) || [] ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts b/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts index e2341b306a878..910725b005411 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts @@ -25,7 +25,6 @@ import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw' import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { should } from './get_service_metadata_icons'; -import { withApmSpan } from '../../utils/with_apm_span'; type ServiceMetadataDetailsRaw = Pick< TransactionRaw, @@ -59,7 +58,7 @@ export interface ServiceMetadataDetails { }; } -export function getServiceMetadataDetails({ +export async function getServiceMetadataDetails({ serviceName, setup, searchAggregatedTransactions, @@ -68,105 +67,106 @@ export function getServiceMetadataDetails({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }): Promise { - return withApmSpan('get_service_metadata_details', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ]; + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ]; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 1, - _source: [SERVICE, AGENT, HOST, CONTAINER_ID, KUBERNETES, CLOUD], - query: { bool: { filter, should } }, - aggs: { - serviceVersions: { - terms: { - field: SERVICE_VERSION, - size: 10, - order: { _key: 'desc' as const }, - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 1, + _source: [SERVICE, AGENT, HOST, CONTAINER_ID, KUBERNETES, CLOUD], + query: { bool: { filter, should } }, + aggs: { + serviceVersions: { + terms: { + field: SERVICE_VERSION, + size: 10, + order: { _key: 'desc' as const }, }, - availabilityZones: { - terms: { - field: CLOUD_AVAILABILITY_ZONE, - size: 10, - }, + }, + availabilityZones: { + terms: { + field: CLOUD_AVAILABILITY_ZONE, + size: 10, }, - machineTypes: { - terms: { - field: CLOUD_MACHINE_TYPE, - size: 10, - }, + }, + machineTypes: { + terms: { + field: CLOUD_MACHINE_TYPE, + size: 10, }, - totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, + totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, - }; - - const response = await apmEventClient.search(params); - - if (response.hits.total.value === 0) { - return { - service: undefined, - container: undefined, - cloud: undefined, - }; - } + }, + }; - const { service, agent, host, kubernetes, container, cloud } = response.hits - .hits[0]._source as ServiceMetadataDetailsRaw; + const response = await apmEventClient.search( + 'get_service_metadata_details', + params + ); - const serviceMetadataDetails = { - versions: response.aggregations?.serviceVersions.buckets.map( - (bucket) => bucket.key as string - ), - runtime: service.runtime, - framework: service.framework?.name, - agent, + if (response.hits.total.value === 0) { + return { + service: undefined, + container: undefined, + cloud: undefined, }; + } + + const { service, agent, host, kubernetes, container, cloud } = response.hits + .hits[0]._source as ServiceMetadataDetailsRaw; - const totalNumberInstances = - response.aggregations?.totalNumberInstances.value; + const serviceMetadataDetails = { + versions: response.aggregations?.serviceVersions.buckets.map( + (bucket) => bucket.key as string + ), + runtime: service.runtime, + framework: service.framework?.name, + agent, + }; - const containerDetails = - host || container || totalNumberInstances || kubernetes - ? { - os: host?.os?.platform, - type: (!!kubernetes ? 'Kubernetes' : 'Docker') as ContainerType, - isContainerized: !!container?.id, - totalNumberInstances, - } - : undefined; + const totalNumberInstances = + response.aggregations?.totalNumberInstances.value; - const cloudDetails = cloud + const containerDetails = + host || container || totalNumberInstances || kubernetes ? { - provider: cloud.provider, - projectName: cloud.project?.name, - availabilityZones: response.aggregations?.availabilityZones.buckets.map( - (bucket) => bucket.key as string - ), - machineTypes: response.aggregations?.machineTypes.buckets.map( - (bucket) => bucket.key as string - ), + os: host?.os?.platform, + type: (!!kubernetes ? 'Kubernetes' : 'Docker') as ContainerType, + isContainerized: !!container?.id, + totalNumberInstances, } : undefined; - return { - service: serviceMetadataDetails, - container: containerDetails, - cloud: cloudDetails, - }; - }); + const cloudDetails = cloud + ? { + provider: cloud.provider, + projectName: cloud.project?.name, + availabilityZones: response.aggregations?.availabilityZones.buckets.map( + (bucket) => bucket.key as string + ), + machineTypes: response.aggregations?.machineTypes.buckets.map( + (bucket) => bucket.key as string + ), + } + : undefined; + + return { + service: serviceMetadataDetails, + container: containerDetails, + cloud: cloudDetails, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts b/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts index 94da6545c5e90..469c788a6cf17 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts @@ -20,7 +20,6 @@ import { rangeQuery } from '../../../server/utils/queries'; import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -import { withApmSpan } from '../../utils/with_apm_span'; type ServiceMetadataIconsRaw = Pick< TransactionRaw, @@ -41,7 +40,7 @@ export const should = [ { exists: { field: AGENT_NAME } }, ]; -export function getServiceMetadataIcons({ +export async function getServiceMetadataIcons({ serviceName, setup, searchAggregatedTransactions, @@ -50,55 +49,56 @@ export function getServiceMetadataIcons({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }): Promise { - return withApmSpan('get_service_metadata_icons', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ]; + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ]; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 1, - _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME], - query: { bool: { filter, should } }, - }, - }; + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 1, + _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME], + query: { bool: { filter, should } }, + }, + }; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search( + 'get_service_metadata_icons', + params + ); - if (response.hits.total.value === 0) { - return { - agentName: undefined, - containerType: undefined, - cloudProvider: undefined, - }; - } + if (response.hits.total.value === 0) { + return { + agentName: undefined, + containerType: undefined, + cloudProvider: undefined, + }; + } - const { kubernetes, cloud, container, agent } = response.hits.hits[0] - ._source as ServiceMetadataIconsRaw; + const { kubernetes, cloud, container, agent } = response.hits.hits[0] + ._source as ServiceMetadataIconsRaw; - let containerType: ContainerType; - if (!!kubernetes) { - containerType = 'Kubernetes'; - } else if (!!container) { - containerType = 'Docker'; - } + let containerType: ContainerType; + if (!!kubernetes) { + containerType = 'Kubernetes'; + } else if (!!container) { + containerType = 'Docker'; + } - return { - agentName: agent?.name, - containerType, - cloudProvider: cloud?.provider, - }; - }); + return { + agentName: agent?.name, + containerType, + cloudProvider: cloud?.provider, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts b/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts index 8eaf9e96c7fd9..0f0c174179052 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts @@ -13,9 +13,8 @@ import { import { NOT_AVAILABLE_LABEL } from '../../../common/i18n'; import { mergeProjection } from '../../projections/util/merge_projection'; import { getServiceNodesProjection } from '../../projections/service_nodes'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceNodeMetadata({ +export async function getServiceNodeMetadata({ kuery, serviceName, serviceNodeName, @@ -26,44 +25,44 @@ export function getServiceNodeMetadata({ serviceNodeName: string; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_service_node_metadata', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const query = mergeProjection( - getServiceNodesProjection({ - kuery, - setup, - serviceName, - serviceNodeName, - }), - { - body: { - size: 0, - aggs: { - host: { - terms: { - field: HOST_NAME, - size: 1, - }, + const query = mergeProjection( + getServiceNodesProjection({ + kuery, + setup, + serviceName, + serviceNodeName, + }), + { + body: { + size: 0, + aggs: { + host: { + terms: { + field: HOST_NAME, + size: 1, }, - containerId: { - terms: { - field: CONTAINER_ID, - size: 1, - }, + }, + containerId: { + terms: { + field: CONTAINER_ID, + size: 1, }, }, }, - } - ); + }, + } + ); - const response = await apmEventClient.search(query); + const response = await apmEventClient.search( + 'get_service_node_metadata', + query + ); - return { - host: response.aggregations?.host.buckets[0]?.key || NOT_AVAILABLE_LABEL, - containerId: - response.aggregations?.containerId.buckets[0]?.key || - NOT_AVAILABLE_LABEL, - }; - }); + return { + host: response.aggregations?.host.buckets[0]?.key || NOT_AVAILABLE_LABEL, + containerId: + response.aggregations?.containerId.buckets[0]?.key || NOT_AVAILABLE_LABEL, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts index f14dba69bf404..36d372e322cbc 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts @@ -21,7 +21,6 @@ import { kqlQuery, } from '../../../server/utils/queries'; import { Coordinate } from '../../../typings/timeseries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, @@ -68,74 +67,72 @@ export async function getServiceTransactionGroupDetailedStatistics({ impact: number; }> > { - return withApmSpan( - 'get_service_transaction_group_detailed_statistics', - async () => { - const { apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets }); + const { apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets }); - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); + const field = getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ); - const response = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], + const response = await apmEventClient.search( + 'get_service_transaction_group_detailed_statistics', + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], + aggs: { + total_duration: { sum: { field } }, + transaction_groups: { + terms: { + field: TRANSACTION_NAME, + include: transactionNames, + size: transactionNames.length, }, - }, - aggs: { - total_duration: { sum: { field } }, - transaction_groups: { - terms: { - field: TRANSACTION_NAME, - include: transactionNames, - size: transactionNames.length, + aggs: { + transaction_group_total_duration: { + sum: { field }, }, - aggs: { - transaction_group_total_duration: { - sum: { field }, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, + }, }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, + aggs: { + throughput_rate: { + rate: { + unit: 'minute', }, }, - aggs: { - throughput_rate: { - rate: { - unit: 'minute', - }, - }, - ...getLatencyAggregation(latencyAggregationType, field), - [EVENT_OUTCOME]: { - terms: { - field: EVENT_OUTCOME, - include: [EventOutcome.failure, EventOutcome.success], - }, + ...getLatencyAggregation(latencyAggregationType, field), + [EVENT_OUTCOME]: { + terms: { + field: EVENT_OUTCOME, + include: [EventOutcome.failure, EventOutcome.success], }, }, }, @@ -143,46 +140,42 @@ export async function getServiceTransactionGroupDetailedStatistics({ }, }, }, - }); - - const buckets = response.aggregations?.transaction_groups.buckets ?? []; - - const totalDuration = response.aggregations?.total_duration.value; - return buckets.map((bucket) => { - const transactionName = bucket.key as string; - const latency = bucket.timeseries.buckets.map((timeseriesBucket) => ({ - x: timeseriesBucket.key, - y: getLatencyValue({ - latencyAggregationType, - aggregation: timeseriesBucket.latency, - }), - })); - const throughput = bucket.timeseries.buckets.map( - (timeseriesBucket) => ({ - x: timeseriesBucket.key, - y: timeseriesBucket.throughput_rate.value, - }) - ); - const errorRate = bucket.timeseries.buckets.map((timeseriesBucket) => ({ - x: timeseriesBucket.key, - y: calculateTransactionErrorPercentage( - timeseriesBucket[EVENT_OUTCOME] - ), - })); - const transactionGroupTotalDuration = - bucket.transaction_group_total_duration.value || 0; - return { - transactionName, - latency, - throughput, - errorRate, - impact: totalDuration - ? (transactionGroupTotalDuration * 100) / totalDuration - : 0, - }; - }); + }, } ); + + const buckets = response.aggregations?.transaction_groups.buckets ?? []; + + const totalDuration = response.aggregations?.total_duration.value; + return buckets.map((bucket) => { + const transactionName = bucket.key as string; + const latency = bucket.timeseries.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: getLatencyValue({ + latencyAggregationType, + aggregation: timeseriesBucket.latency, + }), + })); + const throughput = bucket.timeseries.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: timeseriesBucket.throughput_rate.value, + })); + const errorRate = bucket.timeseries.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: calculateTransactionErrorPercentage(timeseriesBucket[EVENT_OUTCOME]), + })); + const transactionGroupTotalDuration = + bucket.transaction_group_total_duration.value || 0; + return { + transactionName, + latency, + throughput, + errorRate, + impact: totalDuration + ? (transactionGroupTotalDuration * 100) / totalDuration + : 0, + }; + }); } export async function getServiceTransactionGroupDetailedStatisticsPeriods({ diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts index 28574bab4df21..a4cc27c875d73 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts @@ -18,7 +18,6 @@ import { rangeQuery, kqlQuery, } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, @@ -56,14 +55,15 @@ export async function getServiceTransactionGroups({ transactionType: string; latencyAggregationType: LatencyAggregationType; }) { - return withApmSpan('get_service_transaction_groups', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); + const field = getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ); - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_transaction_groups', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -110,45 +110,45 @@ export async function getServiceTransactionGroups({ }, }, }, - }); + } + ); - const totalDuration = response.aggregations?.total_duration.value; + const totalDuration = response.aggregations?.total_duration.value; - const transactionGroups = - response.aggregations?.transaction_groups.buckets.map((bucket) => { - const errorRate = calculateTransactionErrorPercentage( - bucket[EVENT_OUTCOME] - ); + const transactionGroups = + response.aggregations?.transaction_groups.buckets.map((bucket) => { + const errorRate = calculateTransactionErrorPercentage( + bucket[EVENT_OUTCOME] + ); - const transactionGroupTotalDuration = - bucket.transaction_group_total_duration.value || 0; + const transactionGroupTotalDuration = + bucket.transaction_group_total_duration.value || 0; - return { - name: bucket.key as string, - latency: getLatencyValue({ - latencyAggregationType, - aggregation: bucket.latency, - }), - throughput: calculateThroughput({ - start, - end, - value: bucket.doc_count, - }), - errorRate, - impact: totalDuration - ? (transactionGroupTotalDuration * 100) / totalDuration - : 0, - }; - }) ?? []; + return { + name: bucket.key as string, + latency: getLatencyValue({ + latencyAggregationType, + aggregation: bucket.latency, + }), + throughput: calculateThroughput({ + start, + end, + value: bucket.doc_count, + }), + errorRate, + impact: totalDuration + ? (transactionGroupTotalDuration * 100) / totalDuration + : 0, + }; + }) ?? []; - return { - transactionGroups: transactionGroups.map((transactionGroup) => ({ - ...transactionGroup, - transactionType, - })), - isAggregationAccurate: - (response.aggregations?.transaction_groups.sum_other_doc_count ?? 0) === - 0, - }; - }); + return { + transactionGroups: transactionGroups.map((transactionGroup) => ({ + ...transactionGroup, + transactionType, + })), + isAggregationAccurate: + (response.aggregations?.transaction_groups.sum_other_doc_count ?? 0) === + 0, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts index e280ab6db1665..f38a7fba09d96 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts @@ -15,9 +15,8 @@ import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceTransactionTypes({ +export async function getServiceTransactionTypes({ setup, serviceName, searchAggregatedTransactions, @@ -26,41 +25,42 @@ export function getServiceTransactionTypes({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_service_transaction_types', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ], - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ], }, - aggs: { - types: { - terms: { field: TRANSACTION_TYPE, size: 100 }, - }, + }, + aggs: { + types: { + terms: { field: TRANSACTION_TYPE, size: 100 }, }, }, - }; + }, + }; - const { aggregations } = await apmEventClient.search(params); - const transactionTypes = - aggregations?.types.buckets.map((bucket) => bucket.key as string) || []; - return { transactionTypes }; - }); + const { aggregations } = await apmEventClient.search( + 'get_service_transaction_types', + params + ); + const transactionTypes = + aggregations?.types.buckets.map((bucket) => bucket.key as string) || []; + return { transactionTypes }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts index b42fd340bfb42..f33bedb6ef4fb 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts @@ -9,35 +9,31 @@ import { rangeQuery } from '../../../../server/utils/queries'; import { ProcessorEvent } from '../../../../common/processor_event'; import { OBSERVER_VERSION_MAJOR } from '../../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; // returns true if 6.x data is found export async function getLegacyDataStatus(setup: Setup & SetupTimeRange) { - return withApmSpan('get_legacy_data_status', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { range: { [OBSERVER_VERSION_MAJOR]: { lt: 7 } } }, - ...rangeQuery(start, end), - ], - }, + const params = { + terminateAfter: 1, + apm: { + events: [ProcessorEvent.transaction], + includeLegacyData: true, + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { range: { [OBSERVER_VERSION_MAJOR]: { lt: 7 } } }, + ...rangeQuery(start, end), + ], }, }, - }; + }, + }; - const resp = await apmEventClient.search(params, { - includeLegacyData: true, - }); - const hasLegacyData = resp.hits.total.value > 0; - return hasLegacyData; - }); + const resp = await apmEventClient.search('get_legacy_data_status', params); + const hasLegacyData = resp.hits.total.value > 0; + return hasLegacyData; } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts index 1e36df379e964..019ab8770887a 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts @@ -33,7 +33,6 @@ import { getOutcomeAggregation, } from '../../helpers/transaction_error_rate'; import { ServicesItemsSetup } from './get_services_items'; -import { withApmSpan } from '../../../utils/with_apm_span'; interface AggregationParams { environment?: string; @@ -50,23 +49,24 @@ export async function getServiceTransactionStats({ searchAggregatedTransactions, maxNumServices, }: AggregationParams) { - return withApmSpan('get_service_transaction_stats', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const outcomes = getOutcomeAggregation(); + const outcomes = getOutcomeAggregation(); - const metrics = { - avg_duration: { - avg: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + const metrics = { + avg_duration: { + avg: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, - outcomes, - }; + }, + outcomes, + }; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_transaction_stats', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -133,64 +133,64 @@ export async function getServiceTransactionStats({ }, }, }, - }); + } + ); - return ( - response.aggregations?.services.buckets.map((bucket) => { - const topTransactionTypeBucket = - bucket.transactionType.buckets.find( - ({ key }) => - key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD - ) ?? bucket.transactionType.buckets[0]; + return ( + response.aggregations?.services.buckets.map((bucket) => { + const topTransactionTypeBucket = + bucket.transactionType.buckets.find( + ({ key }) => + key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD + ) ?? bucket.transactionType.buckets[0]; - return { - serviceName: bucket.key as string, - transactionType: topTransactionTypeBucket.key as string, - environments: topTransactionTypeBucket.environments.buckets.map( - (environmentBucket) => environmentBucket.key as string + return { + serviceName: bucket.key as string, + transactionType: topTransactionTypeBucket.key as string, + environments: topTransactionTypeBucket.environments.buckets.map( + (environmentBucket) => environmentBucket.key as string + ), + agentName: topTransactionTypeBucket.sample.top[0].metrics[ + AGENT_NAME + ] as AgentName, + avgResponseTime: { + value: topTransactionTypeBucket.avg_duration.value, + timeseries: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.avg_duration.value, + }) ), - agentName: topTransactionTypeBucket.sample.top[0].metrics[ - AGENT_NAME - ] as AgentName, - avgResponseTime: { - value: topTransactionTypeBucket.avg_duration.value, - timeseries: topTransactionTypeBucket.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.avg_duration.value, - }) - ), - }, - transactionErrorRate: { - value: calculateTransactionErrorPercentage( - topTransactionTypeBucket.outcomes - ), - timeseries: topTransactionTypeBucket.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: calculateTransactionErrorPercentage(dateBucket.outcomes), - }) - ), - }, - transactionsPerMinute: { - value: calculateThroughput({ - start, - end, - value: topTransactionTypeBucket.doc_count, - }), - timeseries: topTransactionTypeBucket.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: calculateThroughput({ - start, - end, - value: dateBucket.doc_count, - }), - }) - ), - }, - }; - }) ?? [] - ); - }); + }, + transactionErrorRate: { + value: calculateTransactionErrorPercentage( + topTransactionTypeBucket.outcomes + ), + timeseries: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: calculateTransactionErrorPercentage(dateBucket.outcomes), + }) + ), + }, + transactionsPerMinute: { + value: calculateThroughput({ + start, + end, + value: topTransactionTypeBucket.doc_count, + }), + timeseries: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: calculateThroughput({ + start, + end, + value: dateBucket.doc_count, + }), + }) + ), + }, + }; + }) ?? [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts index 906cc62e64d1a..4692d1122b16c 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts @@ -14,9 +14,8 @@ import { import { environmentQuery, kqlQuery, rangeQuery } from '../../../utils/queries'; import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getServicesFromMetricDocuments({ +export async function getServicesFromMetricDocuments({ environment, setup, maxNumServices, @@ -27,10 +26,11 @@ export function getServicesFromMetricDocuments({ maxNumServices: number; kuery?: string; }) { - return withApmSpan('get_services_from_metric_documents', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_services_from_metric_documents', + { apm: { events: [ProcessorEvent.metric], }, @@ -67,18 +67,18 @@ export function getServicesFromMetricDocuments({ }, }, }, - }); + } + ); - return ( - response.aggregations?.services.buckets.map((bucket) => { - return { - serviceName: bucket.key as string, - environments: bucket.environments.buckets.map( - (envBucket) => envBucket.key as string - ), - agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, - }; - }) ?? [] - ); - }); + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + serviceName: bucket.key as string, + environments: bucket.environments.buckets.map( + (envBucket) => envBucket.key as string + ), + agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, + }; + }) ?? [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts b/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts index 28f6944fd24da..97b8a8fa5505b 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts @@ -6,29 +6,26 @@ */ import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; // Note: this logic is duplicated in tutorials/apm/envs/on_prem export async function hasHistoricalAgentData(setup: Setup) { - return withApmSpan('has_historical_agent_data', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ - ProcessorEvent.error, - ProcessorEvent.metric, - ProcessorEvent.transaction, - ], - }, - body: { - size: 0, - }, - }; + const params = { + terminateAfter: 1, + apm: { + events: [ + ProcessorEvent.error, + ProcessorEvent.metric, + ProcessorEvent.transaction, + ], + }, + body: { + size: 0, + }, + }; - const resp = await apmEventClient.search(params); - return resp.hits.total.value > 0; - }); + const resp = await apmEventClient.search('has_historical_agent_data', params); + return resp.hits.total.value > 0; } diff --git a/x-pack/plugins/apm/server/lib/services/get_throughput.ts b/x-pack/plugins/apm/server/lib/services/get_throughput.ts index 5f5008a28c232..b0cb917d302fc 100644 --- a/x-pack/plugins/apm/server/lib/services/get_throughput.ts +++ b/x-pack/plugins/apm/server/lib/services/get_throughput.ts @@ -21,7 +21,6 @@ import { } from '../helpers/aggregated_transactions'; import { getBucketSize } from '../helpers/get_bucket_size'; import { Setup } from '../helpers/setup_request'; -import { withApmSpan } from '../../utils/with_apm_span'; interface Options { environment?: string; @@ -88,20 +87,18 @@ function fetcher({ }, }; - return apmEventClient.search(params); + return apmEventClient.search('get_throughput_for_service', params); } -export function getThroughput(options: Options) { - return withApmSpan('get_throughput_for_service', async () => { - const response = await fetcher(options); +export async function getThroughput(options: Options) { + const response = await fetcher(options); - return ( - response.aggregations?.timeseries.buckets.map((bucket) => { - return { - x: bucket.key, - y: bucket.throughput.value, - }; - }) ?? [] - ); - }); + return ( + response.aggregations?.timeseries.buckets.map((bucket) => { + return { + x: bucket.key, + y: bucket.throughput.value, + }; + }) ?? [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts index 858f36e6e2c13..bb98abf724db4 100644 --- a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts @@ -41,7 +41,7 @@ const maybeAdd = (to: any[], value: any) => { to.push(value); }; -function getProfilingStats({ +async function getProfilingStats({ apmEventClient, filter, valueTypeField, @@ -50,49 +50,47 @@ function getProfilingStats({ filter: ESFilter[]; valueTypeField: string; }) { - return withApmSpan('get_profiling_stats', async () => { - const response = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.profile], - }, - body: { - size: 0, - query: { - bool: { - filter, - }, + const response = await apmEventClient.search('get_profiling_stats', { + apm: { + events: [ProcessorEvent.profile], + }, + body: { + size: 0, + query: { + bool: { + filter, }, - aggs: { - stacks: { - terms: { - field: PROFILE_TOP_ID, - size: MAX_STACK_IDS, - order: { - value: 'desc', - }, + }, + aggs: { + stacks: { + terms: { + field: PROFILE_TOP_ID, + size: MAX_STACK_IDS, + order: { + value: 'desc', }, - aggs: { - value: { - sum: { - field: valueTypeField, - }, + }, + aggs: { + value: { + sum: { + field: valueTypeField, }, }, }, }, }, - }); + }, + }); - const stacks = - response.aggregations?.stacks.buckets.map((stack) => { - return { - id: stack.key as string, - value: stack.value.value!, - }; - }) ?? []; + const stacks = + response.aggregations?.stacks.buckets.map((stack) => { + return { + id: stack.key as string, + value: stack.value.value!, + }; + }) ?? []; - return stacks; - }); + return stacks; } function getProfilesWithStacks({ @@ -103,8 +101,9 @@ function getProfilesWithStacks({ filter: ESFilter[]; }) { return withApmSpan('get_profiles_with_stacks', async () => { - const cardinalityResponse = await withApmSpan('get_top_cardinality', () => - apmEventClient.search({ + const cardinalityResponse = await apmEventClient.search( + 'get_top_cardinality', + { apm: { events: [ProcessorEvent.profile], }, @@ -121,7 +120,7 @@ function getProfilesWithStacks({ }, }, }, - }) + } ); const cardinality = cardinalityResponse.aggregations?.top.value ?? 0; @@ -140,39 +139,37 @@ function getProfilesWithStacks({ const allResponses = await withApmSpan('get_all_stacks', async () => { return Promise.all( [...new Array(partitions)].map(async (_, num) => { - const response = await withApmSpan('get_partition', () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.profile], - }, - body: { - query: { - bool: { - filter, - }, + const response = await apmEventClient.search('get_partition', { + apm: { + events: [ProcessorEvent.profile], + }, + body: { + query: { + bool: { + filter, }, - aggs: { - top: { - terms: { - field: PROFILE_TOP_ID, - size: Math.max(MAX_STACKS_PER_REQUEST), - include: { - num_partitions: partitions, - partition: num, - }, + }, + aggs: { + top: { + terms: { + field: PROFILE_TOP_ID, + size: Math.max(MAX_STACKS_PER_REQUEST), + include: { + num_partitions: partitions, + partition: num, }, - aggs: { - latest: { - top_hits: { - _source: [PROFILE_TOP_ID, PROFILE_STACK], - }, + }, + aggs: { + latest: { + top_hits: { + _source: [PROFILE_TOP_ID, PROFILE_STACK], }, }, }, }, }, - }) - ); + }, + }); return ( response.aggregations?.top.buckets.flatMap((bucket) => { diff --git a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts index 93fa029da8c72..af3cd6596a8c1 100644 --- a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts +++ b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts @@ -17,7 +17,6 @@ import { } from '../../../../common/profiling'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getBucketSize } from '../../helpers/get_bucket_size'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { kqlQuery } from '../../../utils/queries'; const configMap = mapValues( @@ -38,10 +37,11 @@ export async function getServiceProfilingTimeline({ setup: Setup & SetupTimeRange; environment?: string; }) { - return withApmSpan('get_service_profiling_timeline', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_profiling_timeline', + { apm: { events: [ProcessorEvent.profile], }, @@ -96,29 +96,29 @@ export async function getServiceProfilingTimeline({ }, }, }, - }); + } + ); - const { aggregations } = response; + const { aggregations } = response; - if (!aggregations) { - return []; - } + if (!aggregations) { + return []; + } - return aggregations.timeseries.buckets.map((bucket) => { - return { - x: bucket.key, - valueTypes: { - unknown: bucket.value_type.buckets.unknown.num_profiles.value, - // TODO: use enum as object key. not possible right now - // because of https://github.com/microsoft/TypeScript/issues/37888 - ...mapValues(configMap, (_, key) => { - return ( - bucket.value_type.buckets[key as ProfilingValueType]?.num_profiles - .value ?? 0 - ); - }), - }, - }; - }); + return aggregations.timeseries.buckets.map((bucket) => { + return { + x: bucket.key, + valueTypes: { + unknown: bucket.value_type.buckets.unknown.num_profiles.value, + // TODO: use enum as object key. not possible right now + // because of https://github.com/microsoft/TypeScript/issues/37888 + ...mapValues(configMap, (_, key) => { + return ( + bucket.value_type.buckets[key as ProfilingValueType]?.num_profiles + .value ?? 0 + ); + }), + }, + }; }); } diff --git a/x-pack/plugins/apm/server/lib/services/queries.test.ts b/x-pack/plugins/apm/server/lib/services/queries.test.ts index b167eff65ee0a..6adaca9c1a93d 100644 --- a/x-pack/plugins/apm/server/lib/services/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/services/queries.test.ts @@ -55,7 +55,7 @@ describe('services queries', () => { }) ); - const allParams = mock.spy.mock.calls.map((call) => call[0]); + const allParams = mock.spy.mock.calls.map((call) => call[1]); expect(allParams).toMatchSnapshot(); }); diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts index 1885382435562..c112c3be3362b 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts @@ -12,7 +12,6 @@ import { AgentConfigurationIntake, } from '../../../../common/agent_configuration/configuration_types'; import { APMIndexDocumentParams } from '../../helpers/create_es_client/create_internal_es_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; export function createOrUpdateConfiguration({ configurationId, @@ -23,30 +22,28 @@ export function createOrUpdateConfiguration({ configurationIntake: AgentConfigurationIntake; setup: Setup; }) { - return withApmSpan('create_or_update_configuration', async () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params: APMIndexDocumentParams = { - refresh: true, - index: indices.apmAgentConfigurationIndex, - body: { - agent_name: configurationIntake.agent_name, - service: { - name: configurationIntake.service.name, - environment: configurationIntake.service.environment, - }, - settings: configurationIntake.settings, - '@timestamp': Date.now(), - applied_by_agent: false, - etag: hash(configurationIntake), + const params: APMIndexDocumentParams = { + refresh: true, + index: indices.apmAgentConfigurationIndex, + body: { + agent_name: configurationIntake.agent_name, + service: { + name: configurationIntake.service.name, + environment: configurationIntake.service.environment, }, - }; + settings: configurationIntake.settings, + '@timestamp': Date.now(), + applied_by_agent: false, + etag: hash(configurationIntake), + }, + }; - // by specifying an id elasticsearch will delete the previous doc and insert the updated doc - if (configurationId) { - params.id = configurationId; - } + // by specifying an id elasticsearch will delete the previous doc and insert the updated doc + if (configurationId) { + params.id = configurationId; + } - return internalClient.index(params); - }); + return internalClient.index('create_or_update_agent_configuration', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts index 6ed6f79979889..125c97730a6fa 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; export async function deleteConfiguration({ @@ -15,15 +14,13 @@ export async function deleteConfiguration({ configurationId: string; setup: Setup; }) { - return withApmSpan('delete_agent_configuration', async () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params = { - refresh: 'wait_for' as const, - index: indices.apmAgentConfigurationIndex, - id: configurationId, - }; + const params = { + refresh: 'wait_for' as const, + index: indices.apmAgentConfigurationIndex, + id: configurationId, + }; - return internalClient.delete(params); - }); + return internalClient.delete('delete_agent_configuration', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts index 9fd4849c7640a..3543d38f7b5d1 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts @@ -11,47 +11,45 @@ import { SERVICE_ENVIRONMENT, SERVICE_NAME, } from '../../../../common/elasticsearch_fieldnames'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; import { convertConfigSettingsToString } from './convert_settings_to_string'; -export function findExactConfiguration({ +export async function findExactConfiguration({ service, setup, }: { service: AgentConfiguration['service']; setup: Setup; }) { - return withApmSpan('find_exact_agent_configuration', async () => { - const { internalClient, indices } = setup; - - const serviceNameFilter = service.name - ? { term: { [SERVICE_NAME]: service.name } } - : { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }; - - const environmentFilter = service.environment - ? { term: { [SERVICE_ENVIRONMENT]: service.environment } } - : { bool: { must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }] } }; - - const params = { - index: indices.apmAgentConfigurationIndex, - body: { - query: { - bool: { filter: [serviceNameFilter, environmentFilter] }, - }, + const { internalClient, indices } = setup; + + const serviceNameFilter = service.name + ? { term: { [SERVICE_NAME]: service.name } } + : { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }; + + const environmentFilter = service.environment + ? { term: { [SERVICE_ENVIRONMENT]: service.environment } } + : { bool: { must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }] } }; + + const params = { + index: indices.apmAgentConfigurationIndex, + body: { + query: { + bool: { filter: [serviceNameFilter, environmentFilter] }, }, - }; + }, + }; - const resp = await internalClient.search( - params - ); + const resp = await internalClient.search( + 'find_exact_agent_configuration', + params + ); - const hit = resp.hits.hits[0] as SearchHit | undefined; + const hit = resp.hits.hits[0] as SearchHit | undefined; - if (!hit) { - return; - } + if (!hit) { + return; + } - return convertConfigSettingsToString(hit); - }); + return convertConfigSettingsToString(hit); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts index 379ed12e37389..0b6dd10b42e25 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts @@ -9,7 +9,6 @@ import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup } from '../../helpers/setup_request'; import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; import { AGENT_NAME } from '../../../../common/elasticsearch_fieldnames'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function getAgentNameByService({ serviceName, @@ -18,35 +17,36 @@ export async function getAgentNameByService({ serviceName: string; setup: Setup; }) { - return withApmSpan('get_agent_name_by_service', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ - ProcessorEvent.transaction, - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [{ term: { [SERVICE_NAME]: serviceName } }], - }, + const params = { + terminateAfter: 1, + apm: { + events: [ + ProcessorEvent.transaction, + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [{ term: { [SERVICE_NAME]: serviceName } }], }, - aggs: { - agent_names: { - terms: { field: AGENT_NAME, size: 1 }, - }, + }, + aggs: { + agent_names: { + terms: { field: AGENT_NAME, size: 1 }, }, }, - }; + }, + }; - const { aggregations } = await apmEventClient.search(params); - const agentName = aggregations?.agent_names.buckets[0]?.key; - return agentName as string | undefined; - }); + const { aggregations } = await apmEventClient.search( + 'get_agent_name_by_service', + params + ); + const agentName = aggregations?.agent_names.buckets[0]?.key; + return agentName as string | undefined; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts index 4a32b3c3a370b..124a373d3cf07 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { withApmSpan } from '../../../../utils/with_apm_span'; import { Setup } from '../../../helpers/setup_request'; import { SERVICE_NAME, @@ -20,36 +19,37 @@ export async function getExistingEnvironmentsForService({ serviceName: string | undefined; setup: Setup; }) { - return withApmSpan('get_existing_environments_for_service', async () => { - const { internalClient, indices, config } = setup; - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; + const { internalClient, indices, config } = setup; + const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - const bool = serviceName - ? { filter: [{ term: { [SERVICE_NAME]: serviceName } }] } - : { must_not: [{ exists: { field: SERVICE_NAME } }] }; + const bool = serviceName + ? { filter: [{ term: { [SERVICE_NAME]: serviceName } }] } + : { must_not: [{ exists: { field: SERVICE_NAME } }] }; - const params = { - index: indices.apmAgentConfigurationIndex, - body: { - size: 0, - query: { bool }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - missing: ALL_OPTION_VALUE, - size: maxServiceEnvironments, - }, + const params = { + index: indices.apmAgentConfigurationIndex, + body: { + size: 0, + query: { bool }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + missing: ALL_OPTION_VALUE, + size: maxServiceEnvironments, }, }, }, - }; + }, + }; - const resp = await internalClient.search(params); - const existingEnvironments = - resp.aggregations?.environments.buckets.map( - (bucket) => bucket.key as string - ) || []; - return existingEnvironments; - }); + const resp = await internalClient.search( + 'get_existing_environments_for_service', + params + ); + const existingEnvironments = + resp.aggregations?.environments.buckets.map( + (bucket) => bucket.key as string + ) || []; + return existingEnvironments; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts index 9c56455f45902..0786bc6bc2771 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts @@ -11,52 +11,52 @@ import { PromiseReturnType } from '../../../../../observability/typings/common'; import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; import { ALL_OPTION_VALUE } from '../../../../common/agent_configuration/all_option'; import { getProcessorEventForAggregatedTransactions } from '../../helpers/aggregated_transactions'; -import { withApmSpan } from '../../../utils/with_apm_span'; export type AgentConfigurationServicesAPIResponse = PromiseReturnType< typeof getServiceNames >; -export function getServiceNames({ +export async function getServiceNames({ setup, searchAggregatedTransactions, }: { setup: Setup; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_service_names_for_agent_config', async () => { - const { apmEventClient, config } = setup; - const maxServiceSelection = config['xpack.apm.maxServiceSelection']; + const { apmEventClient, config } = setup; + const maxServiceSelection = config['xpack.apm.maxServiceSelection']; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - timeout: '1ms', - size: 0, - aggs: { - services: { - terms: { - field: SERVICE_NAME, - size: maxServiceSelection, - min_doc_count: 0, - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + timeout: '1ms', + size: 0, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: maxServiceSelection, + min_doc_count: 0, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - const serviceNames = - resp.aggregations?.services.buckets - .map((bucket) => bucket.key as string) - .sort() || []; - return [ALL_OPTION_VALUE, ...serviceNames]; - }); + const resp = await apmEventClient.search( + 'get_service_names_for_agent_config', + params + ); + const serviceNames = + resp.aggregations?.services.buckets + .map((bucket) => bucket.key as string) + .sort() || []; + return [ALL_OPTION_VALUE, ...serviceNames]; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts index adcfe88392dc8..098888c23ccbc 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts @@ -8,7 +8,6 @@ import { Setup } from '../../helpers/setup_request'; import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { convertConfigSettingsToString } from './convert_settings_to_string'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function listConfigurations({ setup }: { setup: Setup }) { const { internalClient, indices } = setup; @@ -18,8 +17,9 @@ export async function listConfigurations({ setup }: { setup: Setup }) { size: 200, }; - const resp = await withApmSpan('list_agent_configurations', () => - internalClient.search(params) + const resp = await internalClient.search( + 'list_agent_configuration', + params ); return resp.hits.hits diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts index 2026742a936a4..5fa4993921570 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts @@ -29,5 +29,8 @@ export async function markAppliedByAgent({ }, }; - return internalClient.index(params); + return internalClient.index( + 'mark_configuration_applied_by_agent', + params + ); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts index 7454128a741d5..4e27953b3a315 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts @@ -13,7 +13,6 @@ import { import { Setup } from '../../helpers/setup_request'; import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { convertConfigSettingsToString } from './convert_settings_to_string'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function searchConfigurations({ service, @@ -22,65 +21,64 @@ export async function searchConfigurations({ service: AgentConfiguration['service']; setup: Setup; }) { - return withApmSpan('search_agent_configurations', async () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - // In the following `constant_score` is being used to disable IDF calculation (where frequency of a term influences scoring). - // Additionally a boost has been added to service.name to ensure it scores higher. - // If there is tie between a config with a matching service.name and a config with a matching environment, the config that matches service.name wins - const serviceNameFilter = service.name - ? [ - { - constant_score: { - filter: { term: { [SERVICE_NAME]: service.name } }, - boost: 2, - }, + // In the following `constant_score` is being used to disable IDF calculation (where frequency of a term influences scoring). + // Additionally a boost has been added to service.name to ensure it scores higher. + // If there is tie between a config with a matching service.name and a config with a matching environment, the config that matches service.name wins + const serviceNameFilter = service.name + ? [ + { + constant_score: { + filter: { term: { [SERVICE_NAME]: service.name } }, + boost: 2, }, - ] - : []; + }, + ] + : []; - const environmentFilter = service.environment - ? [ - { - constant_score: { - filter: { term: { [SERVICE_ENVIRONMENT]: service.environment } }, - boost: 1, - }, + const environmentFilter = service.environment + ? [ + { + constant_score: { + filter: { term: { [SERVICE_ENVIRONMENT]: service.environment } }, + boost: 1, }, - ] - : []; + }, + ] + : []; - const params = { - index: indices.apmAgentConfigurationIndex, - body: { - query: { - bool: { - minimum_should_match: 2, - should: [ - ...serviceNameFilter, - ...environmentFilter, - { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }, - { - bool: { - must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }], - }, + const params = { + index: indices.apmAgentConfigurationIndex, + body: { + query: { + bool: { + minimum_should_match: 2, + should: [ + ...serviceNameFilter, + ...environmentFilter, + { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }, + { + bool: { + must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }], }, - ], - }, + }, + ], }, }, - }; + }, + }; - const resp = await internalClient.search( - params - ); + const resp = await internalClient.search( + 'search_agent_configurations', + params + ); - const hit = resp.hits.hits[0] as SearchHit | undefined; + const hit = resp.hits.hits[0] as SearchHit | undefined; - if (!hit) { - return; - } + if (!hit) { + return; + } - return convertConfigSettingsToString(hit); - }); + return convertConfigSettingsToString(hit); } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts index 47ee91232ea48..051b9e2809e49 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts @@ -39,17 +39,20 @@ describe('Create or Update Custom link', () => { it('creates a new custom link', () => { createOrUpdateCustomLink({ customLink, setup: mockedSetup }); - expect(internalClientIndexMock).toHaveBeenCalledWith({ - refresh: true, - index: 'apmCustomLinkIndex', - body: { - '@timestamp': 1570737000000, - label: 'foo', - url: 'http://elastic.com/{{trace.id}}', - 'service.name': ['opbeans-java'], - 'transaction.type': ['Request'], - }, - }); + expect(internalClientIndexMock).toHaveBeenCalledWith( + 'create_or_update_custom_link', + { + refresh: true, + index: 'apmCustomLinkIndex', + body: { + '@timestamp': 1570737000000, + label: 'foo', + url: 'http://elastic.com/{{trace.id}}', + 'service.name': ['opbeans-java'], + 'transaction.type': ['Request'], + }, + } + ); }); it('update a new custom link', () => { createOrUpdateCustomLink({ @@ -57,17 +60,20 @@ describe('Create or Update Custom link', () => { customLink, setup: mockedSetup, }); - expect(internalClientIndexMock).toHaveBeenCalledWith({ - refresh: true, - index: 'apmCustomLinkIndex', - id: 'bar', - body: { - '@timestamp': 1570737000000, - label: 'foo', - url: 'http://elastic.com/{{trace.id}}', - 'service.name': ['opbeans-java'], - 'transaction.type': ['Request'], - }, - }); + expect(internalClientIndexMock).toHaveBeenCalledWith( + 'create_or_update_custom_link', + { + refresh: true, + index: 'apmCustomLinkIndex', + id: 'bar', + body: { + '@timestamp': 1570737000000, + label: 'foo', + url: 'http://elastic.com/{{trace.id}}', + 'service.name': ['opbeans-java'], + 'transaction.type': ['Request'], + }, + } + ); }); }); diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts index 7e546fb555036..8f14e87fe183b 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts @@ -12,7 +12,6 @@ import { import { Setup } from '../../helpers/setup_request'; import { toESFormat } from './helper'; import { APMIndexDocumentParams } from '../../helpers/create_es_client/create_internal_es_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; export function createOrUpdateCustomLink({ customLinkId, @@ -23,23 +22,21 @@ export function createOrUpdateCustomLink({ customLink: Omit; setup: Setup; }) { - return withApmSpan('create_or_update_custom_link', () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params: APMIndexDocumentParams = { - refresh: true, - index: indices.apmCustomLinkIndex, - body: { - '@timestamp': Date.now(), - ...toESFormat(customLink), - }, - }; + const params: APMIndexDocumentParams = { + refresh: true, + index: indices.apmCustomLinkIndex, + body: { + '@timestamp': Date.now(), + ...toESFormat(customLink), + }, + }; - // by specifying an id elasticsearch will delete the previous doc and insert the updated doc - if (customLinkId) { - params.id = customLinkId; - } + // by specifying an id elasticsearch will delete the previous doc and insert the updated doc + if (customLinkId) { + params.id = customLinkId; + } - return internalClient.index(params); - }); + return internalClient.index('create_or_update_custom_link', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts index 7c88bcc43cc7f..bf7cfb33d87ac 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; export function deleteCustomLink({ @@ -15,15 +14,13 @@ export function deleteCustomLink({ customLinkId: string; setup: Setup; }) { - return withApmSpan('delete_custom_link', () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params = { - refresh: 'wait_for' as const, - index: indices.apmCustomLinkIndex, - id: customLinkId, - }; + const params = { + refresh: 'wait_for' as const, + index: indices.apmCustomLinkIndex, + id: customLinkId, + }; - return internalClient.delete(params); - }); + return internalClient.delete('delete_custom_link', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts index d3d9b45285354..91bc8c85bc014 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts @@ -11,43 +11,43 @@ import { Setup } from '../../helpers/setup_request'; import { ProcessorEvent } from '../../../../common/processor_event'; import { filterOptionsRt } from './custom_link_types'; import { splitFilterValueByComma } from './helper'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getTransaction({ +export async function getTransaction({ setup, filters = {}, }: { setup: Setup; filters?: t.TypeOf; }) { - return withApmSpan('get_transaction_for_custom_link', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const esFilters = compact( - Object.entries(filters) - // loops through the filters splitting the value by comma and removing white spaces - .map(([key, value]) => { - if (value) { - return { terms: { [key]: splitFilterValueByComma(value) } }; - } - }) - ); + const esFilters = compact( + Object.entries(filters) + // loops through the filters splitting the value by comma and removing white spaces + .map(([key, value]) => { + if (value) { + return { terms: { [key]: splitFilterValueByComma(value) } }; + } + }) + ); - const params = { - terminateAfter: 1, - apm: { - events: [ProcessorEvent.transaction as const], - }, - size: 1, - body: { - query: { - bool: { - filter: esFilters, - }, + const params = { + terminateAfter: 1, + apm: { + events: [ProcessorEvent.transaction as const], + }, + size: 1, + body: { + query: { + bool: { + filter: esFilters, }, }, - }; - const resp = await apmEventClient.search(params); - return resp.hits.hits[0]?._source; - }); + }, + }; + const resp = await apmEventClient.search( + 'get_transaction_for_custom_link', + params + ); + return resp.hits.hits[0]?._source; } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts index 0eac2e08d0901..d477da85e0d9b 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts @@ -14,54 +14,54 @@ import { import { Setup } from '../../helpers/setup_request'; import { fromESFormat } from './helper'; import { filterOptionsRt } from './custom_link_types'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function listCustomLinks({ +export async function listCustomLinks({ setup, filters = {}, }: { setup: Setup; filters?: t.TypeOf; }): Promise { - return withApmSpan('list_custom_links', async () => { - const { internalClient, indices } = setup; - const esFilters = Object.entries(filters).map(([key, value]) => { - return { + const { internalClient, indices } = setup; + const esFilters = Object.entries(filters).map(([key, value]) => { + return { + bool: { + minimum_should_match: 1, + should: [ + { term: { [key]: value } }, + { bool: { must_not: [{ exists: { field: key } }] } }, + ] as QueryDslQueryContainer[], + }, + }; + }); + + const params = { + index: indices.apmCustomLinkIndex, + size: 500, + body: { + query: { bool: { - minimum_should_match: 1, - should: [ - { term: { [key]: value } }, - { bool: { must_not: [{ exists: { field: key } }] } }, - ] as QueryDslQueryContainer[], + filter: esFilters, }, - }; - }); - - const params = { - index: indices.apmCustomLinkIndex, - size: 500, - body: { - query: { - bool: { - filter: esFilters, + }, + sort: [ + { + 'label.keyword': { + order: 'asc' as const, }, }, - sort: [ - { - 'label.keyword': { - order: 'asc' as const, - }, - }, - ], - }, - }; - const resp = await internalClient.search(params); - const customLinks = resp.hits.hits.map((item) => - fromESFormat({ - id: item._id, - ...item._source, - }) - ); - return customLinks; - }); + ], + }, + }; + const resp = await internalClient.search( + 'list_custom_links', + params + ); + const customLinks = resp.hits.hits.map((item) => + fromESFormat({ + id: item._id, + ...item._source, + }) + ); + return customLinks; } diff --git a/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts b/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts index 157f09978eaec..68d316ef55df9 100644 --- a/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts +++ b/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts @@ -19,7 +19,6 @@ import { APMError } from '../../../typings/es_schemas/ui/apm_error'; import { rangeQuery } from '../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { PromiseValueType } from '../../../typings/common'; -import { withApmSpan } from '../../utils/with_apm_span'; export interface ErrorsPerTransaction { [transactionId: string]: number; @@ -29,103 +28,94 @@ export async function getTraceItems( traceId: string, setup: Setup & SetupTimeRange ) { - return withApmSpan('get_trace_items', async () => { - const { start, end, apmEventClient, config } = setup; - const maxTraceItems = config['xpack.apm.ui.maxTraceItems']; - const excludedLogLevels = ['debug', 'info', 'warning']; + const { start, end, apmEventClient, config } = setup; + const maxTraceItems = config['xpack.apm.ui.maxTraceItems']; + const excludedLogLevels = ['debug', 'info', 'warning']; - const errorResponsePromise = withApmSpan('get_trace_error_items', () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.error], + const errorResponsePromise = apmEventClient.search('get_trace_items', { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: maxTraceItems, + query: { + bool: { + filter: [ + { term: { [TRACE_ID]: traceId } }, + ...rangeQuery(start, end), + ], + must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, }, - body: { - size: maxTraceItems, - query: { - bool: { - filter: [ - { term: { [TRACE_ID]: traceId } }, - ...rangeQuery(start, end), - ], - must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, - }, - }, - aggs: { - by_transaction_id: { - terms: { - field: TRANSACTION_ID, - size: maxTraceItems, - // high cardinality - execution_hint: 'map' as const, - }, - }, + }, + aggs: { + by_transaction_id: { + terms: { + field: TRANSACTION_ID, + size: maxTraceItems, + // high cardinality + execution_hint: 'map' as const, }, }, - }) - ); + }, + }, + }); - const traceResponsePromise = withApmSpan('get_trace_span_items', () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.span, ProcessorEvent.transaction], - }, - body: { - size: maxTraceItems, - query: { - bool: { - filter: [ - { term: { [TRACE_ID]: traceId } }, - ...rangeQuery(start, end), - ] as QueryDslQueryContainer[], - should: { - exists: { field: PARENT_ID }, - }, - }, + const traceResponsePromise = apmEventClient.search('get_trace_span_items', { + apm: { + events: [ProcessorEvent.span, ProcessorEvent.transaction], + }, + body: { + size: maxTraceItems, + query: { + bool: { + filter: [ + { term: { [TRACE_ID]: traceId } }, + ...rangeQuery(start, end), + ] as QueryDslQueryContainer[], + should: { + exists: { field: PARENT_ID }, }, - sort: [ - { _score: { order: 'asc' as const } }, - { [TRANSACTION_DURATION]: { order: 'desc' as const } }, - { [SPAN_DURATION]: { order: 'desc' as const } }, - ], - track_total_hits: true, }, - }) - ); + }, + sort: [ + { _score: { order: 'asc' as const } }, + { [TRANSACTION_DURATION]: { order: 'desc' as const } }, + { [SPAN_DURATION]: { order: 'desc' as const } }, + ], + track_total_hits: true, + }, + }); - const [errorResponse, traceResponse]: [ - // explicit intermediary types to avoid TS "excessively deep" error - PromiseValueType, - PromiseValueType - ] = (await Promise.all([ - errorResponsePromise, - traceResponsePromise, - ])) as any; + const [errorResponse, traceResponse]: [ + // explicit intermediary types to avoid TS "excessively deep" error + PromiseValueType, + PromiseValueType + ] = (await Promise.all([errorResponsePromise, traceResponsePromise])) as any; - const exceedsMax = traceResponse.hits.total.value > maxTraceItems; + const exceedsMax = traceResponse.hits.total.value > maxTraceItems; - const items = traceResponse.hits.hits.map((hit) => hit._source); + const items = traceResponse.hits.hits.map((hit) => hit._source); - const errorFrequencies: { - errorsPerTransaction: ErrorsPerTransaction; - errorDocs: APMError[]; - } = { - errorDocs: errorResponse.hits.hits.map(({ _source }) => _source), - errorsPerTransaction: - errorResponse.aggregations?.by_transaction_id.buckets.reduce( - (acc, current) => { - return { - ...acc, - [current.key]: current.doc_count, - }; - }, - {} as ErrorsPerTransaction - ) ?? {}, - }; + const errorFrequencies: { + errorsPerTransaction: ErrorsPerTransaction; + errorDocs: APMError[]; + } = { + errorDocs: errorResponse.hits.hits.map(({ _source }) => _source), + errorsPerTransaction: + errorResponse.aggregations?.by_transaction_id.buckets.reduce( + (acc, current) => { + return { + ...acc, + [current.key]: current.doc_count, + }; + }, + {} as ErrorsPerTransaction + ) ?? {}, + }; - return { - items, - exceedsMax, - ...errorFrequencies, - }; - }); + return { + items, + exceedsMax, + ...errorFrequencies, + }; } diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts index 71f803a03bf85..6499e80be9302 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts @@ -31,7 +31,6 @@ import { getOutcomeAggregation, getTransactionErrorRateTimeSeries, } from '../helpers/transaction_error_rate'; -import { withApmSpan } from '../../utils/with_apm_span'; export async function getErrorRate({ environment, @@ -58,81 +57,82 @@ export async function getErrorRate({ transactionErrorRate: Coordinate[]; average: number | null; }> { - return withApmSpan('get_transaction_group_error_rate', async () => { - const { apmEventClient } = setup; - - const transactionNamefilter = transactionName - ? [{ term: { [TRANSACTION_NAME]: transactionName } }] - : []; - const transactionTypefilter = transactionType - ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] - : []; - - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - { - terms: { - [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success], - }, - }, - ...transactionNamefilter, - ...transactionTypefilter, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; - - const outcomes = getOutcomeAggregation(); - - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], + const { apmEventClient } = setup; + + const transactionNamefilter = transactionName + ? [{ term: { [TRANSACTION_NAME]: transactionName } }] + : []; + const transactionTypefilter = transactionType + ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] + : []; + + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + { + terms: { + [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success], }, - body: { - size: 0, - query: { bool: { filter } }, - aggs: { - outcomes, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ start, end }).intervalString, - min_doc_count: 0, - extended_bounds: { min: start, max: end }, - }, - aggs: { - outcomes, - }, + }, + ...transactionNamefilter, + ...transactionTypefilter, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; + + const outcomes = getOutcomeAggregation(); + + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { bool: { filter } }, + aggs: { + outcomes, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ start, end }).intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: { + outcomes, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_transaction_group_error_rate', + params + ); - const noHits = resp.hits.total.value === 0; + const noHits = resp.hits.total.value === 0; - if (!resp.aggregations) { - return { noHits, transactionErrorRate: [], average: null }; - } + if (!resp.aggregations) { + return { noHits, transactionErrorRate: [], average: null }; + } - const transactionErrorRate = getTransactionErrorRateTimeSeries( - resp.aggregations.timeseries.buckets - ); + const transactionErrorRate = getTransactionErrorRateTimeSeries( + resp.aggregations.timeseries.buckets + ); - const average = calculateTransactionErrorPercentage( - resp.aggregations.outcomes - ); + const average = calculateTransactionErrorPercentage( + resp.aggregations.outcomes + ); - return { noHits, transactionErrorRate, average }; - }); + return { noHits, transactionErrorRate, average }; } export async function getErrorRatePeriods({ diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts index 8156d52d984df..34fd86f2fc598 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts @@ -11,7 +11,6 @@ import { TRANSACTION_TYPE } from '../../../common/elasticsearch_fieldnames'; import { arrayUnionToCallable } from '../../../common/utils/array_union_to_callable'; import { TransactionGroupRequestBase, TransactionGroupSetup } from './fetcher'; import { getTransactionDurationFieldForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; interface MetricParams { request: TransactionGroupRequestBase; @@ -39,124 +38,128 @@ function mergeRequestWithAggs< }); } -export function getAverages({ +export async function getAverages({ request, setup, searchAggregatedTransactions, }: MetricParams) { - return withApmSpan('get_avg_transaction_group_duration', async () => { - const params = mergeRequestWithAggs(request, { + const params = mergeRequestWithAggs(request, { + avg: { avg: { - avg: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - avg: bucket.avg.value, - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_avg_transaction_group_duration', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + avg: bucket.avg.value, + }; }); } -export function getCounts({ request, setup }: MetricParams) { - return withApmSpan('get_transaction_group_transaction_count', async () => { - const params = mergeRequestWithAggs(request, { - transaction_type: { - top_metrics: { - sort: { - '@timestamp': 'desc' as const, - }, - metrics: [ - { - field: TRANSACTION_TYPE, - } as const, - ], +export async function getCounts({ request, setup }: MetricParams) { + const params = mergeRequestWithAggs(request, { + transaction_type: { + top_metrics: { + sort: { + '@timestamp': 'desc' as const, }, + metrics: [ + { + field: TRANSACTION_TYPE, + } as const, + ], }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - count: bucket.doc_count, - transactionType: bucket.transaction_type.top[0].metrics[ - TRANSACTION_TYPE - ] as string, - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_transaction_group_transaction_count', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + count: bucket.doc_count, + transactionType: bucket.transaction_type.top[0].metrics[ + TRANSACTION_TYPE + ] as string, + }; }); } -export function getSums({ +export async function getSums({ request, setup, searchAggregatedTransactions, }: MetricParams) { - return withApmSpan('get_transaction_group_latency_sums', async () => { - const params = mergeRequestWithAggs(request, { + const params = mergeRequestWithAggs(request, { + sum: { sum: { - sum: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - sum: bucket.sum.value, - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_transaction_group_latency_sums', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + sum: bucket.sum.value, + }; }); } -export function getPercentiles({ +export async function getPercentiles({ request, setup, searchAggregatedTransactions, }: MetricParams) { - return withApmSpan('get_transaction_group_latency_percentiles', async () => { - const params = mergeRequestWithAggs(request, { - p95: { - percentiles: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - hdr: { number_of_significant_value_digits: 2 }, - percents: [95], - }, + const params = mergeRequestWithAggs(request, { + p95: { + percentiles: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), + hdr: { number_of_significant_value_digits: 2 }, + percents: [95], }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - p95: Object.values(bucket.p95.values)[0], - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_transaction_group_latency_percentiles', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + p95: Object.values(bucket.p95.values)[0], + }; }); } diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts b/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts index 6cb85b62205b8..5c1754cd36ef4 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts @@ -33,7 +33,7 @@ describe('transaction group queries', () => { ) ); - const allParams = mock.spy.mock.calls.map((call) => call[0]); + const allParams = mock.spy.mock.calls.map((call) => call[1]); expect(allParams).toMatchSnapshot(); }); @@ -51,7 +51,7 @@ describe('transaction group queries', () => { ) ); - const allParams = mock.spy.mock.calls.map((call) => call[0]); + const allParams = mock.spy.mock.calls.map((call) => call[1]); expect(allParams).toMatchSnapshot(); }); diff --git a/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts b/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts index 568769b52e2b4..20534a5fa7cbf 100644 --- a/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts @@ -26,9 +26,8 @@ import { import { getMetricsDateHistogramParams } from '../../helpers/metrics'; import { MAX_KPIS } from './constants'; import { getVizColorForIndex } from '../../../../common/viz_colors'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getTransactionBreakdown({ +export async function getTransactionBreakdown({ environment, kuery, setup, @@ -43,205 +42,203 @@ export function getTransactionBreakdown({ transactionName?: string; transactionType: string; }) { - return withApmSpan('get_transaction_breakdown', async () => { - const { apmEventClient, start, end, config } = setup; + const { apmEventClient, start, end, config } = setup; - const subAggs = { - sum_all_self_times: { - sum: { - field: SPAN_SELF_TIME_SUM, - }, + const subAggs = { + sum_all_self_times: { + sum: { + field: SPAN_SELF_TIME_SUM, }, - total_transaction_breakdown_count: { - sum: { - field: TRANSACTION_BREAKDOWN_COUNT, - }, + }, + total_transaction_breakdown_count: { + sum: { + field: TRANSACTION_BREAKDOWN_COUNT, }, - types: { - terms: { - field: SPAN_TYPE, - size: 20, - order: { - _count: 'desc' as const, - }, + }, + types: { + terms: { + field: SPAN_TYPE, + size: 20, + order: { + _count: 'desc' as const, }, - aggs: { - subtypes: { - terms: { - field: SPAN_SUBTYPE, - missing: '', - size: 20, - order: { - _count: 'desc' as const, - }, + }, + aggs: { + subtypes: { + terms: { + field: SPAN_SUBTYPE, + missing: '', + size: 20, + order: { + _count: 'desc' as const, }, - aggs: { - total_self_time_per_subtype: { - sum: { - field: SPAN_SELF_TIME_SUM, - }, + }, + aggs: { + total_self_time_per_subtype: { + sum: { + field: SPAN_SELF_TIME_SUM, }, }, }, }, }, - }; - - const filters = [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - { + }, + }; + + const filters = [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + { + bool: { + should: [ + { exists: { field: SPAN_SELF_TIME_SUM } }, + { exists: { field: TRANSACTION_BREAKDOWN_COUNT } }, + ], + minimum_should_match: 1, + }, + }, + ]; + + if (transactionName) { + filters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const params = { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { bool: { - should: [ - { exists: { field: SPAN_SELF_TIME_SUM } }, - { exists: { field: TRANSACTION_BREAKDOWN_COUNT } }, - ], - minimum_should_match: 1, + filter: filters, }, }, - ]; - - if (transactionName) { - filters.push({ term: { [TRANSACTION_NAME]: transactionName } }); - } - - const params = { - apm: { - events: [ProcessorEvent.metric], - }, - body: { - size: 0, - query: { - bool: { - filter: filters, - }, - }, - aggs: { - ...subAggs, - by_date: { - date_histogram: getMetricsDateHistogramParams( - start, - end, - config['xpack.apm.metricsInterval'] - ), - aggs: subAggs, - }, + aggs: { + ...subAggs, + by_date: { + date_histogram: getMetricsDateHistogramParams( + start, + end, + config['xpack.apm.metricsInterval'] + ), + aggs: subAggs, }, }, + }, + }; + + const resp = await apmEventClient.search('get_transaction_breakdown', params); + + const formatBucket = ( + aggs: + | Required['aggregations'] + | Required['aggregations']['by_date']['buckets'][0] + ) => { + const sumAllSelfTimes = aggs.sum_all_self_times.value || 0; + + const breakdowns = flatten( + aggs.types.buckets.map((bucket) => { + const type = bucket.key as string; + + return bucket.subtypes.buckets.map((subBucket) => { + return { + name: (subBucket.key as string) || type, + percentage: + (subBucket.total_self_time_per_subtype.value || 0) / + sumAllSelfTimes, + }; + }); + }) + ); + + return breakdowns; + }; + + const visibleKpis = resp.aggregations + ? orderBy(formatBucket(resp.aggregations), 'percentage', 'desc').slice( + 0, + MAX_KPIS + ) + : []; + + const kpis = orderBy( + visibleKpis.map((kpi) => ({ + ...kpi, + lowerCaseName: kpi.name.toLowerCase(), + })), + 'lowerCaseName' + ).map((kpi, index) => { + const { lowerCaseName, ...rest } = kpi; + return { + ...rest, + color: getVizColorForIndex(index), }; + }); - const resp = await apmEventClient.search(params); - - const formatBucket = ( - aggs: - | Required['aggregations'] - | Required['aggregations']['by_date']['buckets'][0] - ) => { - const sumAllSelfTimes = aggs.sum_all_self_times.value || 0; - - const breakdowns = flatten( - aggs.types.buckets.map((bucket) => { - const type = bucket.key as string; - - return bucket.subtypes.buckets.map((subBucket) => { - return { - name: (subBucket.key as string) || type, - percentage: - (subBucket.total_self_time_per_subtype.value || 0) / - sumAllSelfTimes, - }; - }); - }) - ); - - return breakdowns; - }; - - const visibleKpis = resp.aggregations - ? orderBy(formatBucket(resp.aggregations), 'percentage', 'desc').slice( - 0, - MAX_KPIS - ) - : []; - - const kpis = orderBy( - visibleKpis.map((kpi) => ({ - ...kpi, - lowerCaseName: kpi.name.toLowerCase(), - })), - 'lowerCaseName' - ).map((kpi, index) => { - const { lowerCaseName, ...rest } = kpi; - return { - ...rest, - color: getVizColorForIndex(index), - }; - }); - - const kpiNames = kpis.map((kpi) => kpi.name); + const kpiNames = kpis.map((kpi) => kpi.name); - const bucketsByDate = resp.aggregations?.by_date.buckets || []; + const bucketsByDate = resp.aggregations?.by_date.buckets || []; - const timeseriesPerSubtype = bucketsByDate.reduce((prev, bucket) => { - const formattedValues = formatBucket(bucket); - const time = bucket.key; + const timeseriesPerSubtype = bucketsByDate.reduce((prev, bucket) => { + const formattedValues = formatBucket(bucket); + const time = bucket.key; - const updatedSeries = kpiNames.reduce((p, kpiName) => { - const { name, percentage } = formattedValues.find( - (val) => val.name === kpiName - ) || { - name: kpiName, - percentage: null, - }; + const updatedSeries = kpiNames.reduce((p, kpiName) => { + const { name, percentage } = formattedValues.find( + (val) => val.name === kpiName + ) || { + name: kpiName, + percentage: null, + }; - if (!p[name]) { - p[name] = []; - } - return { - ...p, - [name]: p[name].concat({ - x: time, - y: percentage, - }), - }; - }, prev); - - const lastValues = Object.values(updatedSeries).map(last); - - // If for a given timestamp, some series have data, but others do not, - // we have to set any null values to 0 to make sure the stacked area chart - // is drawn correctly. - // If we set all values to 0, the chart always displays null values as 0, - // and the chart looks weird. - const hasAnyValues = lastValues.some((value) => value?.y !== null); - const hasNullValues = lastValues.some((value) => value?.y === null); - - if (hasAnyValues && hasNullValues) { - Object.values(updatedSeries).forEach((series) => { - const value = series[series.length - 1]; - const isEmpty = value.y === null; - if (isEmpty) { - // local mutation to prevent complicated map/reduce calls - value.y = 0; - } - }); + if (!p[name]) { + p[name] = []; } + return { + ...p, + [name]: p[name].concat({ + x: time, + y: percentage, + }), + }; + }, prev); + + const lastValues = Object.values(updatedSeries).map(last); + + // If for a given timestamp, some series have data, but others do not, + // we have to set any null values to 0 to make sure the stacked area chart + // is drawn correctly. + // If we set all values to 0, the chart always displays null values as 0, + // and the chart looks weird. + const hasAnyValues = lastValues.some((value) => value?.y !== null); + const hasNullValues = lastValues.some((value) => value?.y === null); + + if (hasAnyValues && hasNullValues) { + Object.values(updatedSeries).forEach((series) => { + const value = series[series.length - 1]; + const isEmpty = value.y === null; + if (isEmpty) { + // local mutation to prevent complicated map/reduce calls + value.y = 0; + } + }); + } - return updatedSeries; - }, {} as Record>); + return updatedSeries; + }, {} as Record>); - const timeseries = kpis.map((kpi) => ({ - title: kpi.name, - color: kpi.color, - type: 'areaStacked', - data: timeseriesPerSubtype[kpi.name], - hideLegend: false, - legendValue: asPercent(kpi.percentage, 1), - })); + const timeseries = kpis.map((kpi) => ({ + title: kpi.name, + color: kpi.color, + type: 'areaStacked', + data: timeseriesPerSubtype[kpi.name], + hideLegend: false, + legendValue: asPercent(kpi.percentage, 1), + })); - return { timeseries }; - }); + return { timeseries }; } diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts index 3b4319c37996d..6259bb75386fb 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts @@ -89,48 +89,47 @@ export async function getBuckets({ ] as QueryDslQueryContainer[]; async function getSamplesForDistributionBuckets() { - const response = await withApmSpan( + const response = await apmEventClient.search( 'get_samples_for_latency_distribution_buckets', - () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - query: { - bool: { - filter: [ - ...commonFilters, - { term: { [TRANSACTION_SAMPLED]: true } }, - ], - should: [ - { term: { [TRACE_ID]: traceId } }, - { term: { [TRANSACTION_ID]: transactionId } }, - ] as QueryDslQueryContainer[], - }, + { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + query: { + bool: { + filter: [ + ...commonFilters, + { term: { [TRANSACTION_SAMPLED]: true } }, + ], + should: [ + { term: { [TRACE_ID]: traceId } }, + { term: { [TRANSACTION_ID]: transactionId } }, + ] as QueryDslQueryContainer[], }, - aggs: { - distribution: { - histogram: getHistogramAggOptions({ - bucketSize, - field: TRANSACTION_DURATION, - distributionMax, - }), - aggs: { - samples: { - top_hits: { - _source: [TRANSACTION_ID, TRACE_ID], - size: 10, - sort: { - _score: 'desc' as const, - }, + }, + aggs: { + distribution: { + histogram: getHistogramAggOptions({ + bucketSize, + field: TRANSACTION_DURATION, + distributionMax, + }), + aggs: { + samples: { + top_hits: { + _source: [TRANSACTION_ID, TRACE_ID], + size: 10, + sort: { + _score: 'desc' as const, }, }, }, }, }, }, - }) + }, + } ); return ( @@ -148,41 +147,40 @@ export async function getBuckets({ } async function getDistributionBuckets() { - const response = await withApmSpan( + const response = await apmEventClient.search( 'get_latency_distribution_buckets', - () => - apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - query: { - bool: { - filter: [ - ...commonFilters, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + query: { + bool: { + filter: [ + ...commonFilters, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ], }, - aggs: { - distribution: { - histogram: getHistogramAggOptions({ - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - bucketSize, - distributionMax, - }), - }, + }, + aggs: { + distribution: { + histogram: getHistogramAggOptions({ + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), + bucketSize, + distributionMax, + }), }, }, - }) + }, + } ); return ( diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts index 2e86f6bb84c81..f3d4e8f6dd92d 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts @@ -20,7 +20,6 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function getDistributionMax({ environment, @@ -39,44 +38,45 @@ export async function getDistributionMax({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_latency_distribution_max', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - { term: { [TRANSACTION_NAME]: transactionName } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + { term: { [TRANSACTION_NAME]: transactionName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], }, - aggs: { - stats: { - max: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + }, + aggs: { + stats: { + max: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - return resp.aggregations?.stats.value ?? null; - }); + const resp = await apmEventClient.search( + 'get_latency_distribution_max', + params + ); + return resp.aggregations?.stats.value ?? null; } diff --git a/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts index 5a58360597828..2d350090fa28b 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts @@ -26,7 +26,6 @@ import { } from '../../../lib/helpers/aggregated_transactions'; import { getBucketSize } from '../../../lib/helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../../lib/helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getLatencyAggregation, getLatencyValue, @@ -112,10 +111,10 @@ function searchLatency({ }, }; - return apmEventClient.search(params); + return apmEventClient.search('get_latency_charts', params); } -export function getLatencyTimeseries({ +export async function getLatencyTimeseries({ environment, kuery, serviceName, @@ -138,40 +137,38 @@ export function getLatencyTimeseries({ start: number; end: number; }) { - return withApmSpan('get_latency_charts', async () => { - const response = await searchLatency({ - environment, - kuery, - serviceName, - transactionType, - transactionName, - setup, - searchAggregatedTransactions, - latencyAggregationType, - start, - end, - }); - - if (!response.aggregations) { - return { latencyTimeseries: [], overallAvgDuration: null }; - } - - return { - overallAvgDuration: - response.aggregations.overall_avg_duration.value || null, - latencyTimeseries: response.aggregations.latencyTimeseries.buckets.map( - (bucket) => { - return { - x: bucket.key, - y: getLatencyValue({ - latencyAggregationType, - aggregation: bucket.latency, - }), - }; - } - ), - }; + const response = await searchLatency({ + environment, + kuery, + serviceName, + transactionType, + transactionName, + setup, + searchAggregatedTransactions, + latencyAggregationType, + start, + end, }); + + if (!response.aggregations) { + return { latencyTimeseries: [], overallAvgDuration: null }; + } + + return { + overallAvgDuration: + response.aggregations.overall_avg_duration.value || null, + latencyTimeseries: response.aggregations.latencyTimeseries.buckets.map( + (bucket) => { + return { + x: bucket.key, + y: getLatencyValue({ + latencyAggregationType, + aggregation: bucket.latency, + }), + }; + } + ), + }; } export async function getLatencyPeriods({ diff --git a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts index a0225eb47e584..f4d9236395252 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts @@ -24,7 +24,6 @@ import { } from '../../../lib/helpers/aggregated_transactions'; import { getBucketSize } from '../../../lib/helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../../lib/helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getThroughputBuckets } from './transform'; export type ThroughputChartsResponse = PromiseReturnType< @@ -96,7 +95,7 @@ function searchThroughput({ }, }; - return apmEventClient.search(params); + return apmEventClient.search('get_transaction_throughput_series', params); } export async function getThroughputCharts({ @@ -116,26 +115,24 @@ export async function getThroughputCharts({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_transaction_throughput_series', async () => { - const { bucketSize, intervalString } = getBucketSize(setup); + const { bucketSize, intervalString } = getBucketSize(setup); - const response = await searchThroughput({ - environment, - kuery, - serviceName, - transactionType, - transactionName, - setup, - searchAggregatedTransactions, - intervalString, - }); - - return { - throughputTimeseries: getThroughputBuckets({ - throughputResultBuckets: response.aggregations?.throughput.buckets, - bucketSize, - setupTimeRange: setup, - }), - }; + const response = await searchThroughput({ + environment, + kuery, + serviceName, + transactionType, + transactionName, + setup, + searchAggregatedTransactions, + intervalString, }); + + return { + throughputTimeseries: getThroughputBuckets({ + throughputResultBuckets: response.aggregations?.throughput.buckets, + bucketSize, + setupTimeRange: setup, + }), + }; } diff --git a/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts index 6987ef0757734..c928b00cefb63 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts @@ -13,9 +13,8 @@ import { rangeQuery } from '../../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { ProcessorEvent } from '../../../../common/processor_event'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getTransaction({ +export async function getTransaction({ transactionId, traceId, setup, @@ -24,27 +23,25 @@ export function getTransaction({ traceId?: string; setup: Setup | (Setup & SetupTimeRange); }) { - return withApmSpan('get_transaction', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const resp = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - size: 1, - query: { - bool: { - filter: asMutableArray([ - { term: { [TRANSACTION_ID]: transactionId } }, - ...(traceId ? [{ term: { [TRACE_ID]: traceId } }] : []), - ...('start' in setup ? rangeQuery(setup.start, setup.end) : []), - ]), - }, + const resp = await apmEventClient.search('get_transaction', { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + size: 1, + query: { + bool: { + filter: asMutableArray([ + { term: { [TRANSACTION_ID]: transactionId } }, + ...(traceId ? [{ term: { [TRACE_ID]: traceId } }] : []), + ...('start' in setup ? rangeQuery(setup.start, setup.end) : []), + ]), }, }, - }); - - return resp.hits.hits[0]?._source; + }, }); + + return resp.hits.hits[0]?._source; } diff --git a/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts index dfdad2f59a848..568ce16e7aedc 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts @@ -11,40 +11,43 @@ import { } from '../../../../common/elasticsearch_fieldnames'; import { Setup } from '../../helpers/setup_request'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getRootTransactionByTraceId(traceId: string, setup: Setup) { - return withApmSpan('get_root_transaction_by_trace_id', async () => { - const { apmEventClient } = setup; +export async function getRootTransactionByTraceId( + traceId: string, + setup: Setup +) { + const { apmEventClient } = setup; - const params = { - apm: { - events: [ProcessorEvent.transaction as const], - }, - body: { - size: 1, - query: { - bool: { - should: [ - { - constant_score: { - filter: { - bool: { - must_not: { exists: { field: PARENT_ID } }, - }, + const params = { + apm: { + events: [ProcessorEvent.transaction as const], + }, + body: { + size: 1, + query: { + bool: { + should: [ + { + constant_score: { + filter: { + bool: { + must_not: { exists: { field: PARENT_ID } }, }, }, }, - ], - filter: [{ term: { [TRACE_ID]: traceId } }], - }, + }, + ], + filter: [{ term: { [TRACE_ID]: traceId } }], }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - return { - transaction: resp.hits.hits[0]?._source, - }; - }); + const resp = await apmEventClient.search( + 'get_root_transaction_by_trace_id', + params + ); + return { + transaction: resp.hits.hits[0]?._source, + }; } diff --git a/x-pack/plugins/apm/server/utils/test_helpers.tsx b/x-pack/plugins/apm/server/utils/test_helpers.tsx index 6252c33c5994d..9c63f0140bdf2 100644 --- a/x-pack/plugins/apm/server/utils/test_helpers.tsx +++ b/x-pack/plugins/apm/server/utils/test_helpers.tsx @@ -109,7 +109,7 @@ export async function inspectSearchParams( } return { - params: spy.mock.calls[0][0], + params: spy.mock.calls[0][1], response, error, spy, diff --git a/x-pack/plugins/apm/server/utils/with_apm_span.ts b/x-pack/plugins/apm/server/utils/with_apm_span.ts index 9762a7213d0a2..1343970f04a3f 100644 --- a/x-pack/plugins/apm/server/utils/with_apm_span.ts +++ b/x-pack/plugins/apm/server/utils/with_apm_span.ts @@ -13,7 +13,7 @@ export function withApmSpan( const options = parseSpanOptions(optionsOrName); const optionsWithDefaults = { - type: 'plugin:apm', + ...(options.intercept ? {} : { type: 'plugin:apm' }), ...options, labels: { plugin: 'apm', From 767b67d0522ec00865a7746d3ec37cd5de077821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Mon, 14 Jun 2021 18:49:58 +0200 Subject: [PATCH 13/91] [APM] Change View full trace button fill (#102066) --- .../WaterfallWithSummmary/MaybeViewTraceLink.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx index 4017495dd3b5d..11a0cc1234f42 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx @@ -45,7 +45,7 @@ export const MaybeViewTraceLink = ({ } )} > - + {viewFullTraceButtonLabel} @@ -67,7 +67,7 @@ export const MaybeViewTraceLink = ({ } )} > - + {viewFullTraceButtonLabel} @@ -92,7 +92,9 @@ export const MaybeViewTraceLink = ({ environment={nextEnvironment} latencyAggregationType={latencyAggregationType} > - {viewFullTraceButtonLabel} + + {viewFullTraceButtonLabel} + ); From 4fb0c943a6b5048c6033f5d8d41729aa2eb4f839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ece=20=C3=96zalp?= Date: Mon, 14 Jun 2021 13:31:20 -0400 Subject: [PATCH 14/91] [Security Solution] Refactor rules for modularization by updating bulkCreate and wrapHits methods (#101544) creates bulkCreate and wrapHits factories for the modularization of the detection engine Co-authored-by: Marshall Main --- .../rules_notification_alert_type.ts | 1 - .../schedule_notification_actions.ts | 3 +- .../__mocks__/build_rule_message.mock.ts | 15 + .../signals/bulk_create_factory.ts | 100 ++++++ .../signals/bulk_create_ml_signals.ts | 15 +- .../signals/executors/eql.test.ts | 39 +-- .../detection_engine/signals/executors/eql.ts | 21 +- .../signals/executors/ml.test.ts | 30 +- .../detection_engine/signals/executors/ml.ts | 21 +- .../signals/executors/query.ts | 12 +- .../signals/executors/threat_match.ts | 12 +- .../signals/executors/threshold.test.ts | 20 +- .../signals/executors/threshold.ts | 27 +- .../signals/filter_duplicate_signals.test.ts | 46 +++ .../signals/filter_duplicate_signals.ts | 14 + .../signals/search_after_bulk_create.test.ts | 67 ++-- .../signals/search_after_bulk_create.ts | 22 +- .../signals/signal_rule_alert_type.test.ts | 30 +- .../signals/signal_rule_alert_type.ts | 37 +- .../signals/single_bulk_create.test.ts | 318 ------------------ .../signals/single_bulk_create.ts | 227 ------------- .../threat_mapping/create_threat_signal.ts | 6 +- .../threat_mapping/create_threat_signals.ts | 7 +- .../signals/threat_mapping/types.ts | 8 +- .../signals/threat_mapping/utils.test.ts | 37 ++ .../signals/threat_mapping/utils.ts | 3 + .../bulk_create_threshold_signals.ts | 146 ++++---- .../lib/detection_engine/signals/types.ts | 14 +- .../detection_engine/signals/utils.test.ts | 13 +- .../lib/detection_engine/signals/utils.ts | 28 +- .../signals/wrap_hits_factory.ts | 35 ++ .../security_solution/server/lib/types.ts | 3 + .../security_solution/server/plugin.ts | 2 + 33 files changed, 543 insertions(+), 836 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts index 6f8dac5b49b31..a4863e577c6bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts @@ -91,7 +91,6 @@ export const rulesNotificationAlertType = ({ signalsCount, resultsLink, ruleParams, - // @ts-expect-error @elastic/elasticsearch _source is optional signals, }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts index e7db10380eea1..bfb96e97edf11 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts @@ -8,7 +8,6 @@ import { mapKeys, snakeCase } from 'lodash/fp'; import { AlertInstance } from '../../../../../alerting/server'; import { RuleParams } from '../schemas/rule_schemas'; -import { SignalSource } from '../signals/types'; export type NotificationRuleTypeParams = RuleParams & { name: string; @@ -20,7 +19,7 @@ interface ScheduleNotificationActions { signalsCount: number; resultsLink: string; ruleParams: NotificationRuleTypeParams; - signals: SignalSource[]; + signals: unknown[]; } export const scheduleNotificationActions = ({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts new file mode 100644 index 0000000000000..f43142d1d0264 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildRuleMessageFactory } from '../rule_messages'; + +export const mockBuildRuleMessage = buildRuleMessageFactory({ + id: 'fake id', + ruleId: 'fake rule id', + index: 'fakeindex', + name: 'fake name', +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts new file mode 100644 index 0000000000000..f518ac2386d0b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { performance } from 'perf_hooks'; +import { countBy, isEmpty, get } from 'lodash'; +import { ElasticsearchClient, Logger } from 'kibana/server'; +import { BuildRuleMessage } from './rule_messages'; +import { RefreshTypes } from '../types'; +import { BaseHit } from '../../../../common/detection_engine/types'; +import { errorAggregator, makeFloatString } from './utils'; + +export interface GenericBulkCreateResponse { + success: boolean; + bulkCreateDuration: string; + createdItemsCount: number; + createdItems: Array; + errors: string[]; +} + +export const bulkCreateFactory = ( + logger: Logger, + esClient: ElasticsearchClient, + buildRuleMessage: BuildRuleMessage, + refreshForBulkCreate: RefreshTypes +) => async (wrappedDocs: Array>): Promise> => { + if (wrappedDocs.length === 0) { + return { + errors: [], + success: true, + bulkCreateDuration: '0', + createdItemsCount: 0, + createdItems: [], + }; + } + + const bulkBody = wrappedDocs.flatMap((wrappedDoc) => [ + { + create: { + _index: wrappedDoc._index, + _id: wrappedDoc._id, + }, + }, + wrappedDoc._source, + ]); + const start = performance.now(); + + const { body: response } = await esClient.bulk({ + refresh: refreshForBulkCreate, + body: bulkBody, + }); + + const end = performance.now(); + logger.debug( + buildRuleMessage( + `individual bulk process time took: ${makeFloatString(end - start)} milliseconds` + ) + ); + logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`)); + const createdItems = wrappedDocs + .map((doc, index) => ({ + _id: response.items[index].create?._id ?? '', + _index: response.items[index].create?._index ?? '', + ...doc._source, + })) + .filter((_, index) => get(response.items[index], 'create.status') === 201); + const createdItemsCount = createdItems.length; + const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; + const errorCountByMessage = errorAggregator(response, [409]); + + logger.debug(buildRuleMessage(`bulk created ${createdItemsCount} signals`)); + if (duplicateSignalsCount > 0) { + logger.debug(buildRuleMessage(`ignored ${duplicateSignalsCount} duplicate signals`)); + } + if (!isEmpty(errorCountByMessage)) { + logger.error( + buildRuleMessage( + `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` + ) + ); + return { + errors: Object.keys(errorCountByMessage), + success: false, + bulkCreateDuration: makeFloatString(end - start), + createdItemsCount, + createdItems, + }; + } else { + return { + errors: [], + success: true, + bulkCreateDuration: makeFloatString(end - start), + createdItemsCount, + createdItems, + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 00ac40fa7e27c..ebb4462817eab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -14,11 +14,10 @@ import { AlertInstanceState, AlertServices, } from '../../../../../alerting/server'; -import { RefreshTypes } from '../types'; -import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; +import { GenericBulkCreateResponse } from './bulk_create_factory'; import { AnomalyResults, Anomaly } from '../../machine_learning'; import { BuildRuleMessage } from './rule_messages'; -import { AlertAttributes } from './types'; +import { AlertAttributes, BulkCreate, WrapHits } from './types'; import { MachineLearningRuleParams } from '../schemas/rule_schemas'; interface BulkCreateMlSignalsParams { @@ -28,8 +27,9 @@ interface BulkCreateMlSignalsParams { logger: Logger; id: string; signalsIndex: string; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } interface EcsAnomaly extends Anomaly { @@ -85,9 +85,10 @@ const transformAnomalyResultsToEcs = ( export const bulkCreateMlSignals = async ( params: BulkCreateMlSignalsParams -): Promise => { +): Promise> => { const anomalyResults = params.someResult; const ecsResults = transformAnomalyResultsToEcs(anomalyResults); - const buildRuleMessage = params.buildRuleMessage; - return singleBulkCreate({ ...params, filteredEvents: ecsResults, buildRuleMessage }); + + const wrappedDocs = params.wrapHits(ecsResults.hits.hits); + return params.bulkCreate(wrappedDocs); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index f8f77bd2bf6e6..947e7d573173e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -7,7 +7,6 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; -import { RuleStatusService } from '../rule_status_service'; import { eqlExecutor } from './eql'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; @@ -23,7 +22,6 @@ describe('eql_executor', () => { const version = '8.0.0'; let logger: ReturnType; let alertServices: AlertServicesMock; - let ruleStatusService: Record; (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION); const eqlSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', @@ -51,17 +49,11 @@ describe('eql_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - ruleStatusService = { - success: jest.fn(), - find: jest.fn(), - goingToRun: jest.fn(), - error: jest.fn(), - partialFailure: jest.fn(), - }; alertServices.scopedClusterClient.asCurrentUser.transport.request.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ hits: { total: { value: 10 }, + events: [], }, }) ); @@ -70,25 +62,16 @@ describe('eql_executor', () => { describe('eqlExecutor', () => { it('should set a warning when exception list for eql rule contains value list exceptions', async () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; - try { - await eqlExecutor({ - rule: eqlSO, - exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, - services: alertServices, - version, - logger, - refresh: false, - searchAfterSize, - }); - } catch (err) { - // eqlExecutor will throw until we have an EQL response mock that conforms to the - // expected EQL response format, so just catch the error and check the status service - } - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' - ); + const response = await eqlExecutor({ + rule: eqlSO, + exceptionItems, + services: alertServices, + version, + logger, + searchAfterSize, + bulkCreate: jest.fn(), + }); + expect(response.warningMessages.length).toEqual(1); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index 8e2f5e92ae502..28d1f3e19baee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -7,9 +7,9 @@ import { ApiResponse } from '@elastic/elasticsearch'; import { performance } from 'perf_hooks'; -import { Logger } from 'src/core/server'; import { SavedObject } from 'src/core/types'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { Logger } from 'src/core/server'; import { AlertInstanceContext, AlertInstanceState, @@ -21,13 +21,12 @@ import { isOutdated } from '../../migrations/helpers'; import { getIndexVersion } from '../../routes/index/get_index_version'; import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template'; import { EqlRuleParams } from '../../schemas/rule_schemas'; -import { RefreshTypes } from '../../types'; import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body'; import { getInputIndex } from '../get_input_output_index'; -import { RuleStatusService } from '../rule_status_service'; -import { bulkInsertSignals, filterDuplicateSignals } from '../single_bulk_create'; +import { filterDuplicateSignals } from '../filter_duplicate_signals'; import { AlertAttributes, + BulkCreate, EqlSignalSearchResponse, SearchAfterAndBulkCreateReturnType, WrappedSignalHit, @@ -37,26 +36,24 @@ import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../uti export const eqlExecutor = async ({ rule, exceptionItems, - ruleStatusService, services, version, - searchAfterSize, logger, - refresh, + searchAfterSize, + bulkCreate, }: { rule: SavedObject>; exceptionItems: ExceptionListItemSchema[]; - ruleStatusService: RuleStatusService; services: AlertServices; version: string; - searchAfterSize: number; logger: Logger; - refresh: RefreshTypes; + searchAfterSize: number; + bulkCreate: BulkCreate; }): Promise => { const result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; if (hasLargeValueItem(exceptionItems)) { - await ruleStatusService.partialFailure( + result.warningMessages.push( 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' ); result.warning = true; @@ -125,7 +122,7 @@ export const eqlExecutor = async ({ } if (newSignals.length > 0) { - const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); + const insertResult = await bulkCreate(newSignals); result.bulkCreateTimes.push(insertResult.bulkCreateDuration); result.createdSignalsCount += insertResult.createdItemsCount; result.createdSignals = insertResult.createdItems; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts index e157750a7d51b..25a9d2c3f510f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -7,7 +7,6 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; -import { RuleStatusService } from '../rule_status_service'; import { mlExecutor } from './ml'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getMlRuleParams } from '../../schemas/rule_schemas.mock'; @@ -17,7 +16,6 @@ import { findMlSignals } from '../find_ml_signals'; import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; import { mlPluginServerMock } from '../../../../../../ml/server/mocks'; import { sampleRuleSO } from '../__mocks__/es_results'; -import { getRuleStatusServiceMock } from '../rule_status_service.mock'; jest.mock('../find_ml_signals'); jest.mock('../bulk_create_ml_signals'); @@ -25,7 +23,6 @@ jest.mock('../bulk_create_ml_signals'); describe('ml_executor', () => { let jobsSummaryMock: jest.Mock; let mlMock: ReturnType; - let ruleStatusService: ReturnType; const exceptionItems = [getExceptionListItemSchemaMock()]; let logger: ReturnType; let alertServices: AlertServicesMock; @@ -45,7 +42,6 @@ describe('ml_executor', () => { mlMock.jobServiceProvider.mockReturnValue({ jobsSummary: jobsSummaryMock, }); - ruleStatusService = getRuleStatusServiceMock(); (findMlSignals as jest.Mock).mockResolvedValue({ _shards: {}, hits: { @@ -66,35 +62,32 @@ describe('ml_executor', () => { rule: mlSO, ml: undefined, exceptionItems, - ruleStatusService, services: alertServices, logger, - refresh: false, buildRuleMessage, listClient: getListClientMock(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }) ).rejects.toThrow('ML plugin unavailable during rule execution'); }); it('should record a partial failure if Machine learning job summary was null', async () => { jobsSummaryMock.mockResolvedValue([]); - await mlExecutor({ + const response = await mlExecutor({ rule: mlSO, ml: mlMock, exceptionItems, - ruleStatusService, services: alertServices, logger, - refresh: false, buildRuleMessage, listClient: getListClientMock(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }); expect(logger.warn).toHaveBeenCalled(); expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job(s) are not started'); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Machine learning job(s) are not started' - ); + expect(response.warningMessages.length).toEqual(1); }); it('should record a partial failure if Machine learning job was not started', async () => { @@ -106,22 +99,19 @@ describe('ml_executor', () => { }, ]); - await mlExecutor({ + const response = await mlExecutor({ rule: mlSO, ml: mlMock, exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, services: alertServices, logger, - refresh: false, buildRuleMessage, listClient: getListClientMock(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }); expect(logger.warn).toHaveBeenCalled(); expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job(s) are not started'); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Machine learning job(s) are not started' - ); + expect(response.warningMessages.length).toEqual(1); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts index 28703046289f5..f5c7d8822b51f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -17,13 +17,11 @@ import { ListClient } from '../../../../../../lists/server'; import { isJobStarted } from '../../../../../common/machine_learning/helpers'; import { SetupPlugins } from '../../../../plugin'; import { MachineLearningRuleParams } from '../../schemas/rule_schemas'; -import { RefreshTypes } from '../../types'; import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; import { filterEventsAgainstList } from '../filters/filter_events_against_list'; import { findMlSignals } from '../find_ml_signals'; import { BuildRuleMessage } from '../rule_messages'; -import { RuleStatusService } from '../rule_status_service'; -import { AlertAttributes } from '../types'; +import { AlertAttributes, BulkCreate, WrapHits } from '../types'; import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils'; export const mlExecutor = async ({ @@ -31,21 +29,21 @@ export const mlExecutor = async ({ ml, listClient, exceptionItems, - ruleStatusService, services, logger, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }: { rule: SavedObject>; ml: SetupPlugins['ml']; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; - ruleStatusService: RuleStatusService; services: AlertServices; logger: Logger; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }) => { const result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; @@ -67,7 +65,7 @@ export const mlExecutor = async ({ jobSummaries.length < 1 || jobSummaries.some((job) => !isJobStarted(job.jobState, job.datafeedState)) ) { - const errorMessage = buildRuleMessage( + const warningMessage = buildRuleMessage( 'Machine learning job(s) are not started:', ...jobSummaries.map((job) => [ @@ -77,9 +75,9 @@ export const mlExecutor = async ({ ].join(', ') ) ); - logger.warn(errorMessage); + result.warningMessages.push(warningMessage); + logger.warn(warningMessage); result.warning = true; - await ruleStatusService.partialFailure(errorMessage); } const anomalyResults = await findMlSignals({ @@ -120,8 +118,9 @@ export const mlExecutor = async ({ logger, id: rule.id, signalsIndex: ruleParams.outputIndex, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }); // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } const shardFailures = diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 05e2e3056e99e..9d76a06afa275 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -14,11 +14,10 @@ import { AlertServices, } from '../../../../../../alerting/server'; import { ListClient } from '../../../../../../lists/server'; -import { RefreshTypes } from '../../types'; import { getFilter } from '../get_filter'; import { getInputIndex } from '../get_input_output_index'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; -import { AlertAttributes, RuleRangeTuple } from '../types'; +import { AlertAttributes, RuleRangeTuple, BulkCreate, WrapHits } from '../types'; import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { QueryRuleParams, SavedQueryRuleParams } from '../../schemas/rule_schemas'; @@ -32,9 +31,10 @@ export const queryExecutor = async ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }: { rule: SavedObject>; tuples: RuleRangeTuple[]; @@ -44,9 +44,10 @@ export const queryExecutor = async ({ version: string; searchAfterSize: number; logger: Logger; - refresh: RefreshTypes; eventsTelemetry: TelemetryEventsSender | undefined; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }) => { const ruleParams = rule.attributes.params; const inputIndex = await getInputIndex(services, version, ruleParams.index); @@ -74,7 +75,8 @@ export const queryExecutor = async ({ signalsIndex: ruleParams.outputIndex, filter: esFilter, pageSize: searchAfterSize, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts index 10b4ce939ca3a..078eb8362069c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -14,9 +14,8 @@ import { AlertServices, } from '../../../../../../alerting/server'; import { ListClient } from '../../../../../../lists/server'; -import { RefreshTypes } from '../../types'; import { getInputIndex } from '../get_input_output_index'; -import { RuleRangeTuple, AlertAttributes } from '../types'; +import { RuleRangeTuple, AlertAttributes, BulkCreate, WrapHits } from '../types'; import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { createThreatSignals } from '../threat_mapping/create_threat_signals'; @@ -31,9 +30,10 @@ export const threatMatchExecutor = async ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }: { rule: SavedObject>; tuples: RuleRangeTuple[]; @@ -43,9 +43,10 @@ export const threatMatchExecutor = async ({ version: string; searchAfterSize: number; logger: Logger; - refresh: RefreshTypes; eventsTelemetry: TelemetryEventsSender | undefined; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }) => { const ruleParams = rule.attributes.params; const inputIndex = await getInputIndex(services, version, ruleParams.index); @@ -67,7 +68,6 @@ export const threatMatchExecutor = async ({ outputIndex: ruleParams.outputIndex, ruleSO: rule, searchAfterSize, - refresh, threatFilters: ruleParams.threatFilters ?? [], threatQuery: ruleParams.threatQuery, threatLanguage: ruleParams.threatLanguage, @@ -76,5 +76,7 @@ export const threatMatchExecutor = async ({ threatIndicatorPath: ruleParams.threatIndicatorPath, concurrentSearches: ruleParams.concurrentSearches ?? 1, itemsPerSearch: ruleParams.itemsPerSearch ?? 9000, + bulkCreate, + wrapHits, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index 5d62b28b73ae8..f03e8b8a147ae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -7,7 +7,6 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; -import { RuleStatusService } from '../rule_status_service'; import { thresholdExecutor } from './threshold'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; @@ -18,7 +17,6 @@ describe('threshold_executor', () => { const version = '8.0.0'; let logger: ReturnType; let alertServices: AlertServicesMock; - let ruleStatusService: Record; const thresholdSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', type: 'alert', @@ -50,34 +48,24 @@ describe('threshold_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - ruleStatusService = { - success: jest.fn(), - find: jest.fn(), - goingToRun: jest.fn(), - error: jest.fn(), - partialFailure: jest.fn(), - }; }); describe('thresholdExecutor', () => { it('should set a warning when exception list for threshold rule contains value list exceptions', async () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; - await thresholdExecutor({ + const response = await thresholdExecutor({ rule: thresholdSO, tuples: [], exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, services: alertServices, version, logger, - refresh: false, buildRuleMessage, startedAt: new Date(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' - ); + expect(response.warningMessages.length).toEqual(1); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index fa0986044e250..5e23128c9c148 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -15,51 +15,55 @@ import { } from '../../../../../../alerting/server'; import { hasLargeValueItem } from '../../../../../common/detection_engine/utils'; import { ThresholdRuleParams } from '../../schemas/rule_schemas'; -import { RefreshTypes } from '../../types'; import { getFilter } from '../get_filter'; import { getInputIndex } from '../get_input_output_index'; -import { BuildRuleMessage } from '../rule_messages'; -import { RuleStatusService } from '../rule_status_service'; import { bulkCreateThresholdSignals, findThresholdSignals, getThresholdBucketFilters, getThresholdSignalHistory, } from '../threshold'; -import { AlertAttributes, RuleRangeTuple, SearchAfterAndBulkCreateReturnType } from '../types'; +import { + AlertAttributes, + BulkCreate, + RuleRangeTuple, + SearchAfterAndBulkCreateReturnType, + WrapHits, +} from '../types'; import { createSearchAfterReturnType, createSearchAfterReturnTypeFromResponse, mergeReturns, } from '../utils'; +import { BuildRuleMessage } from '../rule_messages'; export const thresholdExecutor = async ({ rule, tuples, exceptionItems, - ruleStatusService, services, version, logger, - refresh, buildRuleMessage, startedAt, + bulkCreate, + wrapHits, }: { rule: SavedObject>; tuples: RuleRangeTuple[]; exceptionItems: ExceptionListItemSchema[]; - ruleStatusService: RuleStatusService; services: AlertServices; version: string; logger: Logger; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; startedAt: Date; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }): Promise => { let result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; if (hasLargeValueItem(exceptionItems)) { - await ruleStatusService.partialFailure( + result.warningMessages.push( 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' ); result.warning = true; @@ -126,14 +130,13 @@ export const thresholdExecutor = async ({ filter: esFilter, services, logger, - id: rule.id, inputIndexPattern: inputIndex, signalsIndex: ruleParams.outputIndex, startedAt, from: tuple.from.toDate(), - refresh, thresholdSignalHistory, - buildRuleMessage, + bulkCreate, + wrapHits, }); result = mergeReturns([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts new file mode 100644 index 0000000000000..5c4af83c3b03e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filterDuplicateSignals } from './filter_duplicate_signals'; +import { sampleWrappedSignalHit } from './__mocks__/es_results'; + +const mockRuleId1 = 'aaaaaaaa'; +const mockRuleId2 = 'bbbbbbbb'; +const mockRuleId3 = 'cccccccc'; + +const createWrappedSignalHitWithRuleId = (ruleId: string) => { + const mockSignal = sampleWrappedSignalHit(); + return { + ...mockSignal, + _source: { + ...mockSignal._source, + signal: { + ...mockSignal._source.signal, + ancestors: [ + { + ...mockSignal._source.signal.ancestors[0], + rule: ruleId, + }, + ], + }, + }, + }; +}; +const mockSignals = [ + createWrappedSignalHitWithRuleId(mockRuleId1), + createWrappedSignalHitWithRuleId(mockRuleId2), +]; + +describe('filterDuplicateSignals', () => { + it('filters duplicate signals', () => { + expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1); + }); + + it('does not filter non-duplicate signals', () => { + expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts new file mode 100644 index 0000000000000..a648c05306289 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { WrappedSignalHit } from './types'; + +export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => { + return signals.filter( + (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 52c887c3ca55a..e4eb7e854f670 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -16,29 +16,28 @@ import { sampleDocWithSortId, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; -import { buildRuleMessageFactory } from './rule_messages'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; import uuid from 'uuid'; import { listMock } from '../../../../../lists/server/mocks'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import { BulkResponse, RuleRangeTuple } from './types'; +import { BulkCreate, BulkResponse, RuleRangeTuple, WrapHits } from './types'; import type { SearchListItemArraySchema } from '@kbn/securitysolution-io-ts-list-types'; import { getSearchListItemResponseMock } from '../../../../../lists/common/schemas/response/search_list_item_schema.mock'; import { getRuleRangeTuples } from './utils'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; import { getQueryRuleParams } from '../schemas/rule_schemas.mock'; +import { bulkCreateFactory } from './bulk_create_factory'; +import { wrapHitsFactory } from './wrap_hits_factory'; +import { mockBuildRuleMessage } from './__mocks__/build_rule_message.mock'; -const buildRuleMessage = buildRuleMessageFactory({ - id: 'fake id', - ruleId: 'fake rule id', - index: 'fakeindex', - name: 'fake name', -}); +const buildRuleMessage = mockBuildRuleMessage; describe('searchAfterAndBulkCreate', () => { let mockService: AlertServicesMock; + let bulkCreate: BulkCreate; + let wrapHits: WrapHits; let inputIndexPattern: string[] = []; let listClient = listMock.getListClient(); const someGuids = Array.from({ length: 13 }).map(() => uuid.v4()); @@ -61,6 +60,13 @@ describe('searchAfterAndBulkCreate', () => { maxSignals: sampleParams.maxSignals, buildRuleMessage, })); + bulkCreate = bulkCreateFactory( + mockLogger, + mockService.scopedClusterClient.asCurrentUser, + buildRuleMessage, + false + ); + wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX }); }); test('should return success with number of searches less than max signals', async () => { @@ -166,6 +172,7 @@ describe('searchAfterAndBulkCreate', () => { }, }, ]; + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ tuples, ruleSO, @@ -179,8 +186,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(5); @@ -282,8 +290,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(4); @@ -359,8 +368,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -417,8 +427,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -495,8 +506,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -549,8 +561,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1); @@ -625,18 +638,14 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(createdSignalsCount).toEqual(4); expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - // I don't like testing log statements since logs change but this is the best - // way I can think of to ensure this section is getting hit with this test case. - expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[14][0]).toContain( - 'ran out of sort ids to sort on name: "fake name" id: "fake id" rule id: "fake rule id" signals index: "fakeindex"' - ); }); test('should return success when no exceptions list provided', async () => { @@ -703,8 +712,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -746,8 +756,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(mockLogger.error).toHaveBeenCalled(); expect(success).toEqual(false); @@ -792,8 +803,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(createdSignalsCount).toEqual(0); @@ -852,8 +864,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(false); expect(createdSignalsCount).toEqual(0); // should not create signals if search threw error @@ -977,8 +990,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(false); expect(errors).toEqual(['error on creation']); @@ -1072,8 +1086,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(mockEnrichment).toHaveBeenCalledWith( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index b0dcc1810a639..bb2e57b0606e5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -8,7 +8,6 @@ import { identity } from 'lodash'; import type { estypes } from '@elastic/elasticsearch'; import { singleSearchAfter } from './single_search_after'; -import { singleBulkCreate } from './single_bulk_create'; import { filterEventsAgainstList } from './filters/filter_events_against_list'; import { sendAlertTelemetryEvents } from './send_telemetry_events'; import { @@ -31,14 +30,13 @@ export const searchAfterAndBulkCreate = async ({ listClient, logger, eventsTelemetry, - id, inputIndexPattern, - signalsIndex, filter, pageSize, - refresh, buildRuleMessage, enrichment = identity, + bulkCreate, + wrapHits, }: SearchAfterAndBulkCreateParams): Promise => { const ruleParams = ruleSO.attributes.params; let toReturn = createSearchAfterReturnType(); @@ -149,6 +147,7 @@ export const searchAfterAndBulkCreate = async ({ ); } const enrichedEvents = await enrichment(filteredEvents); + const wrappedDocs = wrapHits(enrichedEvents.hits.hits); const { bulkCreateDuration: bulkDuration, @@ -156,16 +155,7 @@ export const searchAfterAndBulkCreate = async ({ createdItems, success: bulkSuccess, errors: bulkErrors, - } = await singleBulkCreate({ - buildRuleMessage, - filteredEvents: enrichedEvents, - ruleSO, - services, - logger, - id, - signalsIndex, - refresh, - }); + } = await bulkCreate(wrappedDocs); toReturn = mergeReturns([ toReturn, createSearchAfterReturnType({ @@ -180,10 +170,10 @@ export const searchAfterAndBulkCreate = async ({ logger.debug(buildRuleMessage(`created ${createdCount} signals`)); logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); logger.debug( - buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`) + buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) ); - sendAlertTelemetryEvents(logger, eventsTelemetry, filteredEvents, buildRuleMessage); + sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); } if (!hasSortId) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 823d694f36514..d8c919b50e9db 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -274,35 +274,6 @@ describe('signal_rule_alert_type', () => { expect(ruleStatusService.error).toHaveBeenCalledTimes(0); }); - it("should set refresh to 'wait_for' when actions are present", async () => { - const ruleAlert = getAlertMock(getQueryRuleParams()); - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - await alert.executor(payload); - expect((queryExecutor as jest.Mock).mock.calls[0][0].refresh).toEqual('wait_for'); - }); - - it('should set refresh to false when actions are not present', async () => { - await alert.executor(payload); - expect((queryExecutor as jest.Mock).mock.calls[0][0].refresh).toEqual(false); - }); - it('should call scheduleActions if signalsCount was greater than 0 and rule has actions defined', async () => { const ruleAlert = getAlertMock(getQueryRuleParams()); ruleAlert.actions = [ @@ -462,6 +433,7 @@ describe('signal_rule_alert_type', () => { lastLookBackDate: null, createdSignalsCount: 0, createdSignals: [], + warningMessages: [], errors: ['Error that bubbled up.'], }; (queryExecutor as jest.Mock).mockResolvedValue(result); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 13a63df6ed8b6..0a2e22bc44b60 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -65,6 +65,8 @@ import { RuleParams, savedQueryRuleParams, } from '../schemas/rule_schemas'; +import { bulkCreateFactory } from './bulk_create_factory'; +import { wrapHitsFactory } from './wrap_hits_factory'; export const signalRulesAlertType = ({ logger, @@ -218,6 +220,19 @@ export const signalRulesAlertType = ({ client: exceptionsClient, lists: params.exceptionsList ?? [], }); + + const bulkCreate = bulkCreateFactory( + logger, + services.scopedClusterClient.asCurrentUser, + buildRuleMessage, + refresh + ); + + const wrapHits = wrapHitsFactory({ + ruleSO: savedObject, + signalsIndex: params.outputIndex, + }); + if (isMlRule(type)) { const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams); result = await mlExecutor({ @@ -225,11 +240,11 @@ export const signalRulesAlertType = ({ ml, listClient, exceptionItems, - ruleStatusService, services, logger, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }); } else if (isThresholdRule(type)) { const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams); @@ -237,13 +252,13 @@ export const signalRulesAlertType = ({ rule: thresholdRuleSO, tuples, exceptionItems, - ruleStatusService, services, version, logger, - refresh, buildRuleMessage, startedAt, + bulkCreate, + wrapHits, }); } else if (isThreatMatchRule(type)) { const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams); @@ -256,9 +271,10 @@ export const signalRulesAlertType = ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }); } else if (isQueryRule(type)) { const queryRuleSO = validateQueryRuleTypes(savedObject); @@ -271,25 +287,30 @@ export const signalRulesAlertType = ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }); } else if (isEqlRule(type)) { const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams); result = await eqlExecutor({ rule: eqlRuleSO, exceptionItems, - ruleStatusService, services, version, searchAfterSize, + bulkCreate, logger, - refresh, }); } else { throw new Error(`unknown rule type ${type}`); } + if (result.warningMessages.length) { + const warningMessage = buildRuleMessage(result.warningMessages.join()); + await ruleStatusService.partialFailure(warningMessage); + } + if (result.success) { if (actions.length) { const notificationRuleParams: NotificationRuleTypeParams = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts deleted file mode 100644 index 3fbb8c1a607e9..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { generateId } from './utils'; -import { - sampleDocSearchResultsNoSortId, - mockLogger, - sampleRuleGuid, - sampleDocSearchResultsNoSortIdNoVersion, - sampleEmptyDocSearchResults, - sampleBulkCreateDuplicateResult, - sampleBulkCreateErrorResult, - sampleDocWithAncestors, - sampleRuleSO, -} from './__mocks__/es_results'; -import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; -import { singleBulkCreate, filterDuplicateRules } from './single_bulk_create'; -import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; -import { buildRuleMessageFactory } from './rule_messages'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; -import { getQueryRuleParams } from '../schemas/rule_schemas.mock'; - -const buildRuleMessage = buildRuleMessageFactory({ - id: 'fake id', - ruleId: 'fake rule id', - index: 'fakeindex', - name: 'fake name', -}); -describe('singleBulkCreate', () => { - const mockService: AlertServicesMock = alertsMock.createAlertServices(); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('create signal id gereateId', () => { - test('two docs with same index, id, and version should have same id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const generatedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid, version, ruleId); - expect(firstHash).toEqual(generatedHash); - expect(secondHash).toEqual(generatedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - }); - test('two docs with different index, id, and version should have different id', () => { - const findex = 'myfakeindex'; - const findex2 = 'mysecondfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // 'mysecondfakeindexsomefakeid1rule-1' - const secondGeneratedHash = - 'a852941273f805ffe9006e574601acc8ae1148d6c0b3f7f8c4785cba8f6b768a'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex2, fid, version, ruleId); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - test('two docs with same index, different id, and same version should have different id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const fid2 = 'somefakeid2'; - const version = '1'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // 'myfakeindexsomefakeid21rule-1' - const secondGeneratedHash = - '7d33faea18159fd010c4b79890620e8b12cdc88ec1d370149d0e5552ce860255'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid2, version, ruleId); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - test('two docs with same index, same id, and different version should have different id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const version2 = '2'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // myfakeindexsomefakeid2rule-1' - const secondGeneratedHash = - 'f016f3071fa9df9221d2fb2ba92389d4d388a4347c6ec7a4012c01cb1c640a40'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid, version2, ruleId); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - test('Ensure generated id is less than 512 bytes, even for really really long strings', () => { - const longIndexName = - 'myfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - const firstHash = generateId(longIndexName, fid, version, ruleId); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - }); - test('two docs with same index, same id, same version number, and different rule ids should have different id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - const ruleId2 = 'rule-2'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // myfakeindexsomefakeid1rule-2' - const secondGeneratedHash = - '1eb04f997086f8b3b143d4d9b18ac178c4a7423f71a5dad9ba8b9e92603c6863'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid, version, ruleId2); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - }); - - test('create successful bulk create', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( - // @ts-expect-error not compatible response interface - elasticsearchClientMock.createSuccessTransportRequestPromise({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(0); - }); - - test('create successful bulk create with docs with no versioning', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( - // @ts-expect-error not compatible response interface - elasticsearchClientMock.createSuccessTransportRequestPromise({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortIdNoVersion(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(0); - }); - - test('create unsuccessful bulk create due to empty search results', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValue( - // @ts-expect-error not full response interface - elasticsearchClientMock.createSuccessTransportRequestPromise(false) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleEmptyDocSearchResults(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(0); - }); - - test('create successful bulk create when bulk create has duplicate errors', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleBulkCreateDuplicateResult) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - - expect(mockLogger.error).not.toHaveBeenCalled(); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(1); - }); - - test('create failed bulk create when bulk create has multiple error statuses', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleBulkCreateErrorResult) - ); - const { success, createdItemsCount, errors } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(mockLogger.error).toHaveBeenCalled(); - expect(errors).toEqual(['[4]: internal server error']); - expect(success).toEqual(false); - expect(createdItemsCount).toEqual(1); - }); - - test('filter duplicate rules will return an empty array given an empty array', () => { - const filtered = filterDuplicateRules( - '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - sampleEmptyDocSearchResults() - ); - expect(filtered).toEqual([]); - }); - - test('filter duplicate rules will return nothing filtered when the two rule ids do not match with each other', () => { - const filtered = filterDuplicateRules('some id', sampleDocWithAncestors()); - expect(filtered).toEqual(sampleDocWithAncestors().hits.hits); - }); - - test('filters duplicate rules will return empty array when the two rule ids match each other', () => { - const filtered = filterDuplicateRules( - '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - sampleDocWithAncestors() - ); - expect(filtered).toEqual([]); - }); - - test('filter duplicate rules will return back search responses if they do not have a signal and will NOT filter the source out', () => { - const ancestors = sampleDocSearchResultsNoSortId(); - const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', ancestors); - expect(filtered).toEqual(ancestors.hits.hits); - }); - - test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { - const doc = { ...sampleDocWithAncestors(), _source: { signal: 1234 } }; - const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); - expect(filtered).toEqual([]); - }); - - test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own object signal type', () => { - const doc = { ...sampleDocWithAncestors(), _source: { signal: {} } }; - const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); - expect(filtered).toEqual([]); - }); - - test('create successful and returns proper createdItemsCount', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleBulkCreateDuplicateResult) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(1); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts deleted file mode 100644 index 92d01fef6e50c..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { countBy, isEmpty, get } from 'lodash'; -import { performance } from 'perf_hooks'; -import { - AlertInstanceContext, - AlertInstanceState, - AlertServices, -} from '../../../../../alerting/server'; -import { AlertAttributes, SignalHit, SignalSearchResponse, WrappedSignalHit } from './types'; -import { RefreshTypes } from '../types'; -import { generateId, makeFloatString, errorAggregator } from './utils'; -import { buildBulkBody } from './build_bulk_body'; -import { BuildRuleMessage } from './rule_messages'; -import { Logger, SavedObject } from '../../../../../../../src/core/server'; -import { isEventTypeSignal } from './build_event_type_signal'; - -interface SingleBulkCreateParams { - filteredEvents: SignalSearchResponse; - ruleSO: SavedObject; - services: AlertServices; - logger: Logger; - id: string; - signalsIndex: string; - refresh: RefreshTypes; - buildRuleMessage: BuildRuleMessage; -} - -/** - * This is for signals on signals to work correctly. If given a rule id this will check if - * that rule id already exists in the ancestor tree of each signal search response and remove - * those documents so they cannot be created as a signal since we do not want a rule id to - * ever be capable of re-writing the same signal continuously if both the _input_ and _output_ - * of the signals index happens to be the same index. - * @param ruleId The rule id - * @param signalSearchResponse The search response that has all the documents - */ -export const filterDuplicateRules = ( - ruleId: string, - signalSearchResponse: SignalSearchResponse -) => { - return signalSearchResponse.hits.hits.filter((doc) => { - if (doc._source?.signal == null || !isEventTypeSignal(doc)) { - return true; - } else { - return !( - doc._source?.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) || - doc._source?.signal.rule.id === ruleId - ); - } - }); -}; - -/** - * Similar to filterDuplicateRules, but operates on candidate signal documents rather than events that matched - * the detection query. This means we only have to compare the ruleId against the ancestors array. - * @param ruleId The rule id - * @param signals The candidate new signals - */ -export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => { - return signals.filter( - (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) - ); -}; - -export interface SingleBulkCreateResponse { - success: boolean; - bulkCreateDuration?: string; - createdItemsCount: number; - createdItems: SignalHit[]; - errors: string[]; -} - -export interface BulkInsertSignalsResponse { - bulkCreateDuration: string; - createdItemsCount: number; - createdItems: SignalHit[]; -} - -// Bulk Index documents. -export const singleBulkCreate = async ({ - buildRuleMessage, - filteredEvents, - ruleSO, - services, - logger, - id, - signalsIndex, - refresh, -}: SingleBulkCreateParams): Promise => { - const ruleParams = ruleSO.attributes.params; - filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents); - logger.debug(buildRuleMessage(`about to bulk create ${filteredEvents.hits.hits.length} events`)); - if (filteredEvents.hits.hits.length === 0) { - logger.debug(buildRuleMessage(`all events were duplicates`)); - return { success: true, createdItemsCount: 0, createdItems: [], errors: [] }; - } - // index documents after creating an ID based on the - // source documents' originating index, and the original - // document _id. This will allow two documents from two - // different indexes with the same ID to be - // indexed, and prevents us from creating any updates - // to the documents once inserted into the signals index, - // while preventing duplicates from being added to the - // signals index if rules are re-run over the same time - // span. Also allow for versioning. - const bulkBody = filteredEvents.hits.hits.flatMap((doc) => [ - { - create: { - _index: signalsIndex, - _id: generateId( - doc._index, - doc._id, - doc._version ? doc._version.toString() : '', - ruleParams.ruleId ?? '' - ), - }, - }, - buildBulkBody(ruleSO, doc), - ]); - const start = performance.now(); - const { body: response } = await services.scopedClusterClient.asCurrentUser.bulk({ - index: signalsIndex, - refresh, - body: bulkBody, - }); - const end = performance.now(); - logger.debug( - buildRuleMessage( - `individual bulk process time took: ${makeFloatString(end - start)} milliseconds` - ) - ); - logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`)); - const createdItems = filteredEvents.hits.hits - .map((doc, index) => ({ - _id: response.items[index].create?._id ?? '', - _index: response.items[index].create?._index ?? '', - ...buildBulkBody(ruleSO, doc), - })) - .filter((_, index) => get(response.items[index], 'create.status') === 201); - const createdItemsCount = createdItems.length; - const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; - const errorCountByMessage = errorAggregator(response, [409]); - - logger.debug(buildRuleMessage(`bulk created ${createdItemsCount} signals`)); - if (duplicateSignalsCount > 0) { - logger.debug(buildRuleMessage(`ignored ${duplicateSignalsCount} duplicate signals`)); - } - - if (!isEmpty(errorCountByMessage)) { - logger.error( - buildRuleMessage( - `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` - ) - ); - return { - errors: Object.keys(errorCountByMessage), - success: false, - bulkCreateDuration: makeFloatString(end - start), - createdItemsCount, - createdItems, - }; - } else { - return { - errors: [], - success: true, - bulkCreateDuration: makeFloatString(end - start), - createdItemsCount, - createdItems, - }; - } -}; - -// Bulk Index new signals. -export const bulkInsertSignals = async ( - signals: WrappedSignalHit[], - logger: Logger, - services: AlertServices, - refresh: RefreshTypes -): Promise => { - // index documents after creating an ID based on the - // id and index of each parent and the rule ID - const bulkBody = signals.flatMap((doc) => [ - { - create: { - _index: doc._index, - _id: doc._id, - }, - }, - doc._source, - ]); - const start = performance.now(); - const { body: response } = await services.scopedClusterClient.asCurrentUser.bulk({ - refresh, - body: bulkBody, - }); - const end = performance.now(); - logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); - logger.debug(`took property says bulk took: ${response.took} milliseconds`); - - if (response.errors) { - const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; - logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`); - const errorCountByMessage = errorAggregator(response, [409]); - if (!isEmpty(errorCountByMessage)) { - logger.error( - `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` - ); - } - } - - const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; - const createdItems = signals - .map((doc, index) => ({ - ...doc._source, - _id: response.items[index].create?._id ?? '', - _index: response.items[index].create?._index ?? '', - })) - .filter((_, index) => get(response.items[index], 'create.status') === 201); - logger.debug(`bulk created ${createdItemsCount} signals`); - return { bulkCreateDuration: makeFloatString(end - start), createdItems, createdItemsCount }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 37b0b88d88eda..3e30a08f1ae69 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -31,10 +31,11 @@ export const createThreatSignal = async ({ outputIndex, ruleSO, searchAfterSize, - refresh, buildRuleMessage, currentThreatList, currentResult, + bulkCreate, + wrapHits, }: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, @@ -81,9 +82,10 @@ export const createThreatSignal = async ({ signalsIndex: outputIndex, filter: esFilter, pageSize: searchAfterSize, - refresh, buildRuleMessage, enrichment: threatEnrichment, + bulkCreate, + wrapHits, }); logger.debug( buildRuleMessage( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index b3e0e376c7794..5054ab1b2cca5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -32,7 +32,6 @@ export const createThreatSignals = async ({ outputIndex, ruleSO, searchAfterSize, - refresh, threatFilters, threatQuery, threatLanguage, @@ -41,6 +40,8 @@ export const createThreatSignals = async ({ threatIndicatorPath, concurrentSearches, itemsPerSearch, + bulkCreate, + wrapHits, }: CreateThreatSignalsOptions): Promise => { const params = ruleSO.attributes.params; logger.debug(buildRuleMessage('Indicator matching rule starting')); @@ -55,6 +56,7 @@ export const createThreatSignals = async ({ createdSignalsCount: 0, createdSignals: [], errors: [], + warningMessages: [], }; let threatListCount = await getThreatListCount({ @@ -120,10 +122,11 @@ export const createThreatSignals = async ({ outputIndex, ruleSO, searchAfterSize, - refresh, buildRuleMessage, currentThreatList: slicedChunk, currentResult: results, + bulkCreate, + wrapHits, }) ); const searchesPerformed = await Promise.all(concurrentSearchesPerformed); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index acb64f826f3f2..34b064b0f8805 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -29,9 +29,11 @@ import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { AlertAttributes, + BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, SignalsEnrichment, + WrapHits, } from '../types'; import { ThreatRuleParams } from '../../schemas/rule_schemas'; @@ -55,7 +57,6 @@ export interface CreateThreatSignalsOptions { outputIndex: string; ruleSO: SavedObject>; searchAfterSize: number; - refresh: false | 'wait_for'; threatFilters: unknown[]; threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; @@ -64,6 +65,8 @@ export interface CreateThreatSignalsOptions { threatLanguage: ThreatLanguageOrUndefined; concurrentSearches: ConcurrentSearches; itemsPerSearch: ItemsPerSearch; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } export interface CreateThreatSignalOptions { @@ -85,10 +88,11 @@ export interface CreateThreatSignalOptions { outputIndex: string; ruleSO: SavedObject>; searchAfterSize: number; - refresh: false | 'wait_for'; buildRuleMessage: BuildRuleMessage; currentThreatList: ThreatListItem[]; currentResult: SearchAfterAndBulkCreateReturnType; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } export interface BuildThreatMappingFilterOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 6c6447bad0975..ec826b44023f6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -58,6 +58,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -69,6 +70,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults.success).toEqual(true); @@ -84,6 +86,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -95,6 +98,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults.success).toEqual(false); @@ -110,6 +114,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -121,6 +126,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); @@ -136,6 +142,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -147,6 +154,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults).toEqual( @@ -167,6 +175,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -178,6 +187,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 4', 'error 1', 'error 3', 'error 5'], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults).toEqual( @@ -289,6 +299,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, @@ -299,6 +310,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, []); expect(combinedResults).toEqual(expectedResult); @@ -314,6 +326,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { success: true, @@ -324,6 +337,7 @@ describe('utils', () => { createdSignalsCount: 0, createdSignals: [], errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, @@ -334,6 +348,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -350,6 +365,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, @@ -360,6 +376,7 @@ describe('utils', () => { createdSignalsCount: 5, createdSignals: Array(5).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, @@ -370,6 +387,7 @@ describe('utils', () => { createdSignalsCount: 8, createdSignals: Array(8).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -381,6 +399,7 @@ describe('utils', () => { createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) createdSignals: Array(16).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); @@ -397,6 +416,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, @@ -407,6 +427,7 @@ describe('utils', () => { createdSignalsCount: 5, createdSignals: Array(5).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, @@ -417,6 +438,7 @@ describe('utils', () => { createdSignalsCount: 8, createdSignals: Array(8).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -428,6 +450,7 @@ describe('utils', () => { createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) createdSignals: Array(16).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped @@ -444,6 +467,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, @@ -454,6 +478,7 @@ describe('utils', () => { createdSignalsCount: 5, createdSignals: Array(5).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, @@ -464,6 +489,7 @@ describe('utils', () => { createdSignalsCount: 8, createdSignals: Array(8).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -475,6 +501,7 @@ describe('utils', () => { createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) createdSignals: Array(16).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); @@ -491,6 +518,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -502,6 +530,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults.success).toEqual(true); @@ -517,6 +546,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -528,6 +558,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults.success).toEqual(false); @@ -543,6 +574,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -554,6 +586,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); @@ -569,6 +602,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -580,6 +614,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults).toEqual( @@ -600,6 +635,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -611,6 +647,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 4', 'error 1', 'error 3', 'error 5'], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults).toEqual( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 47a32915dd83f..4d9fda43f032e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -75,6 +75,7 @@ export const combineResults = ( lastLookBackDate: newResult.lastLookBackDate, createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, createdSignals: [...currentResult.createdSignals, ...newResult.createdSignals], + warningMessages: [...currentResult.warningMessages, ...newResult.warningMessages], errors: [...new Set([...currentResult.errors, ...newResult.errors])], }); @@ -100,6 +101,7 @@ export const combineConcurrentResults = ( lastLookBackDate, createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, createdSignals: [...accum.createdSignals, ...item.createdSignals], + warningMessages: [...accum.warningMessages, ...item.warningMessages], errors: [...new Set([...accum.errors, ...item.errors])], }; }, @@ -112,6 +114,7 @@ export const combineConcurrentResults = ( createdSignalsCount: 0, createdSignals: [], errors: [], + warningMessages: [], } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts index 197065f205fc5..08fa2f14a0fd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts @@ -19,20 +19,20 @@ import { } from '../../../../../../alerting/server'; import { BaseHit } from '../../../../../common/detection_engine/types'; import { TermAggregationBucket } from '../../../types'; -import { RefreshTypes } from '../../types'; -import { singleBulkCreate, SingleBulkCreateResponse } from '../single_bulk_create'; +import { GenericBulkCreateResponse } from '../bulk_create_factory'; import { calculateThresholdSignalUuid, getThresholdAggregationParts, getThresholdTermsHash, } from '../utils'; -import { BuildRuleMessage } from '../rule_messages'; import type { MultiAggBucket, SignalSource, SignalSearchResponse, ThresholdSignalHistory, AlertAttributes, + BulkCreate, + WrapHits, } from '../types'; import { ThresholdRuleParams } from '../../schemas/rule_schemas'; @@ -42,14 +42,13 @@ interface BulkCreateThresholdSignalsParams { services: AlertServices; inputIndexPattern: string[]; logger: Logger; - id: string; filter: unknown; signalsIndex: string; - refresh: RefreshTypes; startedAt: Date; from: Date; thresholdSignalHistory: ThresholdSignalHistory; - buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } const getTransformedHits = ( @@ -76,7 +75,7 @@ const getTransformedHits = ( return []; } - const getCombinations = (buckets: TermAggregationBucket[], i: number, field: string) => { + const getCombinations = (buckets: TermAggregationBucket[], i: number, field: string | null) => { return buckets.reduce((acc: MultiAggBucket[], bucket: TermAggregationBucket) => { if (i < threshold.field.length - 1) { const nextLevelIdx = i + 1; @@ -100,7 +99,7 @@ const getTransformedHits = ( topThresholdHits: val.topThresholdHits, docCount: val.docCount, }; - acc.push(el); + acc.push(el as MultiAggBucket); }); } else { const el = { @@ -121,80 +120,76 @@ const getTransformedHits = ( topThresholdHits: bucket.top_threshold_hits, docCount: bucket.doc_count, }; - acc.push(el); + acc.push(el as MultiAggBucket); } return acc; }, []); }; - // Recurse through the nested buckets and collect each unique combination of terms. Collect the - // cardinality and document count from the leaf buckets and return a signal for each set of terms. - // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response - return getCombinations(results.aggregations![aggParts.name].buckets, 0, aggParts.field).reduce( - (acc: Array>, bucket) => { - const hit = bucket.topThresholdHits?.hits.hits[0]; - if (hit == null) { - return acc; - } - - const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); - if (timestampArray == null) { - return acc; - } - - const timestamp = timestampArray[0]; - if (typeof timestamp !== 'string') { - return acc; - } - - const termsHash = getThresholdTermsHash(bucket.terms); - const signalHit = thresholdSignalHistory[termsHash]; - - const source = { - '@timestamp': timestamp, - ...bucket.terms.reduce((termAcc, term) => { - if (!term.field.startsWith('signal.')) { - return { - ...termAcc, - [term.field]: term.value, - }; - } - return termAcc; - }, {}), - threshold_result: { - terms: bucket.terms, - cardinality: bucket.cardinality, - count: bucket.docCount, - // Store `from` in the signal so that we know the lower bound for the - // threshold set in the timeline search. The upper bound will always be - // the `original_time` of the signal (the timestamp of the latest event - // in the set). - from: - signalHit?.lastSignalTimestamp != null - ? new Date(signalHit!.lastSignalTimestamp) - : from, - }, - }; + return getCombinations( + (results.aggregations![aggParts.name] as { buckets: TermAggregationBucket[] }).buckets, + 0, + aggParts.field + ).reduce((acc: Array>, bucket) => { + const hit = bucket.topThresholdHits?.hits.hits[0]; + if (hit == null) { + return acc; + } - acc.push({ - _index: inputIndex, - _id: calculateThresholdSignalUuid( - ruleId, - startedAt, - threshold.field, - bucket.terms - .map((term) => term.value) - .sort() - .join(',') - ), - _source: source, - }); + const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); + if (timestampArray == null) { + return acc; + } + const timestamp = timestampArray[0]; + if (typeof timestamp !== 'string') { return acc; - }, - [] - ); + } + + const termsHash = getThresholdTermsHash(bucket.terms); + const signalHit = thresholdSignalHistory[termsHash]; + + const source = { + '@timestamp': timestamp, + ...bucket.terms.reduce((termAcc, term) => { + if (!term.field.startsWith('signal.')) { + return { + ...termAcc, + [term.field]: term.value, + }; + } + return termAcc; + }, {}), + threshold_result: { + terms: bucket.terms, + cardinality: bucket.cardinality, + count: bucket.docCount, + // Store `from` in the signal so that we know the lower bound for the + // threshold set in the timeline search. The upper bound will always be + // the `original_time` of the signal (the timestamp of the latest event + // in the set). + from: + signalHit?.lastSignalTimestamp != null ? new Date(signalHit!.lastSignalTimestamp) : from, + }, + }; + + acc.push({ + _index: inputIndex, + _id: calculateThresholdSignalUuid( + ruleId, + startedAt, + threshold.field, + bucket.terms + .map((term) => term.value) + .sort() + .join(',') + ), + _source: source, + }); + + return acc; + }, []); }; export const transformThresholdResultsToEcs = ( @@ -238,7 +233,7 @@ export const transformThresholdResultsToEcs = ( export const bulkCreateThresholdSignals = async ( params: BulkCreateThresholdSignalsParams -): Promise => { +): Promise> => { const ruleParams = params.ruleSO.attributes.params; const thresholdResults = params.someResult; const ecsResults = transformThresholdResultsToEcs( @@ -253,7 +248,6 @@ export const bulkCreateThresholdSignals = async ( ruleParams.timestampOverride, params.thresholdSignalHistory ); - const buildRuleMessage = params.buildRuleMessage; - return singleBulkCreate({ ...params, filteredEvents: ecsResults, buildRuleMessage }); + return params.bulkCreate(params.wrapHits(ecsResults.hits.hits)); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 4205c2d6d8b2c..c35eb04ba1270 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -26,12 +26,12 @@ import { RuleAlertAction, SearchTypes, } from '../../../../common/detection_engine/types'; -import { RefreshTypes } from '../types'; import { ListClient } from '../../../../../lists/server'; import { Logger, SavedObject } from '../../../../../../../src/core/server'; import { BuildRuleMessage } from './rule_messages'; import { TelemetryEventsSender } from '../../telemetry/sender'; import { RuleParams } from '../schemas/rule_schemas'; +import { GenericBulkCreateResponse } from './bulk_create_factory'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -255,6 +255,12 @@ export interface QueryFilter { export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise; +export type BulkCreate = (docs: Array>) => Promise>; + +export type WrapHits = ( + hits: Array> +) => Array>; + export interface SearchAfterAndBulkCreateParams { tuples: Array<{ to: moment.Moment; @@ -272,9 +278,10 @@ export interface SearchAfterAndBulkCreateParams { signalsIndex: string; pageSize: number; filter: unknown; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; enrichment?: SignalsEnrichment; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } export interface SearchAfterAndBulkCreateReturnType { @@ -284,8 +291,9 @@ export interface SearchAfterAndBulkCreateReturnType { bulkCreateTimes: string[]; lastLookBackDate: Date | null | undefined; createdSignalsCount: number; - createdSignals: SignalHit[]; + createdSignals: unknown[]; errors: string[]; + warningMessages: string[]; totalToFromTuples?: Array<{ to: Moment | undefined; from: Moment | undefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 60bf0ec337f3d..616cf714d6a8c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1083,6 +1083,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(newSearchResult).toEqual(expected); }); @@ -1102,6 +1103,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(newSearchResult).toEqual(expected); }); @@ -1380,6 +1382,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1394,6 +1397,7 @@ describe('utils', () => { searchAfterTimes: ['123'], success: false, warning: true, + warningMessages: ['test warning'], }); const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: ['123'], @@ -1404,6 +1408,7 @@ describe('utils', () => { searchAfterTimes: ['123'], success: false, warning: true, + warningMessages: ['test warning'], }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1423,6 +1428,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1440,6 +1446,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(merged).toEqual(expected); }); @@ -1494,6 +1501,7 @@ describe('utils', () => { lastLookBackDate: new Date('2020-08-21T18:51:25.193Z'), searchAfterTimes: ['123'], success: true, + warningMessages: ['warning1'], }), createSearchAfterReturnType({ bulkCreateTimes: ['456'], @@ -1503,6 +1511,8 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['567'], success: true, + warningMessages: ['warning2'], + warning: true, }), ]); const expected: SearchAfterAndBulkCreateReturnType = { @@ -1513,7 +1523,8 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), // takes the next lastLookBackDate searchAfterTimes: ['123', '567'], // concatenates the searchAfterTimes together success: true, // Defaults to success true is all of it was successful - warning: false, + warning: true, + warningMessages: ['warning1', 'warning2'], }; expect(merged).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 1de76f64fabec..6d67bab6eb2f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { createHash } from 'crypto'; import moment from 'moment'; import uuidv5 from 'uuid/v5'; @@ -17,6 +16,7 @@ import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-i import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; +import { ElasticsearchClient } from '@kbn/securitysolution-es-utils'; import { TimestampOverrideOrUndefined, Privilege, @@ -38,6 +38,7 @@ import { WrappedSignalHit, RuleRangeTuple, BaseSignalHit, + SignalSourceHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { ShardError } from '../../types'; @@ -163,9 +164,15 @@ export const hasTimestampFields = async ( export const checkPrivileges = async ( services: AlertServices, indices: string[] +): Promise => + checkPrivilegesFromEsClient(services.scopedClusterClient.asCurrentUser, indices); + +export const checkPrivilegesFromEsClient = async ( + esClient: ElasticsearchClient, + indices: string[] ): Promise => ( - await services.scopedClusterClient.asCurrentUser.transport.request({ + await esClient.transport.request({ path: '/_security/user/_has_privileges', method: 'POST', body: { @@ -608,7 +615,7 @@ export const getValidDateFromDoc = ({ doc.fields != null && doc.fields[timestamp] != null ? doc.fields[timestamp][0] : doc._source != null - ? doc._source[timestamp] + ? (doc._source as { [key: string]: unknown })[timestamp] : undefined; const lastTimestamp = typeof timestampValue === 'string' || typeof timestampValue === 'number' @@ -657,6 +664,7 @@ export const createSearchAfterReturnType = ({ createdSignalsCount, createdSignals, errors, + warningMessages, }: { success?: boolean | undefined; warning?: boolean; @@ -664,8 +672,9 @@ export const createSearchAfterReturnType = ({ bulkCreateTimes?: string[] | undefined; lastLookBackDate?: Date | undefined; createdSignalsCount?: number | undefined; - createdSignals?: SignalHit[] | undefined; + createdSignals?: unknown[] | undefined; errors?: string[] | undefined; + warningMessages?: string[] | undefined; } = {}): SearchAfterAndBulkCreateReturnType => { return { success: success ?? true, @@ -676,10 +685,12 @@ export const createSearchAfterReturnType = ({ createdSignalsCount: createdSignalsCount ?? 0, createdSignals: createdSignals ?? [], errors: errors ?? [], + warningMessages: warningMessages ?? [], }; }; export const createSearchResultReturnType = (): SignalSearchResponse => { + const hits: SignalSourceHit[] = []; return { took: 0, timed_out: false, @@ -693,7 +704,7 @@ export const createSearchResultReturnType = (): SignalSearchResponse => { hits: { total: 0, max_score: 0, - hits: [], + hits, }, }; }; @@ -711,7 +722,8 @@ export const mergeReturns = ( createdSignalsCount: existingCreatedSignalsCount, createdSignals: existingCreatedSignals, errors: existingErrors, - } = prev; + warningMessages: existingWarningMessages, + }: SearchAfterAndBulkCreateReturnType = prev; const { success: newSuccess, @@ -722,7 +734,8 @@ export const mergeReturns = ( createdSignalsCount: newCreatedSignalsCount, createdSignals: newCreatedSignals, errors: newErrors, - } = next; + warningMessages: newWarningMessages, + }: SearchAfterAndBulkCreateReturnType = next; return { success: existingSuccess && newSuccess, @@ -733,6 +746,7 @@ export const mergeReturns = ( createdSignalsCount: existingCreatedSignalsCount + newCreatedSignalsCount, createdSignals: [...existingCreatedSignals, ...newCreatedSignals], errors: [...new Set([...existingErrors, ...newErrors])], + warningMessages: [...existingWarningMessages, ...newWarningMessages], }; }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts new file mode 100644 index 0000000000000..3f3e4ef3631bd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SearchAfterAndBulkCreateParams, + SignalSourceHit, + WrapHits, + WrappedSignalHit, +} from './types'; +import { generateId } from './utils'; +import { buildBulkBody } from './build_bulk_body'; +import { filterDuplicateSignals } from './filter_duplicate_signals'; + +export const wrapHitsFactory = ({ + ruleSO, + signalsIndex, +}: { + ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; + signalsIndex: string; +}): WrapHits => (events) => { + const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [ + { + _index: signalsIndex, + // TODO: bring back doc._version + _id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''), + _source: buildBulkBody(ruleSO, doc as SignalSourceHit), + }, + ]); + + return filterDuplicateSignals(ruleSO.id, wrappedDocs); +}; diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index f1c7a275e162c..6ef51bc3c53d4 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -17,6 +17,7 @@ import { Notes } from './timeline/saved_object/notes'; import { PinnedEvent } from './timeline/saved_object/pinned_events'; import { Timeline } from './timeline/saved_object/timelines'; import { TotalValue, BaseHit, Explanation } from '../../common/detection_engine/types'; +import { SignalHit } from './detection_engine/signals/types'; export interface AppDomainLibs { fields: IndexFields; @@ -100,6 +101,8 @@ export interface SearchResponse extends BaseSearchResponse { export type SearchHit = SearchResponse['hits']['hits'][0]; +export type SearchSignalHit = SearchResponse['hits']['hits'][0]; + export interface TermAggregationBucket { key: string; doc_count: number; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 609ee88c319f9..a0f466512cc1d 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -260,6 +260,8 @@ export class Plugin implements IPlugin Date: Mon, 14 Jun 2021 20:25:04 +0200 Subject: [PATCH 15/91] [Discover] Unskip runtime field editor test (#101059) --- .../apps/discover/_runtime_fields_editor.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/functional/apps/discover/_runtime_fields_editor.ts b/test/functional/apps/discover/_runtime_fields_editor.ts index 648fa3efe337c..46fe5c34f4cf3 100644 --- a/test/functional/apps/discover/_runtime_fields_editor.ts +++ b/test/functional/apps/discover/_runtime_fields_editor.ts @@ -31,7 +31,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await fieldEditor.save(); }; - describe.skip('discover integration with runtime fields editor', function describeIndexTests() { + describe('discover integration with runtime fields editor', function describeIndexTests() { before(async function () { await esArchiver.load('test/functional/fixtures/es_archiver/discover'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); @@ -104,7 +104,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // flaky https://github.com/elastic/kibana/issues/100966 it('doc view includes runtime fields', async function () { // navigate to doc view const table = await PageObjects.discover.getDocTable(); @@ -121,10 +120,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await rowActions[idxToClick].click(); }); - const hasDocHit = await testSubjects.exists('doc-hit'); - expect(hasDocHit).to.be(true); - const runtimeFieldsRow = await testSubjects.exists('tableDocViewRow-discover runtimefield'); - expect(runtimeFieldsRow).to.be(true); + await retry.waitFor('doc viewer is displayed with runtime field', async () => { + const hasDocHit = await testSubjects.exists('doc-hit'); + if (!hasDocHit) { + // Maybe loading has not completed + throw new Error('test subject doc-hit is not yet displayed'); + } + const runtimeFieldsRow = await testSubjects.exists('tableDocViewRow-discover runtimefield'); + + return hasDocHit && runtimeFieldsRow; + }); }); }); } From 8f5dad98a181eed7c4b1efe6c1cf41f906cfd042 Mon Sep 17 00:00:00 2001 From: Kerry Gallagher <471693+Kerry350@users.noreply.github.com> Date: Mon, 14 Jun 2021 19:28:08 +0100 Subject: [PATCH 16/91] [Logs / Metrics UI] Convert logs and metrics pages to the new Observability page template (#101239) * Convert Logs and Metrics pages to use the Observability page template --- x-pack/plugins/infra/kibana.json | 5 +- .../metric_anomaly/components/expression.tsx | 4 +- .../infra/public/apps/common_styles.ts | 13 +- x-pack/plugins/infra/public/apps/logs_app.tsx | 2 +- .../plugins/infra/public/apps/metrics_app.tsx | 6 +- .../public/assets/anomaly_chart_minified.svg | 1 - .../components/empty_states/no_indices.tsx | 28 +-- .../infra/public/components/error_page.tsx | 97 +++----- .../infra/public/components/loading_page.tsx | 44 ++-- .../logging/log_analysis_setup/index.ts | 2 - .../logging/log_analysis_setup/setup_page.tsx | 60 ----- .../logging/log_source_error_page.tsx | 14 +- .../components/navigation/app_navigation.tsx | 33 --- .../components/navigation/routed_tabs.tsx | 62 ----- .../infra/public/components/page_template.tsx | 22 ++ .../subscription_splash_content.tsx | 123 +++------- .../infra/public/components/toolbar_panel.ts | 27 -- x-pack/plugins/infra/public/index.scss | 16 -- .../pages/link_to/link_to_logs.test.tsx | 16 +- .../pages/logs/log_entry_categories/page.tsx | 5 +- .../log_entry_categories/page_content.tsx | 61 ++++- .../page_results_content.tsx | 77 ++++-- .../page_setup_content.tsx | 34 ++- .../top_categories/top_categories_section.tsx | 68 +---- .../public/pages/logs/log_entry_rate/page.tsx | 5 +- .../logs/log_entry_rate/page_content.tsx | 58 ++++- .../log_entry_rate/page_results_content.tsx | 160 ++++++------ .../log_entry_rate/page_setup_content.tsx | 34 ++- .../sections/anomalies/index.tsx | 19 -- .../infra/public/pages/logs/page_content.tsx | 25 +- .../infra/public/pages/logs/page_template.tsx | 24 ++ .../source_configuration_settings.tsx | 196 +++++++-------- .../infra/public/pages/logs/stream/page.tsx | 7 +- .../public/pages/logs/stream/page_content.tsx | 33 ++- .../pages/logs/stream/page_logs_content.tsx | 4 +- .../public/pages/logs/stream/page_toolbar.tsx | 7 +- .../infra/public/pages/metrics/index.tsx | 199 +++++++-------- .../components/bottom_drawer.tsx | 4 +- .../inventory_view/components/filter_bar.tsx | 24 +- .../ml/anomaly_detection/flyout_home.tsx | 8 +- .../waffle/waffle_time_controls.tsx | 13 +- .../pages/metrics/inventory_view/index.tsx | 157 +++++++----- .../components/node_details_page.tsx | 118 ++++----- .../metric_detail/components/side_nav.tsx | 25 +- .../pages/metrics/metric_detail/index.tsx | 80 +++--- .../metrics_explorer/components/toolbar.tsx | 5 +- .../pages/metrics/metrics_explorer/index.tsx | 75 +++--- .../public/pages/metrics/page_template.tsx | 24 ++ .../source_configuration_settings.tsx | 232 +++++++++--------- x-pack/plugins/infra/public/plugin.ts | 59 ++++- .../public/components/shared/index.tsx | 1 + .../components/shared/page_template/index.ts | 1 + .../shared/page_template/page_template.tsx | 1 + x-pack/plugins/observability/public/index.ts | 2 + .../translations/translations/ja-JP.json | 6 - .../translations/translations/zh-CN.json | 6 - .../page_objects/infra_home_page.ts | 24 +- 57 files changed, 1149 insertions(+), 1307 deletions(-) delete mode 100644 x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg delete mode 100644 x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx delete mode 100644 x-pack/plugins/infra/public/components/navigation/app_navigation.tsx delete mode 100644 x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx create mode 100644 x-pack/plugins/infra/public/components/page_template.tsx delete mode 100644 x-pack/plugins/infra/public/components/toolbar_panel.ts create mode 100644 x-pack/plugins/infra/public/pages/logs/page_template.tsx create mode 100644 x-pack/plugins/infra/public/pages/metrics/page_template.tsx diff --git a/x-pack/plugins/infra/kibana.json b/x-pack/plugins/infra/kibana.json index c0d567ef83ced..ec1b11c90f7a3 100644 --- a/x-pack/plugins/infra/kibana.json +++ b/x-pack/plugins/infra/kibana.json @@ -11,9 +11,10 @@ "dataEnhanced", "visTypeTimeseries", "alerting", - "triggersActionsUi" + "triggersActionsUi", + "observability" ], - "optionalPlugins": ["ml", "observability", "home", "embeddable"], + "optionalPlugins": ["ml", "home", "embeddable"], "server": true, "ui": true, "configPath": ["xpack", "infra"], diff --git a/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx b/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx index afbd6ffa8b5f7..e44a747aa07e7 100644 --- a/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx +++ b/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx @@ -11,7 +11,7 @@ import { EuiFlexGroup, EuiSpacer, EuiText, EuiLoadingContent } from '@elastic/eu import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; import { useInfraMLCapabilities } from '../../../containers/ml/infra_ml_capabilities'; -import { SubscriptionSplashContent } from '../../../components/subscription_splash_content'; +import { SubscriptionSplashPrompt } from '../../../components/subscription_splash_content'; import { AlertPreview } from '../../common'; import { METRIC_ANOMALY_ALERT_TYPE_ID, @@ -185,7 +185,7 @@ export const Expression: React.FC = (props) => { }, [metadata, derivedIndexPattern, defaultExpression, source, space]); // eslint-disable-line react-hooks/exhaustive-deps if (isLoadingMLCapabilities) return ; - if (!hasInfraMLCapabilities) return ; + if (!hasInfraMLCapabilities) return ; return ( // https://github.com/elastic/kibana/issues/89506 diff --git a/x-pack/plugins/infra/public/apps/common_styles.ts b/x-pack/plugins/infra/public/apps/common_styles.ts index be12c6cdc937f..68c820d538ca9 100644 --- a/x-pack/plugins/infra/public/apps/common_styles.ts +++ b/x-pack/plugins/infra/public/apps/common_styles.ts @@ -5,10 +5,15 @@ * 2.0. */ +import { APP_WRAPPER_CLASS } from '../../../../../src/core/public'; + export const CONTAINER_CLASSNAME = 'infra-container-element'; -export const prepareMountElement = (element: HTMLElement) => { - // Ensure the element we're handed from application mounting is assigned a class - // for our index.scss styles to apply to. - element.classList.add(CONTAINER_CLASSNAME); +export const prepareMountElement = (element: HTMLElement, testSubject?: string) => { + // Ensure all wrapping elements have the APP_WRAPPER_CLASS so that the KinanaPageTemplate works as expected + element.classList.add(APP_WRAPPER_CLASS); + + if (testSubject) { + element.setAttribute('data-test-subj', testSubject); + } }; diff --git a/x-pack/plugins/infra/public/apps/logs_app.tsx b/x-pack/plugins/infra/public/apps/logs_app.tsx index 61082efe43647..b512b5ce4a176 100644 --- a/x-pack/plugins/infra/public/apps/logs_app.tsx +++ b/x-pack/plugins/infra/public/apps/logs_app.tsx @@ -27,7 +27,7 @@ export const renderApp = ( ) => { const storage = new Storage(window.localStorage); - prepareMountElement(element); + prepareMountElement(element, 'infraLogsPage'); ReactDOM.render( { const storage = new Storage(window.localStorage); - prepareMountElement(element); + prepareMountElement(element, 'infraMetricsPage'); ReactDOM.render( )} - {uiCapabilities?.infrastructure?.show && ( - - )} {uiCapabilities?.infrastructure?.show && ( )} diff --git a/x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg b/x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg deleted file mode 100644 index dd1b39248bba2..0000000000000 --- a/x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx b/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx index 264428e44a44a..c61a567ac73b1 100644 --- a/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx +++ b/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx @@ -7,8 +7,7 @@ import { EuiEmptyPrompt } from '@elastic/eui'; import React from 'react'; - -import { euiStyled } from '../../../../../../src/plugins/kibana_react/common'; +import { PageTemplate } from '../page_template'; interface NoIndicesProps { message: string; @@ -17,15 +16,16 @@ interface NoIndicesProps { 'data-test-subj'?: string; } -export const NoIndices: React.FC = ({ actions, message, title, ...rest }) => ( - {title}} - body={

{message}

} - actions={actions} - {...rest} - /> -); - -const CenteredEmptyPrompt = euiStyled(EuiEmptyPrompt)` - align-self: center; -`; +// Represents a fully constructed page, including page template. +export const NoIndices: React.FC = ({ actions, message, title, ...rest }) => { + return ( + + {title}} + body={

{message}

} + actions={actions} + {...rest} + /> + + ); +}; diff --git a/x-pack/plugins/infra/public/components/error_page.tsx b/x-pack/plugins/infra/public/components/error_page.tsx index 184901b4fdd9b..da6716ddc7f72 100644 --- a/x-pack/plugins/infra/public/components/error_page.tsx +++ b/x-pack/plugins/infra/public/components/error_page.tsx @@ -5,20 +5,10 @@ * 2.0. */ -import { - EuiButton, - EuiCallOut, - EuiFlexGroup, - EuiFlexItem, - EuiPageBody, - EuiPageContent, - EuiPageContentBody, - EuiSpacer, -} from '@elastic/eui'; +import { EuiButton, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import React from 'react'; -import { euiStyled } from '../../../../../src/plugins/kibana_react/common'; -import { FlexPage } from './page'; +import { PageTemplate } from './page_template'; interface Props { detailedMessage?: React.ReactNode; @@ -26,51 +16,40 @@ interface Props { shortMessage: React.ReactNode; } -export const ErrorPage: React.FC = ({ detailedMessage, retry, shortMessage }) => ( - - - = ({ detailedMessage, retry, shortMessage }) => { + return ( + + + } > - - - } - > - - {shortMessage} - {retry ? ( - - - - - - ) : null} - - {detailedMessage ? ( - <> - -
{detailedMessage}
- - ) : null} - - - - - -); - -const MinimumPageContent = euiStyled(EuiPageContent)` - min-width: 50vh; -`; + + {shortMessage} + {retry ? ( + + + + + + ) : null} + + {detailedMessage ? ( + <> + +
{detailedMessage}
+ + ) : null} + + + ); +}; diff --git a/x-pack/plugins/infra/public/components/loading_page.tsx b/x-pack/plugins/infra/public/components/loading_page.tsx index 755511374b75f..2b2859707a20d 100644 --- a/x-pack/plugins/infra/public/components/loading_page.tsx +++ b/x-pack/plugins/infra/public/components/loading_page.tsx @@ -5,34 +5,38 @@ * 2.0. */ -import { - EuiFlexGroup, - EuiFlexItem, - EuiLoadingSpinner, - EuiPageBody, - EuiPageContent, -} from '@elastic/eui'; +import { EuiEmptyPrompt, EuiLoadingSpinner, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React, { ReactNode } from 'react'; - -import { FlexPage } from './page'; +import { PageTemplate } from './page_template'; interface LoadingPageProps { message?: ReactNode; 'data-test-subj'?: string; } +// Represents a fully constructed page, including page template. export const LoadingPage = ({ message, 'data-test-subj': dataTestSubj = 'loadingPage', -}: LoadingPageProps) => ( - - - - - - {message} +}: LoadingPageProps) => { + return ( + + + + ); +}; + +export const LoadingPrompt = ({ message }: LoadingPageProps) => { + return ( + + + + + {message} - - - -); + } + /> + ); +}; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts index db5a996c604fc..9ca08c69cf600 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts @@ -5,8 +5,6 @@ * 2.0. */ -export * from './setup_page'; - export * from './initial_configuration_step'; export * from './process_step'; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx deleted file mode 100644 index a998d0c304a5e..0000000000000 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - CommonProps, - EuiPage, - EuiPageBody, - EuiPageContent, - EuiPageContentBody, - EuiPageContentHeader, - EuiPageContentHeaderSection, - EuiTitle, -} from '@elastic/eui'; -import React from 'react'; - -import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; - -export const LogAnalysisSetupPage: React.FunctionComponent = ({ - children, - ...rest -}) => { - return ( - - - - {children} - - - - ); -}; - -export const LogAnalysisSetupPageHeader: React.FunctionComponent = ({ children }) => ( - - - -

{children}

- - - -); - -export const LogAnalysisSetupPageContent = EuiPageContentBody; - -// !important due to https://github.com/elastic/eui/issues/2232 -const LogEntryRateSetupPageContent = euiStyled(EuiPageContent)` - max-width: 768px !important; -`; - -const LogEntryRateSetupPage = euiStyled(EuiPage)` - height: 100%; -`; diff --git a/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx b/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx index 8ea35fd8f259f..6c757f7383a06 100644 --- a/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx @@ -5,14 +5,7 @@ * 2.0. */ -import { - EuiButton, - EuiButtonEmpty, - EuiCallOut, - EuiEmptyPrompt, - EuiPageTemplate, - EuiSpacer, -} from '@elastic/eui'; +import { EuiButton, EuiButtonEmpty, EuiCallOut, EuiEmptyPrompt, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import React from 'react'; import { SavedObjectNotFound } from '../../../../../../src/plugins/kibana_utils/common'; @@ -22,6 +15,7 @@ import { ResolveLogSourceConfigurationError, } from '../../../common/log_sources'; import { useLinkProps } from '../../hooks/use_link_props'; +import { LogsPageTemplate } from '../../pages/logs/page_template'; export const LogSourceErrorPage: React.FC<{ errors: Error[]; @@ -30,7 +24,7 @@ export const LogSourceErrorPage: React.FC<{ const settingsLinkProps = useLinkProps({ app: 'logs', pathname: '/settings' }); return ( - + , ]} /> - + ); }; diff --git a/x-pack/plugins/infra/public/components/navigation/app_navigation.tsx b/x-pack/plugins/infra/public/components/navigation/app_navigation.tsx deleted file mode 100644 index 966b91537d3bd..0000000000000 --- a/x-pack/plugins/infra/public/components/navigation/app_navigation.tsx +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import React from 'react'; -import { euiStyled } from '../../../../../../src/plugins/kibana_react/common'; - -interface AppNavigationProps { - 'aria-label': string; - children: React.ReactNode; -} - -export const AppNavigation = ({ 'aria-label': label, children }: AppNavigationProps) => ( - -); - -const Nav = euiStyled.nav` - background: ${(props) => props.theme.eui.euiColorEmptyShade}; - border-bottom: ${(props) => props.theme.eui.euiBorderThin}; - padding: ${(props) => `${props.theme.eui.euiSizeS} ${props.theme.eui.euiSizeL}`}; - .euiTabs { - padding-left: 3px; - margin-left: -3px; - }; -`; diff --git a/x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx b/x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx deleted file mode 100644 index 2a5ffcd826e7c..0000000000000 --- a/x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiLink, EuiTab, EuiTabs } from '@elastic/eui'; -import React from 'react'; -import { Route } from 'react-router-dom'; - -import { euiStyled } from '../../../../../../src/plugins/kibana_react/common'; -import { useLinkProps } from '../../hooks/use_link_props'; -import { LinkDescriptor } from '../../hooks/use_link_props'; - -interface TabConfig { - title: string | React.ReactNode; -} - -type TabConfiguration = TabConfig & LinkDescriptor; - -interface RoutedTabsProps { - tabs: TabConfiguration[]; -} - -const noop = () => {}; - -export const RoutedTabs = ({ tabs }: RoutedTabsProps) => { - return ( - - {tabs.map((tab) => { - return ; - })} - - ); -}; - -const Tab = ({ title, pathname, app }: TabConfiguration) => { - const linkProps = useLinkProps({ app, pathname }); - return ( - { - return ( - - - - {title} - - - - ); - }} - /> - ); -}; - -const TabContainer = euiStyled.div` - .euiLink { - color: inherit !important; - } -`; diff --git a/x-pack/plugins/infra/public/components/page_template.tsx b/x-pack/plugins/infra/public/components/page_template.tsx new file mode 100644 index 0000000000000..1a10a6cd831b9 --- /dev/null +++ b/x-pack/plugins/infra/public/components/page_template.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { useKibanaContextForPlugin } from '../hooks/use_kibana'; +import type { LazyObservabilityPageTemplateProps } from '../../../observability/public'; + +export const PageTemplate: React.FC = (pageTemplateProps) => { + const { + services: { + observability: { + navigation: { PageTemplate: Template }, + }, + }, + } = useKibanaContextForPlugin(); + + return