From b921a49b3b2b68ac7d14181962b2c7a7b70f703d Mon Sep 17 00:00:00 2001 From: Michael Dokolin Date: Mon, 24 Oct 2022 17:45:25 +0200 Subject: [PATCH 01/15] [Embeddable] Add seamless React integration (#143131) * Add support of React nodes rendering to the embeddable panel * Add support of React nodes rendering by the error handler * Update visualizations embeddable to use React for error handling --- .../component/control_frame_component.tsx | 73 ++++++++++++--- src/plugins/embeddable/README.md | 51 ++++++++--- .../__stories__/embeddable_panel.stories.tsx | 17 ++-- .../__stories__/embeddable_root.stories.tsx | 2 +- .../__stories__/error_embeddable.stories.tsx | 37 ++------ .../__stories__/hello_world_embeddable.tsx | 20 ++--- .../public/lib/embeddables/embeddable.tsx | 11 +-- .../lib/embeddables/embeddable_root.test.tsx | 9 +- .../lib/embeddables/embeddable_root.tsx | 52 +++++++---- .../lib/embeddables/error_embeddable.test.tsx | 41 --------- .../lib/embeddables/error_embeddable.tsx | 81 ++--------------- .../public/lib/embeddables/i_embeddable.ts | 11 ++- .../lib/panel/embeddable_panel.test.tsx | 88 +++++++++++++++---- .../public/lib/panel/embeddable_panel.tsx | 49 +++++++---- .../lib/panel/embeddable_panel_error.tsx | 34 ++++--- .../contact_card/contact_card_embeddable.tsx | 4 +- .../contact_card_embeddable_factory.tsx | 2 +- .../contact_card_embeddable_react.tsx | 19 ++++ .../contact_card_embeddable_react_factory.ts | 28 ++++++ .../embeddables/contact_card/index.ts | 2 + .../fixtures/hello_world_embeddable_react.tsx | 16 ++++ .../embeddable/public/tests/fixtures/index.ts | 1 + .../embeddable/visualize_embeddable.tsx | 15 ++-- .../translations/translations/fr-FR.json | 2 +- .../translations/translations/ja-JP.json | 2 +- .../translations/translations/zh-CN.json | 2 +- 26 files changed, 385 insertions(+), 284 deletions(-) create mode 100644 src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react.tsx create mode 100644 src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react_factory.ts create mode 100644 src/plugins/embeddable/public/tests/fixtures/hello_world_embeddable_react.tsx diff --git a/src/plugins/controls/public/control_group/component/control_frame_component.tsx b/src/plugins/controls/public/control_group/component/control_frame_component.tsx index 196b2cfe28be3..8f53e2e29c63e 100644 --- a/src/plugins/controls/public/control_group/component/control_frame_component.tsx +++ b/src/plugins/controls/public/control_group/component/control_frame_component.tsx @@ -13,12 +13,17 @@ import { EuiFormControlLayout, EuiFormLabel, EuiFormRow, + EuiIcon, + EuiLink, EuiLoadingChart, + EuiPopover, + EuiText, EuiToolTip, } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { Markdown } from '@kbn/kibana-react-plugin/public'; import { useReduxContainerContext } from '@kbn/presentation-util-plugin/public'; -import { ErrorEmbeddable } from '@kbn/embeddable-plugin/public'; import { ControlGroupReduxState } from '../types'; import { pluginServices } from '../../services'; import { EditControlButton } from '../editor/edit_control'; @@ -26,6 +31,44 @@ import { ControlGroupStrings } from '../control_group_strings'; import { useChildEmbeddable } from '../../hooks/use_child_embeddable'; import { TIME_SLIDER_CONTROL } from '../../../common'; +interface ControlFrameErrorProps { + error: Error; +} + +const ControlFrameError = ({ error }: ControlFrameErrorProps) => { + const [isPopoverOpen, setPopoverOpen] = useState(false); + const popoverButton = ( + + setPopoverOpen((open) => !open)} + > + + + + + ); + + return ( + setPopoverOpen(false)} + > + + + ); +}; + export interface ControlFrameProps { customPrepend?: JSX.Element; enableActions?: boolean; @@ -40,7 +83,7 @@ export const ControlFrame = ({ embeddableType, }: ControlFrameProps) => { const embeddableRoot: React.RefObject = useMemo(() => React.createRef(), []); - const [hasFatalError, setHasFatalError] = useState(false); + const [fatalError, setFatalError] = useState(); const { useEmbeddableSelector: select, @@ -61,19 +104,14 @@ export const ControlFrame = ({ const usingTwoLineLayout = controlStyle === 'twoLine'; useEffect(() => { - if (embeddableRoot.current && embeddable) { - embeddable.render(embeddableRoot.current); + if (embeddableRoot.current) { + embeddable?.render(embeddableRoot.current); } const inputSubscription = embeddable ?.getInput$() .subscribe((newInput) => setTitle(newInput.title)); const errorSubscription = embeddable?.getOutput$().subscribe({ - error: (error: Error) => { - if (!embeddableRoot.current) return; - const errorEmbeddable = new ErrorEmbeddable(error, { id: embeddable.id }, undefined, true); - errorEmbeddable.render(embeddableRoot.current); - setHasFatalError(true); - }, + error: setFatalError, }); return () => { inputSubscription?.unsubscribe(); @@ -88,7 +126,7 @@ export const ControlFrame = ({ 'controlFrameFloatingActions--oneLine': !usingTwoLineLayout, })} > - {!hasFatalError && embeddableType !== TIME_SLIDER_CONTROL && ( + {!fatalError && embeddableType !== TIME_SLIDER_CONTROL && ( @@ -119,7 +157,7 @@ export const ControlFrame = ({ const embeddableParentClassNames = classNames('controlFrame__control', { 'controlFrame--twoLine': controlStyle === 'twoLine', 'controlFrame--oneLine': controlStyle === 'oneLine', - 'controlFrame--fatalError': hasFatalError, + 'controlFrame--fatalError': !!fatalError, }); function renderEmbeddablePrepend() { @@ -149,12 +187,19 @@ export const ControlFrame = ({ } > - {embeddable && ( + {embeddable && !fatalError && (
+ > + {fatalError && } +
+ )} + {fatalError && ( +
+ {} +
)} {!embeddable && (
diff --git a/src/plugins/embeddable/README.md b/src/plugins/embeddable/README.md index 14fab2f8412f3..eae9ef04cfb9b 100644 --- a/src/plugins/embeddable/README.md +++ b/src/plugins/embeddable/README.md @@ -90,7 +90,6 @@ export class HelloWorldEmbeddableFactoryDefinition implements EmbeddableFactoryD The embeddable should implement the `IEmbeddable` interface, and usually, that just extends the base class `Embeddable`. ```tsx import React from 'react'; -import { render } from 'react-dom'; import { Embeddable } from '@kbn/embeddable-plugin/public'; export const HELLO_WORLD = 'HELLO_WORLD'; @@ -98,8 +97,8 @@ export const HELLO_WORLD = 'HELLO_WORLD'; export class HelloWorld extends Embeddable { readonly type = HELLO_WORLD; - render(node: HTMLElement) { - render(
{this.getTitle()}
, node); + render() { + return
{this.getTitle()}
; } reload() {} @@ -126,6 +125,21 @@ export class HelloWorld extends Embeddable { } ``` +There is also an option to return a [React node](https://reactjs.org/docs/react-component.html#render) directly. +In that case, the returned node will be automatically mounted and unmounted. +```tsx +import React from 'react'; +import { Embeddable } from '@kbn/embeddable-plugin/public'; + +export class HelloWorld extends Embeddable { + // ... + + render() { + return
{this.getTitle()}
; + } +} +``` + #### `reload` This hook is called after every input update to perform some UI changes. ```typescript @@ -150,13 +164,13 @@ export class HelloWorld extends Embeddable { } ``` -#### `renderError` +#### `catchError` This is an optional error handler to provide a custom UI for the error state. The embeddable may change its state in the future so that the error should be able to disappear. In that case, the method should return a callback performing cleanup actions for the error UI. -If there is no implementation provided for the `renderError` hook, the embeddable will render a fallback error UI. +If there is no implementation provided for the `catchError` hook, the embeddable will render a fallback error UI. In case of an error, the embeddable UI will not be destroyed or unmounted. The default behavior is to hide that visually and show the error message on top of that. @@ -169,7 +183,7 @@ import { Embeddable } from '@kbn/embeddable-plugin/public'; export class HelloWorld extends Embeddable { // ... - renderError(node: HTMLElement, error: Error) { + catchError(error: Error, node: HTMLElement) { render(
Something went wrong: {error.message}
, node); return () => unmountComponentAtNode(node); @@ -177,6 +191,21 @@ export class HelloWorld extends Embeddable { } ``` +There is also an option to return a [React node](https://reactjs.org/docs/react-component.html#render) directly. +In that case, the returned node will be automatically mounted and unmounted. +```typescript +import React from 'react'; +import { Embeddable } from '@kbn/embeddable-plugin/public'; + +export class HelloWorld extends Embeddable { + // ... + + catchError(error: Error) { + return
Something went wrong: {error.message}
; + } +} +``` + #### `destroy` This hook is invoked when the embeddable is destroyed and should perform cleanup actions. ```typescript @@ -366,7 +395,6 @@ To perform state mutations, the plugin also exposes a pre-defined state of the a Here is an example of initializing a Redux store: ```tsx import React from 'react'; -import { render } from 'react-dom'; import { connect, Provider } from 'react-redux'; import { Embeddable, IEmbeddable } from '@kbn/embeddable-plugin/public'; import { createStore, State } from '@kbn/embeddable-plugin/public/store'; @@ -381,16 +409,15 @@ export class HelloWorld extends Embeddable { reload() {} - render(node: HTMLElement) { + render() { const Component = connect((state: State) => ({ title: state.input.title }))( HelloWorldComponent ); - render( + return ( - , - node + ); } } @@ -434,7 +461,6 @@ That means there is no need to reimplement already existing actions. ```tsx import React from 'react'; -import { render } from 'react-dom'; import { createSlice } from '@reduxjs/toolkit'; import { Embeddable, @@ -523,7 +549,6 @@ This can be achieved by passing a custom reducer. ```tsx import React from 'react'; -import { render } from 'react-dom'; import { createSlice } from '@reduxjs/toolkit'; import { Embeddable, IEmbeddable } from '@kbn/embeddable-plugin/public'; import { createStore, State } from '@kbn/embeddable-plugin/public/store'; diff --git a/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx b/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx index 97e8b83b9f7b1..c7fba909068da 100644 --- a/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx +++ b/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx @@ -15,7 +15,6 @@ import React, { useMemo, useRef, } from 'react'; -import { render, unmountComponentAtNode } from 'react-dom'; import { ReplaySubject } from 'rxjs'; import { ThemeContext } from '@emotion/react'; import { DecoratorFn, Meta } from '@storybook/react'; @@ -251,15 +250,13 @@ DefaultWithError.argTypes = { export function DefaultWithCustomError({ message, ...props }: DefaultWithErrorProps) { const ref = useRef>(null); - useEffect( - () => - ref.current?.embeddable.setErrorRenderer((node, error) => { - render(, node); - - return () => unmountComponentAtNode(node); - }), - [] - ); + useEffect(() => { + if (ref.current) { + ref.current.embeddable.catchError = (error) => { + return ; + }; + } + }, []); useEffect( () => void ref.current?.embeddable.store.dispatch(actions.output.setError(new Error(message))), [message] diff --git a/src/plugins/embeddable/public/__stories__/embeddable_root.stories.tsx b/src/plugins/embeddable/public/__stories__/embeddable_root.stories.tsx index e8ccc0edd66a2..2539185022f3f 100644 --- a/src/plugins/embeddable/public/__stories__/embeddable_root.stories.tsx +++ b/src/plugins/embeddable/public/__stories__/embeddable_root.stories.tsx @@ -65,7 +65,7 @@ Default.args = { loading: false, }; -export const DefaultWithError = Default as Meta; +export const DefaultWithError = Default.bind({}) as Meta; DefaultWithError.args = { ...Default.args, diff --git a/src/plugins/embeddable/public/__stories__/error_embeddable.stories.tsx b/src/plugins/embeddable/public/__stories__/error_embeddable.stories.tsx index ad65b2412c4c4..2d83d9b8ed702 100644 --- a/src/plugins/embeddable/public/__stories__/error_embeddable.stories.tsx +++ b/src/plugins/embeddable/public/__stories__/error_embeddable.stories.tsx @@ -6,14 +6,10 @@ * Side Public License, v 1. */ -import React, { useContext, useEffect, useMemo, useRef } from 'react'; -import { filter, ReplaySubject } from 'rxjs'; -import { ThemeContext } from '@emotion/react'; +import { useEffect, useMemo } from 'react'; import { Meta } from '@storybook/react'; -import { CoreTheme } from '@kbn/core-theme-browser'; import { ErrorEmbeddable } from '..'; -import { setTheme } from '../services'; export default { title: 'components/ErrorEmbeddable', @@ -26,32 +22,17 @@ export default { } as Meta; interface ErrorEmbeddableWrapperProps { - compact?: boolean; message: string; } -function ErrorEmbeddableWrapper({ compact, message }: ErrorEmbeddableWrapperProps) { +function ErrorEmbeddableWrapper({ message }: ErrorEmbeddableWrapperProps) { const embeddable = useMemo( - () => new ErrorEmbeddable(message, { id: `${Math.random()}` }, undefined, compact), - [compact, message] + () => new ErrorEmbeddable(message, { id: `${Math.random()}` }, undefined), + [message] ); - const root = useRef(null); - const theme$ = useMemo(() => new ReplaySubject(1), []); - const theme = useContext(ThemeContext) as CoreTheme; + useEffect(() => () => embeddable.destroy(), [embeddable]); - useEffect(() => setTheme({ theme$: theme$.pipe(filter(Boolean)) }), [theme$]); - useEffect(() => theme$.next(theme), [theme$, theme]); - useEffect(() => { - if (!root.current) { - return; - } - - embeddable.render(root.current); - - return () => embeddable.destroy(); - }, [embeddable]); - - return
; + return embeddable.render(); } export const Default = ErrorEmbeddableWrapper as Meta; @@ -59,9 +40,3 @@ export const Default = ErrorEmbeddableWrapper as Meta ( - -)) as Meta; - -DefaultCompact.args = { ...Default.args }; diff --git a/src/plugins/embeddable/public/__stories__/hello_world_embeddable.tsx b/src/plugins/embeddable/public/__stories__/hello_world_embeddable.tsx index 5cf2c5fdc46e8..d343425bced3e 100644 --- a/src/plugins/embeddable/public/__stories__/hello_world_embeddable.tsx +++ b/src/plugins/embeddable/public/__stories__/hello_world_embeddable.tsx @@ -7,10 +7,9 @@ */ import React from 'react'; -import { render } from 'react-dom'; import { connect, Provider } from 'react-redux'; import { EuiEmptyPrompt } from '@elastic/eui'; -import { Embeddable, IEmbeddable } from '..'; +import { Embeddable } from '..'; import { createStore, State } from '../store'; export class HelloWorldEmbeddable extends Embeddable { @@ -19,22 +18,15 @@ export class HelloWorldEmbeddable extends Embeddable { readonly type = 'hello-world'; - renderError: IEmbeddable['renderError']; - reload() {} - render(node: HTMLElement) { - const App = connect((state: State) => ({ body: state.input.title }))(EuiEmptyPrompt); + render() { + const HelloWorld = connect((state: State) => ({ body: state.input.title }))(EuiEmptyPrompt); - render( + return ( - - , - node + + ); } - - setErrorRenderer(renderer: IEmbeddable['renderError']) { - this.renderError = renderer; - } } diff --git a/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx b/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx index 94025320ec86d..d1871ce2ffc98 100644 --- a/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx @@ -14,7 +14,7 @@ import { debounceTime, distinctUntilChanged, map, skip } from 'rxjs/operators'; import { RenderCompleteDispatcher } from '@kbn/kibana-utils-plugin/public'; import { Adapters } from '../types'; import { IContainer } from '../containers'; -import { EmbeddableOutput, IEmbeddable } from './i_embeddable'; +import { EmbeddableError, EmbeddableOutput, IEmbeddable } from './i_embeddable'; import { EmbeddableInput, ViewMode } from '../../../common/types'; import { genericEmbeddableInputIsEqual, omitGenericEmbeddableInput } from './diff_embeddable_input'; @@ -23,8 +23,9 @@ function getPanelTitle(input: EmbeddableInput, output: EmbeddableOutput) { } export abstract class Embeddable< TEmbeddableInput extends EmbeddableInput = EmbeddableInput, - TEmbeddableOutput extends EmbeddableOutput = EmbeddableOutput -> implements IEmbeddable + TEmbeddableOutput extends EmbeddableOutput = EmbeddableOutput, + TNode = any +> implements IEmbeddable { static runtimeId: number = 0; @@ -33,6 +34,7 @@ export abstract class Embeddable< public readonly parent?: IContainer; public readonly isContainer: boolean = false; public readonly deferEmbeddableLoad: boolean = false; + public catchError?(error: EmbeddableError, domNode: HTMLElement | Element): TNode | (() => void); public abstract readonly type: string; public readonly id: string; @@ -209,14 +211,13 @@ export abstract class Embeddable< } } - public render(el: HTMLElement): void { + public render(el: HTMLElement): TNode | void { this.renderComplete.setEl(el); this.renderComplete.setTitle(this.output.title || ''); if (this.destroyed) { throw new Error('Embeddable has been destroyed'); } - return; } /** diff --git a/src/plugins/embeddable/public/lib/embeddables/embeddable_root.test.tsx b/src/plugins/embeddable/public/lib/embeddables/embeddable_root.test.tsx index c5912427893a6..056c652e104f0 100644 --- a/src/plugins/embeddable/public/lib/embeddables/embeddable_root.test.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/embeddable_root.test.tsx @@ -7,7 +7,7 @@ */ import React from 'react'; -import { HelloWorldEmbeddable } from '../../tests/fixtures'; +import { HelloWorldEmbeddable, HelloWorldEmbeddableReact } from '../../tests/fixtures'; import { EmbeddableRoot } from './embeddable_root'; import { mount } from 'enzyme'; import { findTestSubject } from '@elastic/eui/lib/test'; @@ -25,6 +25,13 @@ test('EmbeddableRoot renders an embeddable', async () => { expect(findTestSubject(component, 'embedError').length).toBe(0); }); +test('EmbeddableRoot renders a React-based embeddable', async () => { + const embeddable = new HelloWorldEmbeddableReact({ id: 'hello' }); + const component = mount(); + + expect(component.find('[data-test-subj="helloWorldEmbeddable"]')).toHaveLength(1); +}); + test('EmbeddableRoot updates input', async () => { const embeddable = new HelloWorldEmbeddable({ id: 'hello' }); const component = mount(); diff --git a/src/plugins/embeddable/public/lib/embeddables/embeddable_root.tsx b/src/plugins/embeddable/public/lib/embeddables/embeddable_root.tsx index cab7fcbd54e1d..bfaefe09b5e6b 100644 --- a/src/plugins/embeddable/public/lib/embeddables/embeddable_root.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/embeddable_root.tsx @@ -6,19 +6,25 @@ * Side Public License, v 1. */ -import React from 'react'; +import React, { ReactNode } from 'react'; import { EuiLoadingSpinner } from '@elastic/eui'; import { EuiText } from '@elastic/eui'; -import { EmbeddableInput, IEmbeddable } from './i_embeddable'; +import { isPromise } from '@kbn/std'; +import { MaybePromise } from '@kbn/utility-types'; +import { EmbeddableInput, EmbeddableOutput, IEmbeddable } from './i_embeddable'; interface Props { - embeddable?: IEmbeddable; + embeddable?: IEmbeddable>; loading?: boolean; error?: string; input?: EmbeddableInput; } -export class EmbeddableRoot extends React.Component { +interface State { + node?: ReactNode; +} + +export class EmbeddableRoot extends React.Component { private root?: React.RefObject; private alreadyMounted: boolean = false; @@ -26,20 +32,33 @@ export class EmbeddableRoot extends React.Component { super(props); this.root = React.createRef(); + this.state = {}; } + private updateNode = (node: MaybePromise) => { + if (isPromise(node)) { + node.then(this.updateNode); + + return; + } + + this.setState({ node }); + }; + public componentDidMount() { - if (this.root && this.root.current && this.props.embeddable) { - this.alreadyMounted = true; - this.props.embeddable.render(this.root.current); + if (!this.root?.current || !this.props.embeddable) { + return; } + + this.alreadyMounted = true; + this.updateNode(this.props.embeddable.render(this.root.current) ?? undefined); } public componentDidUpdate(prevProps?: Props) { let justRendered = false; - if (this.root && this.root.current && this.props.embeddable && !this.alreadyMounted) { + if (this.root?.current && this.props.embeddable && !this.alreadyMounted) { this.alreadyMounted = true; - this.props.embeddable.render(this.root.current); + this.updateNode(this.props.embeddable.render(this.root.current) ?? undefined); justRendered = true; } @@ -56,20 +75,21 @@ export class EmbeddableRoot extends React.Component { } } - public shouldComponentUpdate(newProps: Props) { + public shouldComponentUpdate({ embeddable, error, input, loading }: Props, { node }: State) { return Boolean( - newProps.error !== this.props.error || - newProps.loading !== this.props.loading || - newProps.embeddable !== this.props.embeddable || - (this.root && this.root.current && newProps.embeddable && !this.alreadyMounted) || - newProps.input !== this.props.input + error !== this.props.error || + loading !== this.props.loading || + embeddable !== this.props.embeddable || + (this.root && this.root.current && embeddable && !this.alreadyMounted) || + input !== this.props.input || + node !== this.state.node ); } public render() { return ( -
+
{this.state.node}
{this.props.loading && } {this.props.error && {this.props.error}} diff --git a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.test.tsx b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.test.tsx index 8f17a3bf84198..d932018c3f4fe 100644 --- a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.test.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.test.tsx @@ -20,47 +20,6 @@ test('ErrorEmbeddable renders an embeddable', async () => { expect(getByText(/some error occurred/i)).toBeVisible(); }); -test('ErrorEmbeddable renders in compact mode', async () => { - const embeddable = new ErrorEmbeddable( - 'some error occurred', - { id: '123', title: 'Error' }, - undefined, - true - ); - const component = render(); - - expect(component.baseElement).toMatchInlineSnapshot(` - -
-
-
-
-
- -
-
-
-
-
- - `); -}); - test('ErrorEmbeddable renders an embeddable with markdown message', async () => { const error = '[some link](http://localhost:5601/takeMeThere)'; const embeddable = new ErrorEmbeddable(error, { id: '123', title: 'Error' }); diff --git a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx index 55946aad5da02..8dff4ecee8976 100644 --- a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx @@ -6,16 +6,12 @@ * Side Public License, v 1. */ -import { EuiText, EuiIcon, EuiPopover, EuiLink, EuiEmptyPrompt } from '@elastic/eui'; -import React, { useState } from 'react'; -import ReactDOM from 'react-dom'; -import { KibanaThemeProvider, Markdown } from '@kbn/kibana-react-plugin/public'; -import { i18n } from '@kbn/i18n'; +import { EuiEmptyPrompt } from '@elastic/eui'; +import React, { ReactNode } from 'react'; +import { Markdown } from '@kbn/kibana-react-plugin/public'; import { Embeddable } from './embeddable'; import { EmbeddableInput, EmbeddableOutput, IEmbeddable } from './i_embeddable'; import { IContainer } from '../containers'; -import { getTheme } from '../../services'; -import './error_embedabble.scss'; export const ERROR_EMBEDDABLE_TYPE = 'error'; @@ -25,91 +21,32 @@ export function isErrorEmbeddable( return Boolean(embeddable.fatalError || (embeddable as ErrorEmbeddable).error !== undefined); } -export class ErrorEmbeddable extends Embeddable { +export class ErrorEmbeddable extends Embeddable { public readonly type = ERROR_EMBEDDABLE_TYPE; public error: Error | string; - private dom?: HTMLElement; - constructor( - error: Error | string, - input: EmbeddableInput, - parent?: IContainer, - private compact: boolean = false - ) { + constructor(error: Error | string, input: EmbeddableInput, parent?: IContainer) { super(input, {}, parent); this.error = error; } public reload() {} - public render(dom: HTMLElement) { + public render() { const title = typeof this.error === 'string' ? this.error : this.error.message; - this.dom = dom; - let theme; - try { - theme = getTheme(); - } catch (err) { - theme = {}; - } - const errorMarkdown = ( + const body = ( ); - const node = this.compact ? ( - {errorMarkdown} - ) : ( + return (
); - const content = - theme && theme.theme$ ? ( - {node} - ) : ( - node - ); - - ReactDOM.render(content, dom); - } - - public destroy() { - if (this.dom) { - ReactDOM.unmountComponentAtNode(this.dom); - } } } - -const CompactEmbeddableError = ({ children }: { children?: React.ReactNode }) => { - const [isPopoverOpen, setPopoverOpen] = useState(false); - - const popoverButton = ( - - setPopoverOpen((open) => !open)} - > - - {i18n.translate('embeddableApi.panel.errorEmbeddable.message', { - defaultMessage: 'An error has occurred. Read more', - })} - - - ); - - return ( - setPopoverOpen(false)} - > - {children} - - ); -}; diff --git a/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts b/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts index 1c9bdebcefc9b..1d3cc7980ad62 100644 --- a/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts +++ b/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts @@ -33,7 +33,8 @@ export interface EmbeddableOutput { export interface IEmbeddable< I extends EmbeddableInput = EmbeddableInput, - O extends EmbeddableOutput = EmbeddableOutput + O extends EmbeddableOutput = EmbeddableOutput, + N = any > { /** * Is this embeddable an instance of a Container class, can it contain @@ -172,15 +173,17 @@ export interface IEmbeddable< /** * Renders the embeddable at the given node. * @param domNode + * @returns A React node to mount or void in the case when rendering is done without React. */ - render(domNode: HTMLElement | Element): void; + render(domNode: HTMLElement | Element): N | void; /** * Renders a custom embeddable error at the given node. + * @param error * @param domNode - * @returns A callback that will be called on error destroy. + * @returns A React node or callback that will be called on error destroy. */ - renderError?(domNode: HTMLElement | Element, error: ErrorLike): () => void; + catchError?(error: EmbeddableError, domNode: HTMLElement | Element): N | (() => void); /** * Reload the embeddable so output and rendering is up to date. Especially relevant diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx index a79f19cb4225c..8f096020ae60e 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx @@ -17,17 +17,17 @@ import { Action, UiActionsStart } from '@kbn/ui-actions-plugin/public'; import { Trigger, ViewMode } from '../types'; import { isErrorEmbeddable } from '../embeddables'; import { EmbeddablePanel } from './embeddable_panel'; -import { createEditModeAction } from '../test_samples/actions'; -import { - ContactCardEmbeddableFactory, - CONTACT_CARD_EMBEDDABLE, -} from '../test_samples/embeddables/contact_card/contact_card_embeddable_factory'; -import { HelloWorldContainer } from '../test_samples/embeddables/hello_world_container'; import { + createEditModeAction, ContactCardEmbeddable, ContactCardEmbeddableInput, ContactCardEmbeddableOutput, -} from '../test_samples/embeddables/contact_card/contact_card_embeddable'; + ContactCardEmbeddableFactory, + ContactCardEmbeddableReactFactory, + CONTACT_CARD_EMBEDDABLE, + CONTACT_CARD_EMBEDDABLE_REACT, + HelloWorldContainer, +} from '../test_samples'; import { inspectorPluginMock } from '@kbn/inspector-plugin/public/mocks'; import { EuiBadge } from '@elastic/eui'; import { embeddablePluginMock } from '../../mocks'; @@ -43,12 +43,17 @@ const trigger: Trigger = { id: CONTEXT_MENU_TRIGGER, }; const embeddableFactory = new ContactCardEmbeddableFactory((() => null) as any, {} as any); +const embeddableReactFactory = new ContactCardEmbeddableReactFactory( + (() => null) as any, + {} as any +); const applicationMock = applicationServiceMock.createStartContract(); const theme = themeServiceMock.createStartContract(); actionRegistry.set(editModeAction.id, editModeAction); triggerRegistry.set(trigger.id, trigger); setup.registerEmbeddableFactory(embeddableFactory.type, embeddableFactory); +setup.registerEmbeddableFactory(embeddableReactFactory.type, embeddableReactFactory); const start = doStart(); const getEmbeddableFactory = start.getEmbeddableFactory; @@ -198,7 +203,7 @@ describe('HelloWorldContainer in error state', () => { ); - jest.spyOn(embeddable, 'renderError'); + jest.spyOn(embeddable, 'catchError'); }); test('renders a custom error', () => { @@ -207,9 +212,9 @@ describe('HelloWorldContainer in error state', () => { const embeddableError = findTestSubject(component, 'embeddableError'); - expect(embeddable.renderError).toHaveBeenCalledWith( - expect.any(HTMLElement), - new Error('something') + expect(embeddable.catchError).toHaveBeenCalledWith( + new Error('something'), + expect.any(HTMLElement) ); expect(embeddableError).toHaveProperty('length', 1); expect(embeddableError.text()).toBe('something'); @@ -222,21 +227,21 @@ describe('HelloWorldContainer in error state', () => { const embeddableError = findTestSubject(component, 'embeddableError'); - expect(embeddable.renderError).toHaveBeenCalledWith( - expect.any(HTMLElement), - new Error('something') + expect(embeddable.catchError).toHaveBeenCalledWith( + new Error('something'), + expect.any(HTMLElement) ); expect(embeddableError).toHaveProperty('length', 1); expect(embeddableError.text()).toBe('something'); }); test('destroys previous error', () => { - const { renderError } = embeddable as Required; - let destroyError: jest.MockedFunction>; + const { catchError } = embeddable as Required; + let destroyError: jest.MockedFunction>; - (embeddable.renderError as jest.MockedFunction).mockImplementationOnce( + (embeddable.catchError as jest.MockedFunction).mockImplementationOnce( (...args) => { - destroyError = jest.fn(renderError(...args)); + destroyError = jest.fn(catchError(...args)); return destroyError; } @@ -254,7 +259,7 @@ describe('HelloWorldContainer in error state', () => { }); test('renders a default error', async () => { - embeddable.renderError = undefined; + embeddable.catchError = undefined; embeddable.triggerError(new Error('something')); component.update(); @@ -263,6 +268,17 @@ describe('HelloWorldContainer in error state', () => { expect(embeddableError).toHaveProperty('length', 1); expect(embeddableError.children.length).toBeGreaterThan(0); }); + + test('renders a React node', () => { + (embeddable.catchError as jest.Mock).mockReturnValueOnce(
Something
); + embeddable.triggerError(new Error('something')); + component.update(); + + const embeddableError = findTestSubject(component, 'embeddableError'); + + expect(embeddableError).toHaveProperty('length', 1); + expect(embeddableError.text()).toBe('Something'); + }); }); const renderInEditModeAndOpenContextMenu = async ( @@ -735,3 +751,37 @@ test('Should work in minimal way rendering only the inspector action', async () const action = findTestSubject(component, `embeddablePanelAction-ACTION_CUSTOMIZE_PANEL`); expect(action.length).toBe(0); }); + +test('Renders an embeddable returning a React node', async () => { + const container = new HelloWorldContainer( + { id: '123', panels: {}, viewMode: ViewMode.VIEW, hidePanelTitles: false }, + { getEmbeddableFactory } as any + ); + + const embeddable = await container.addNewEmbeddable< + ContactCardEmbeddableInput, + ContactCardEmbeddableOutput, + ContactCardEmbeddable + >(CONTACT_CARD_EMBEDDABLE_REACT, { + firstName: 'Bran', + lastName: 'Stark', + }); + + const component = mount( + + Promise.resolve([])} + getAllEmbeddableFactories={start.getEmbeddableFactories} + getEmbeddableFactory={start.getEmbeddableFactory} + notifications={{} as any} + overlays={{} as any} + application={applicationMock} + SavedObjectFinder={() => null} + theme={theme} + /> + + ); + + expect(component.find('.embPanel__titleText').text()).toBe('Hello Bran Stark'); +}); diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx index 52b70f3b53406..f5b072a591225 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx @@ -8,12 +8,14 @@ import { EuiContextMenuPanelDescriptor, EuiPanel, htmlIdGenerator } from '@elastic/eui'; import classNames from 'classnames'; -import React from 'react'; +import React, { ReactNode } from 'react'; import { Subscription } from 'rxjs'; import deepEqual from 'fast-deep-equal'; import { CoreStart, OverlayStart, ThemeServiceStart } from '@kbn/core/public'; import { toMountPoint } from '@kbn/kibana-react-plugin/public'; +import { isPromise } from '@kbn/std'; import { UsageCollectionStart } from '@kbn/usage-collection-plugin/public'; +import { MaybePromise } from '@kbn/utility-types'; import { buildContextMenuForActions, UiActionsService, Action } from '../ui_actions'; import { Start as InspectorStartContract } from '../inspector'; @@ -66,7 +68,7 @@ export interface EmbeddableContainerContext { } interface Props { - embeddable: IEmbeddable; + embeddable: IEmbeddable>; /** * Ordinal number of the embeddable in the container, used as a @@ -105,6 +107,7 @@ interface State { loading?: boolean; error?: EmbeddableError; destroyError?(): void; + node?: ReactNode; } interface InspectorPanelAction { @@ -304,27 +307,37 @@ export class EmbeddablePanel extends React.Component { error={this.state.error} /> )} -
+
+ {this.state.node} +
); } public componentDidMount() { - if (this.embeddableRoot.current) { - this.subscription.add( - this.props.embeddable.getOutput$().subscribe( - (output: EmbeddableOutput) => { - this.setState({ - error: output.error, - loading: output.loading, - }); - }, - (error) => { - this.setState({ error }); - } - ) - ); - this.props.embeddable.render(this.embeddableRoot.current); + if (!this.embeddableRoot.current) { + return; + } + + this.subscription.add( + this.props.embeddable.getOutput$().subscribe( + (output: EmbeddableOutput) => { + this.setState({ + error: output.error, + loading: output.loading, + }); + }, + (error) => { + this.setState({ error }); + } + ) + ); + + const node = this.props.embeddable.render(this.embeddableRoot.current) ?? undefined; + if (isPromise(node)) { + node.then((resolved) => this.setState({ node: resolved })); + } else { + this.setState({ node }); } } diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel_error.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel_error.tsx index 46e26fd1448bb..69af8e7220e62 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel_error.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel_error.tsx @@ -6,17 +6,20 @@ * Side Public License, v 1. */ -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import { isFunction } from 'lodash'; +import React, { ReactNode, useEffect, useMemo, useRef, useState } from 'react'; import { EuiPanel } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { isPromise } from '@kbn/std'; +import type { MaybePromise } from '@kbn/utility-types'; import { ErrorLike } from '@kbn/expressions-plugin/common'; import { distinctUntilChanged, merge, of, switchMap } from 'rxjs'; import { EditPanelAction } from '../actions'; -import { ErrorEmbeddable, IEmbeddable } from '../embeddables'; +import { EmbeddableInput, EmbeddableOutput, ErrorEmbeddable, IEmbeddable } from '../embeddables'; interface EmbeddablePanelErrorProps { editPanelAction?: EditPanelAction; - embeddable: IEmbeddable; + embeddable: IEmbeddable>; error: ErrorLike; } @@ -26,6 +29,7 @@ export function EmbeddablePanelError({ error, }: EmbeddablePanelErrorProps) { const [isEditable, setEditable] = useState(false); + const [node, setNode] = useState(); const ref = useRef(null); const handleErrorClick = useMemo( () => (isEditable ? () => editPanelAction?.execute({ embeddable }) : undefined), @@ -63,14 +67,22 @@ export function EmbeddablePanelError({ return; } - if (embeddable.renderError) { - return embeddable.renderError(ref.current, error); - } + if (!embeddable.catchError) { + const errorEmbeddable = new ErrorEmbeddable(error, { id: embeddable.id }); + setNode(errorEmbeddable.render()); - const errorEmbeddable = new ErrorEmbeddable(error, { id: embeddable.id }); - errorEmbeddable.render(ref.current); + return () => errorEmbeddable.destroy(); + } - return () => errorEmbeddable.destroy(); + const renderedNode = embeddable.catchError(error, ref.current); + if (isFunction(renderedNode)) { + return renderedNode; + } + if (isPromise(renderedNode)) { + renderedNode.then(setNode); + } else { + setNode(renderedNode); + } }, [embeddable, error]); return ( @@ -84,6 +96,8 @@ export function EmbeddablePanelError({ role={isEditable ? 'button' : undefined} aria-label={isEditable ? ariaLabel : undefined} onClick={handleErrorClick} - /> + > + {node} + ); } diff --git a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable.tsx b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable.tsx index 7d9a929299f35..0287b9d115827 100644 --- a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable.tsx +++ b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable.tsx @@ -47,7 +47,7 @@ export class ContactCardEmbeddable extends Embeddable< constructor( initialInput: ContactCardEmbeddableInput, - private readonly options: ContactCardEmbeddableOptions, + protected readonly options: ContactCardEmbeddableOptions, parent?: Container ) { super( @@ -77,7 +77,7 @@ export class ContactCardEmbeddable extends Embeddable< ); } - public renderError?(node: HTMLElement, error: ErrorLike) { + public catchError?(error: ErrorLike, node: HTMLElement) { ReactDom.render(
{error.message}
, node); return () => ReactDom.unmountComponentAtNode(node); diff --git a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_factory.tsx b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_factory.tsx index 282f6a8c627c2..317e0d5e741c8 100644 --- a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_factory.tsx +++ b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_factory.tsx @@ -25,7 +25,7 @@ export class ContactCardEmbeddableFactory public readonly type = CONTACT_CARD_EMBEDDABLE; constructor( - private readonly execTrigger: UiActionsStart['executeTriggerActions'], + protected readonly execTrigger: UiActionsStart['executeTriggerActions'], private readonly overlays: CoreStart['overlays'] ) {} diff --git a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react.tsx b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react.tsx new file mode 100644 index 0000000000000..d42ba42a0cfb3 --- /dev/null +++ b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react.tsx @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React from 'react'; +import { ContactCardEmbeddableComponent } from './contact_card'; +import { ContactCardEmbeddable } from './contact_card_embeddable'; + +export class ContactCardEmbeddableReact extends ContactCardEmbeddable { + public render() { + return ( + + ); + } +} diff --git a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react_factory.ts b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react_factory.ts new file mode 100644 index 0000000000000..7378dc24ea5f8 --- /dev/null +++ b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/contact_card_embeddable_react_factory.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Container } from '../../../containers'; +import { ContactCardEmbeddableInput } from './contact_card_embeddable'; +import { ContactCardEmbeddableFactory } from './contact_card_embeddable_factory'; +import { ContactCardEmbeddableReact } from './contact_card_embeddable_react'; + +export const CONTACT_CARD_EMBEDDABLE_REACT = 'CONTACT_CARD_EMBEDDABLE_REACT'; + +export class ContactCardEmbeddableReactFactory extends ContactCardEmbeddableFactory { + public readonly type = CONTACT_CARD_EMBEDDABLE_REACT as ContactCardEmbeddableFactory['type']; + + public create = async (initialInput: ContactCardEmbeddableInput, parent?: Container) => { + return new ContactCardEmbeddableReact( + initialInput, + { + execAction: this.execTrigger, + }, + parent + ); + }; +} diff --git a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/index.ts b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/index.ts index 526256d375963..fc63fcacbab79 100644 --- a/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/index.ts +++ b/src/plugins/embeddable/public/lib/test_samples/embeddables/contact_card/index.ts @@ -11,5 +11,7 @@ export * from './contact_card_embeddable'; export * from './contact_card_embeddable_factory'; export * from './contact_card_exportable_embeddable'; export * from './contact_card_exportable_embeddable_factory'; +export * from './contact_card_embeddable_react'; +export * from './contact_card_embeddable_react_factory'; export * from './contact_card_initializer'; export * from './slow_contact_card_embeddable_factory'; diff --git a/src/plugins/embeddable/public/tests/fixtures/hello_world_embeddable_react.tsx b/src/plugins/embeddable/public/tests/fixtures/hello_world_embeddable_react.tsx new file mode 100644 index 0000000000000..aa9ac3175fd5e --- /dev/null +++ b/src/plugins/embeddable/public/tests/fixtures/hello_world_embeddable_react.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React from 'react'; +import { HelloWorldEmbeddable } from './hello_world_embeddable'; + +export class HelloWorldEmbeddableReact extends HelloWorldEmbeddable { + public render() { + return
HELLO WORLD!
; + } +} diff --git a/src/plugins/embeddable/public/tests/fixtures/index.ts b/src/plugins/embeddable/public/tests/fixtures/index.ts index a155f65d47858..ea9533a359ca7 100644 --- a/src/plugins/embeddable/public/tests/fixtures/index.ts +++ b/src/plugins/embeddable/public/tests/fixtures/index.ts @@ -8,3 +8,4 @@ export * from './hello_world_embeddable'; export * from './hello_world_embeddable_factory'; +export * from './hello_world_embeddable_react'; diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx b/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx index ef9e8d53a4f11..f2a2a7f8ae000 100644 --- a/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx +++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable.tsx @@ -10,7 +10,7 @@ import _, { get } from 'lodash'; import { Subscription } from 'rxjs'; import { i18n } from '@kbn/i18n'; import React from 'react'; -import { render, unmountComponentAtNode } from 'react-dom'; +import { render } from 'react-dom'; import { EuiLoadingChart } from '@elastic/eui'; import { Filter, onlyDisabledFiltersChanged, Query, TimeRange } from '@kbn/es-query'; import type { KibanaExecutionContext, SavedObjectAttributes } from '@kbn/core/public'; @@ -503,7 +503,7 @@ export class VisualizeEmbeddable const { error } = this.getOutput(); if (error) { - this.renderError(this.domNode, error); + render(this.catchError(error), this.domNode); } }) ); @@ -511,9 +511,9 @@ export class VisualizeEmbeddable await this.updateHandler(); } - public renderError(domNode: HTMLElement, error: ErrorLike | string) { + public catchError(error: ErrorLike | string) { if (isFallbackDataView(this.vis.data.indexPattern)) { - render( + return ( , - domNode + /> ); - } else { - render(, domNode); } - return () => unmountComponentAtNode(domNode); + return ; } public destroy() { diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index bf55cf6174fda..0bf0d3908eeb9 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -372,6 +372,7 @@ "controls.controlGroup.management.validate.title": "Valider les sélections utilisateur", "controls.controlGroup.title": "Groupe de contrôle", "controls.controlGroup.toolbarButtonTitle": "Contrôles", + "controls.frame.error.message": "Une erreur s'est produite. En savoir plus", "controls.optionsList.description": "Ajoutez un menu pour la sélection de valeurs de champ.", "controls.optionsList.displayName": "Liste des options", "controls.optionsList.editor.allowMultiselectTitle": "Permettre des sélections multiples dans une liste déroulante", @@ -2331,7 +2332,6 @@ "embeddableApi.errors.paneldoesNotExist": "Panneau introuvable", "embeddableApi.helloworld.displayName": "bonjour", "embeddableApi.panel.dashboardPanelAriaLabel": "Panneau du tableau de bord", - "embeddableApi.panel.errorEmbeddable.message": "Une erreur s'est produite. En savoir plus", "embeddableApi.panel.inspectPanel.displayName": "Inspecter", "embeddableApi.panel.inspectPanel.untitledEmbeddableFilename": "sans titre", "embeddableApi.panel.optionsMenu.panelOptionsButtonAriaLabel": "Options de panneau", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index e792e7981b0ee..91d6ae6f2bcb5 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -372,6 +372,7 @@ "controls.controlGroup.management.validate.title": "ユーザー選択を検証", "controls.controlGroup.title": "コントロールグループ", "controls.controlGroup.toolbarButtonTitle": "コントロール", + "controls.frame.error.message": "エラーが発生しました。続きを読む", "controls.optionsList.description": "フィールド値を選択するメニューを追加", "controls.optionsList.displayName": "オプションリスト", "controls.optionsList.editor.allowMultiselectTitle": "ドロップダウンでの複数選択を許可", @@ -2327,7 +2328,6 @@ "embeddableApi.errors.paneldoesNotExist": "パネルが見つかりません", "embeddableApi.helloworld.displayName": "こんにちは", "embeddableApi.panel.dashboardPanelAriaLabel": "ダッシュボードパネル", - "embeddableApi.panel.errorEmbeddable.message": "エラーが発生しました。続きを読む", "embeddableApi.panel.inspectPanel.displayName": "検査", "embeddableApi.panel.inspectPanel.untitledEmbeddableFilename": "無題", "embeddableApi.panel.optionsMenu.panelOptionsButtonAriaLabel": "パネルオプション", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 0530b872ef3b7..c929f0f0d80ba 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -372,6 +372,7 @@ "controls.controlGroup.management.validate.title": "验证用户选择", "controls.controlGroup.title": "控件组", "controls.controlGroup.toolbarButtonTitle": "控件", + "controls.frame.error.message": "发生错误。阅读更多内容", "controls.optionsList.description": "添加用于选择字段值的菜单。", "controls.optionsList.displayName": "选项列表", "controls.optionsList.editor.allowMultiselectTitle": "下拉列表中允许多选", @@ -2331,7 +2332,6 @@ "embeddableApi.errors.paneldoesNotExist": "未找到面板", "embeddableApi.helloworld.displayName": "hello world", "embeddableApi.panel.dashboardPanelAriaLabel": "仪表板面板", - "embeddableApi.panel.errorEmbeddable.message": "发生错误。阅读更多内容", "embeddableApi.panel.inspectPanel.displayName": "检查", "embeddableApi.panel.inspectPanel.untitledEmbeddableFilename": "未命名", "embeddableApi.panel.optionsMenu.panelOptionsButtonAriaLabel": "面板选项", From 9d819bac922a48d77f627ab6cba67e9b54ad96b8 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Mon, 24 Oct 2022 18:45:09 +0200 Subject: [PATCH 02/15] [Lens] Random sampling feature (#143221) * :sparkles: First pass with UI for random sampling * :sparkles: Initial working version * :fire: Remove unused stuff * :wrench: Refactor layer settings panel * :bug: Fix terms other query and some refactor * :label: Fix types issues * :bug: Fix sampling for other terms agg * :bug: Fix issue with count operation * :white_check_mark: Fix jest tests * :bug: fix test stability * :bug: fix test with newer params * :lipstick: Add tech preview label * :white_check_mark: Add new tests for sampling * :white_check_mark: Add more tests * :white_check_mark: Add new test for suggestions * :white_check_mark: Add functional tests for layer actions and random sampling * Update x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx Co-authored-by: Michael Marcialis * :ok_hand: Integrated design feedback Co-authored-by: Joe Reuter Co-authored-by: Michael Marcialis --- .../common/search/aggs/agg_configs.test.ts | 155 +++++ .../data/common/search/aggs/agg_configs.ts | 36 +- .../_terms_other_bucket_helper.test.ts | 584 ++++++++++-------- .../buckets/_terms_other_bucket_helper.ts | 61 +- .../data/common/search/aggs/utils/sampler.ts | 29 + .../common/search/aggs/utils/time_splits.ts | 8 +- .../search/expressions/esaggs/esaggs_fn.ts | 17 + .../data/common/search/tabify/tabify.test.ts | 331 +++++----- .../data/common/search/tabify/tabify.ts | 4 +- .../public/search/expressions/esaggs.test.ts | 5 +- .../data/public/search/expressions/esaggs.ts | 7 +- .../server/search/expressions/esaggs.test.ts | 1 + .../datasources/form_based/form_based.test.ts | 121 +++- .../datasources/form_based/form_based.tsx | 49 +- .../form_based_suggestions.test.tsx | 32 +- .../form_based/form_based_suggestions.ts | 6 + .../datasources/form_based/layer_settings.tsx | 75 +++ .../datasources/form_based/to_expression.ts | 12 +- .../public/datasources/form_based/types.ts | 1 + .../config_panel/dimension_container.scss | 49 -- .../config_panel/dimension_container.tsx | 142 +---- .../config_panel/flyout_container.scss | 48 ++ .../config_panel/flyout_container.tsx | 157 +++++ .../layer_actions/layer_actions.tsx | 3 + .../editor_frame/config_panel/layer_panel.tsx | 54 +- .../editor_frame/expression_helpers.ts | 10 +- .../workspace_panel/workspace_panel.tsx | 11 +- x-pack/plugins/lens/public/types.ts | 33 +- .../test/functional/apps/lens/group1/index.ts | 1 + .../apps/lens/group1/layer_actions.ts | 91 +++ .../test/functional/page_objects/lens_page.ts | 17 + 31 files changed, 1494 insertions(+), 656 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/sampler.ts create mode 100644 x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx create mode 100644 x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss create mode 100644 x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx create mode 100644 x-pack/test/functional/apps/lens/group1/layer_actions.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index cd0495a3f78c6..c3b7ffe937b35 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -16,6 +16,15 @@ import type { DataView } from '@kbn/data-views-plugin/common'; import { stubIndexPattern } from '../../stubs'; import { IEsSearchResponse } from '..'; +// Mute moment.tz warnings about not finding a mock timezone +jest.mock('../utils', () => { + const original = jest.requireActual('../utils'); + return { + ...original, + getUserTimeZone: jest.fn(() => 'US/Pacific'), + }; +}); + describe('AggConfigs', () => { const indexPattern: DataView = stubIndexPattern; let typesRegistry: AggTypesRegistryStart; @@ -563,6 +572,82 @@ describe('AggConfigs', () => { '1-bucket>_count' ); }); + + it('prepends a sampling agg whenever sampling is enabled', () => { + const configStates = [ + { + enabled: true, + id: '1', + type: 'avg_bucket', + schema: 'metric', + params: { + customBucket: { + id: '1-bucket', + type: 'date_histogram', + schema: 'bucketAgg', + params: { + field: '@timestamp', + interval: '10s', + }, + }, + customMetric: { + id: '1-metric', + type: 'count', + schema: 'metricAgg', + params: {}, + }, + }, + }, + { + enabled: true, + id: '2', + type: 'terms', + schema: 'bucket', + params: { + field: 'clientip', + }, + }, + { + enabled: true, + id: '3', + type: 'terms', + schema: 'bucket', + params: { + field: 'machine.os.raw', + }, + }, + ]; + + const ac = new AggConfigs( + indexPattern, + configStates, + { typesRegistry, hierarchical: true, probability: 0.5 }, + jest.fn() + ); + const topLevelDsl = ac.toDsl(); + + expect(Object.keys(topLevelDsl)).toContain('sampling'); + expect(Object.keys(topLevelDsl.sampling)).toEqual(['random_sampler', 'aggs']); + expect(Object.keys(topLevelDsl.sampling.aggs)).toContain('2'); + expect(Object.keys(topLevelDsl.sampling.aggs['2'].aggs)).toEqual(['1', '3', '1-bucket']); + }); + + it('should not prepend a sampling agg when no nested agg is avaialble', () => { + const ac = new AggConfigs( + indexPattern, + [ + { + enabled: true, + type: 'count', + schema: 'metric', + }, + ], + { typesRegistry, probability: 0.5 }, + jest.fn() + ); + const topLevelDsl = ac.toDsl(); + expect(Object.keys(topLevelDsl)).not.toContain('sampling'); + }); }); describe('#postFlightTransform', () => { @@ -854,4 +939,74 @@ describe('AggConfigs', () => { `); }); }); + + describe('isSamplingEnabled', () => { + it('should return false if probability is 1', () => { + const ac = new AggConfigs( + indexPattern, + [{ enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }], + { typesRegistry, probability: 1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeFalsy(); + }); + + it('should return true if probability is less than 1', () => { + const ac = new AggConfigs( + indexPattern, + [{ enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }], + { typesRegistry, probability: 0.1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeTruthy(); + }); + + it('should return false when all aggs have hasNoDsl flag enabled', () => { + const ac = new AggConfigs( + indexPattern, + [ + { + enabled: true, + type: 'count', + schema: 'metric', + }, + ], + { typesRegistry, probability: 1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeFalsy(); + }); + + it('should return false when no nested aggs are avaialble', () => { + const ac = new AggConfigs( + indexPattern, + [{ enabled: false, type: 'avg', schema: 'metric', params: { field: 'bytes' } }], + { typesRegistry, probability: 1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeFalsy(); + }); + + it('should return true if at least one nested agg is available and probability < 1', () => { + const ac = new AggConfigs( + indexPattern, + [ + { + enabled: true, + type: 'count', + schema: 'metric', + }, + { enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }, + ], + { typesRegistry, probability: 0.1 }, + jest.fn() + ); + + expect(ac.isSamplingEnabled()).toBeTruthy(); + }); + }); }); diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index c61ca69e0c6df..7cab863fba11d 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -26,6 +26,7 @@ import { AggTypesDependencies, GetConfigFn, getUserTimeZone } from '../..'; import { getTime, calculateBounds } from '../..'; import type { IBucketAggConfig } from './buckets'; import { insertTimeShiftSplit, mergeTimeShifts } from './utils/time_splits'; +import { createSamplerAgg, isSamplingEnabled } from './utils/sampler'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -55,6 +56,8 @@ export interface AggConfigsOptions { hierarchical?: boolean; aggExecutionContext?: AggTypesDependencies['aggExecutionContext']; partialRows?: boolean; + probability?: number; + samplerSeed?: number; } export type CreateAggConfigParams = Assign; @@ -107,6 +110,17 @@ export class AggConfigs { return this.opts.partialRows ?? false; } + public get samplerConfig() { + return { probability: this.opts.probability ?? 1, seed: this.opts.samplerSeed }; + } + + isSamplingEnabled() { + return ( + isSamplingEnabled(this.opts.probability) && + this.getRequestAggs().filter((agg) => !agg.type.hasNoDsl).length > 0 + ); + } + setTimeFields(timeFields: string[] | undefined) { this.timeFields = timeFields; } @@ -225,7 +239,7 @@ export class AggConfigs { } toDsl(): Record { - const dslTopLvl = {}; + const dslTopLvl: Record = {}; let dslLvlCursor: Record; let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; @@ -254,10 +268,21 @@ export class AggConfigs { (config) => 'splitForTimeShift' in config.type && config.type.splitForTimeShift(config, this) ); + if (this.isSamplingEnabled()) { + dslTopLvl.sampling = createSamplerAgg({ + probability: this.opts.probability ?? 1, + seed: this.opts.samplerSeed, + }); + } + requestAggs.forEach((config: AggConfig, i: number, list) => { if (!dslLvlCursor) { // start at the top level dslLvlCursor = dslTopLvl; + // when sampling jump directly to the aggs + if (this.isSamplingEnabled()) { + dslLvlCursor = dslLvlCursor.sampling.aggs; + } } else { const prevConfig: AggConfig = list[i - 1]; const prevDsl = dslLvlCursor[prevConfig.id]; @@ -452,7 +477,12 @@ export class AggConfigs { doc_count: response.rawResponse.hits?.total as estypes.AggregationsAggregate, }; } - const aggCursor = transformedRawResponse.aggregations!; + const aggCursor = this.isSamplingEnabled() + ? (transformedRawResponse.aggregations!.sampling! as Record< + string, + estypes.AggregationsAggregate + >) + : transformedRawResponse.aggregations!; mergeTimeShifts(this, aggCursor); return { @@ -531,6 +561,8 @@ export class AggConfigs { metricsAtAllLevels: this.hierarchical, partialRows: this.partialRows, aggs: this.aggs.map((agg) => buildExpression(agg.toExpressionAst())), + probability: this.opts.probability, + samplerSeed: this.opts.samplerSeed, }), ]).toAst(); } diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 6ce588b98fa9c..be2e969f279b1 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -18,6 +18,8 @@ import { AggConfigs, CreateAggConfigParams } from '../agg_configs'; import { BUCKET_TYPES } from './bucket_agg_types'; import { IBucketAggConfig } from './bucket_agg_type'; import { mockAggTypesRegistry } from '../test_helpers'; +import { estypes } from '@elastic/elasticsearch'; +import { isSamplingEnabled } from '../utils/sampler'; const indexPattern = { id: '1234', @@ -281,71 +283,63 @@ const nestedOtherResponse = { describe('Terms Agg Other bucket helper', () => { const typesRegistry = mockAggTypesRegistry(); - const getAggConfigs = (aggs: CreateAggConfigParams[] = []) => { - return new AggConfigs(indexPattern, [...aggs], { typesRegistry }, jest.fn()); - }; - - describe('buildOtherBucketAgg', () => { - test('returns a function', () => { - const aggConfigs = getAggConfigs(singleTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig, - singleTermResponse - ); - expect(typeof agg).toBe('function'); - }); - - test('correctly builds query with single terms agg', () => { - const aggConfigs = getAggConfigs(singleTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig, - singleTermResponse - ); - const expectedResponse = { - aggs: undefined, - filters: { - filters: { - '': { - bool: { - must: [], - filter: [{ exists: { field: 'machine.os.raw' } }], - should: [], - must_not: [ - { match_phrase: { 'machine.os.raw': 'ios' } }, - { match_phrase: { 'machine.os.raw': 'win xp' } }, - ], - }, - }, + for (const probability of [1, 0.5, undefined]) { + function getTitlePostfix() { + if (!isSamplingEnabled(probability)) { + return ''; + } + return ` - with sampling (probability = ${probability})`; + } + function enrichResponseWithSampling(response: any) { + if (!isSamplingEnabled(probability)) { + return response; + } + return { + ...response, + aggregations: { + sampling: { + ...response.aggregations, }, }, }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()['other-filter']).toEqual(expectedResponse); - } - }); + } - test('correctly builds query for nested terms agg', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponse - ); - const expectedResponse = { - 'other-filter': { + function getAggConfigs(aggs: CreateAggConfigParams[] = []) { + return new AggConfigs(indexPattern, [...aggs], { typesRegistry, probability }, jest.fn()); + } + + function getTopAggregations(updatedResponse: estypes.SearchResponse) { + return !isSamplingEnabled(probability) + ? updatedResponse.aggregations! + : (updatedResponse.aggregations!.sampling as Record); + } + + describe(`buildOtherBucketAgg${getTitlePostfix()}`, () => { + test('returns a function', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + enrichResponseWithSampling(singleTermResponse) + ); + expect(typeof agg).toBe('function'); + }); + + test('correctly builds query with single terms agg', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + enrichResponseWithSampling(singleTermResponse) + ); + const expectedResponse = { aggs: undefined, filters: { filters: { - [`${SEP}IN-with-dash`]: { + '': { bool: { must: [], - filter: [ - { match_phrase: { 'geo.src': 'IN-with-dash' } }, - { exists: { field: 'machine.os.raw' } }, - ], + filter: [{ exists: { field: 'machine.os.raw' } }], should: [], must_not: [ { match_phrase: { 'machine.os.raw': 'ios' } }, @@ -353,272 +347,322 @@ describe('Terms Agg Other bucket helper', () => { ], }, }, - [`${SEP}US-with-dash`]: { - bool: { - must: [], - filter: [ - { match_phrase: { 'geo.src': 'US-with-dash' } }, - { exists: { field: 'machine.os.raw' } }, - ], - should: [], - must_not: [ - { match_phrase: { 'machine.os.raw': 'ios' } }, - { match_phrase: { 'machine.os.raw': 'win xp' } }, - ], + }, + }, + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + expect(topAgg['other-filter']).toEqual(expectedResponse); + } + }); + + test('correctly builds query for nested terms agg', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(nestedTermResponse) + ); + const expectedResponse = { + 'other-filter': { + aggs: undefined, + filters: { + filters: { + [`${SEP}IN-with-dash`]: { + bool: { + must: [], + filter: [ + { match_phrase: { 'geo.src': 'IN-with-dash' } }, + { exists: { field: 'machine.os.raw' } }, + ], + should: [], + must_not: [ + { match_phrase: { 'machine.os.raw': 'ios' } }, + { match_phrase: { 'machine.os.raw': 'win xp' } }, + ], + }, + }, + [`${SEP}US-with-dash`]: { + bool: { + must: [], + filter: [ + { match_phrase: { 'geo.src': 'US-with-dash' } }, + { exists: { field: 'machine.os.raw' } }, + ], + should: [], + must_not: [ + { match_phrase: { 'machine.os.raw': 'ios' } }, + { match_phrase: { 'machine.os.raw': 'win xp' } }, + ], + }, }, }, }, }, - }, - }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()).toEqual(expectedResponse); - } - }); + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + // console.log({ probability }, JSON.stringify(topAgg, null, 2)); + expect(topAgg).toEqual(expectedResponse); + } + }); - test('correctly builds query for nested terms agg with one disabled', () => { - const oneDisabledNestedTerms = { - aggs: [ - { - id: '2', - type: BUCKET_TYPES.TERMS, - enabled: false, - params: { - field: { - name: 'machine.os.raw', - indexPattern, - filterable: true, + test('correctly builds query for nested terms agg with one disabled', () => { + const oneDisabledNestedTerms = { + aggs: [ + { + id: '2', + type: BUCKET_TYPES.TERMS, + enabled: false, + params: { + field: { + name: 'machine.os.raw', + indexPattern, + filterable: true, + }, + size: 2, + otherBucket: false, + missingBucket: true, }, - size: 2, - otherBucket: false, - missingBucket: true, }, - }, - { - id: '1', - type: BUCKET_TYPES.TERMS, - params: { - field: { - name: 'geo.src', - indexPattern, - filterable: true, + { + id: '1', + type: BUCKET_TYPES.TERMS, + params: { + field: { + name: 'geo.src', + indexPattern, + filterable: true, + }, + size: 2, + otherBucket: true, + missingBucket: false, }, - size: 2, - otherBucket: true, - missingBucket: false, }, - }, - ], - }; - const aggConfigs = getAggConfigs(oneDisabledNestedTerms.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - singleTermResponse - ); - const expectedResponse = { - 'other-filter': { - aggs: undefined, - filters: { + ], + }; + const aggConfigs = getAggConfigs(oneDisabledNestedTerms.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(singleTermResponse) + ); + const expectedResponse = { + 'other-filter': { + aggs: undefined, filters: { - '': { - bool: { - filter: [ - { - exists: { - field: 'geo.src', + filters: { + '': { + bool: { + filter: [ + { + exists: { + field: 'geo.src', + }, }, - }, - ], - must: [], - must_not: [ - { - match_phrase: { - 'geo.src': 'ios', + ], + must: [], + must_not: [ + { + match_phrase: { + 'geo.src': 'ios', + }, }, - }, - { - match_phrase: { - 'geo.src': 'win xp', + { + match_phrase: { + 'geo.src': 'win xp', + }, }, - }, - ], - should: [], + ], + should: [], + }, }, }, }, }, - }, - }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()).toEqual(expectedResponse); - } - }); + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + expect(topAgg).toEqual(expectedResponse); + } + }); - test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - expect( - buildOtherBucketAgg( + test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + expect( + buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(exhaustiveNestedTermResponse) + ) + ).toBeFalsy(); + }); + + test('excludes exists filter for scripted fields', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + aggConfigs.aggs[1].params.field = { + ...aggConfigs.aggs[1].params.field, + scripted: true, + }; + const agg = buildOtherBucketAgg( aggConfigs, aggConfigs.aggs[1] as IBucketAggConfig, - exhaustiveNestedTermResponse - ) - ).toBeFalsy(); - }); - - test('excludes exists filter for scripted fields', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - aggConfigs.aggs[1].params.field.scripted = true; - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponse - ); - const expectedResponse = { - 'other-filter': { - aggs: undefined, - filters: { + enrichResponseWithSampling(nestedTermResponse) + ); + const expectedResponse = { + 'other-filter': { + aggs: undefined, filters: { - [`${SEP}IN-with-dash`]: { - bool: { - must: [], - filter: [{ match_phrase: { 'geo.src': 'IN-with-dash' } }], - should: [], - must_not: [ - { - script: { + filters: { + [`${SEP}IN-with-dash`]: { + bool: { + must: [], + filter: [{ match_phrase: { 'geo.src': 'IN-with-dash' } }], + should: [], + must_not: [ + { script: { - lang: undefined, - params: { value: 'ios' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'ios' }, + source: '(undefined) == value', + }, }, }, - }, - { - script: { + { script: { - lang: undefined, - params: { value: 'win xp' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'win xp' }, + source: '(undefined) == value', + }, }, }, - }, - ], + ], + }, }, - }, - [`${SEP}US-with-dash`]: { - bool: { - must: [], - filter: [{ match_phrase: { 'geo.src': 'US-with-dash' } }], - should: [], - must_not: [ - { - script: { + [`${SEP}US-with-dash`]: { + bool: { + must: [], + filter: [{ match_phrase: { 'geo.src': 'US-with-dash' } }], + should: [], + must_not: [ + { script: { - lang: undefined, - params: { value: 'ios' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'ios' }, + source: '(undefined) == value', + }, }, }, - }, - { - script: { + { script: { - lang: undefined, - params: { value: 'win xp' }, - source: '(undefined) == value', + script: { + lang: undefined, + params: { value: 'win xp' }, + source: '(undefined) == value', + }, }, }, - }, - ], + ], + }, }, }, }, }, - }, - }; - expect(agg).toBeDefined(); - if (agg) { - expect(agg()).toEqual(expectedResponse); - } - }); + }; + expect(agg).toBeDefined(); + if (agg) { + const resp = agg(); + const topAgg = !isSamplingEnabled(probability) ? resp : resp.sampling!.aggs; + expect(topAgg).toEqual(expectedResponse); + } + }); - test('returns false when nested terms agg has no buckets', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const agg = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponseNoResults - ); + test('returns false when nested terms agg has no buckets', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + enrichResponseWithSampling(nestedTermResponseNoResults) + ); - expect(agg).toEqual(false); + expect(agg).toEqual(false); + }); }); - }); - describe('mergeOtherBucketAggResponse', () => { - test('correctly merges other bucket with single terms agg', () => { - const aggConfigs = getAggConfigs(singleTerm.aggs); - const otherAggConfig = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig, - singleTermResponse - ); - - expect(otherAggConfig).toBeDefined(); - if (otherAggConfig) { - const mergedResponse = mergeOtherBucketAggResponse( + describe(`mergeOtherBucketAggResponse${getTitlePostfix()}`, () => { + test('correctly merges other bucket with single terms agg', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const otherAggConfig = buildOtherBucketAgg( aggConfigs, - singleTermResponse, - singleOtherResponse, aggConfigs.aggs[0] as IBucketAggConfig, - otherAggConfig(), - constructSingleTermOtherFilter + enrichResponseWithSampling(singleTermResponse) ); - expect((mergedResponse!.aggregations!['1'] as any).buckets[3].key).toEqual('__other__'); - } - }); - test('correctly merges other bucket with nested terms agg', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const otherAggConfig = buildOtherBucketAgg( - aggConfigs, - aggConfigs.aggs[1] as IBucketAggConfig, - nestedTermResponse - ); + expect(otherAggConfig).toBeDefined(); + if (otherAggConfig) { + const mergedResponse = mergeOtherBucketAggResponse( + aggConfigs, + enrichResponseWithSampling(singleTermResponse), + enrichResponseWithSampling(singleOtherResponse), + aggConfigs.aggs[0] as IBucketAggConfig, + otherAggConfig(), + constructSingleTermOtherFilter + ); - expect(otherAggConfig).toBeDefined(); - if (otherAggConfig) { - const mergedResponse = mergeOtherBucketAggResponse( + const topAgg = getTopAggregations(mergedResponse); + expect((topAgg['1'] as any).buckets[3].key).toEqual('__other__'); + } + }); + + test('correctly merges other bucket with nested terms agg', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const otherAggConfig = buildOtherBucketAgg( aggConfigs, - nestedTermResponse, - nestedOtherResponse, aggConfigs.aggs[1] as IBucketAggConfig, - otherAggConfig(), - constructSingleTermOtherFilter + enrichResponseWithSampling(nestedTermResponse) ); - expect((mergedResponse!.aggregations!['1'] as any).buckets[1]['2'].buckets[3].key).toEqual( - '__other__' - ); - } + expect(otherAggConfig).toBeDefined(); + if (otherAggConfig) { + const mergedResponse = mergeOtherBucketAggResponse( + aggConfigs, + enrichResponseWithSampling(nestedTermResponse), + enrichResponseWithSampling(nestedOtherResponse), + aggConfigs.aggs[1] as IBucketAggConfig, + otherAggConfig(), + constructSingleTermOtherFilter + ); + + const topAgg = getTopAggregations(mergedResponse); + expect((topAgg['1'] as any).buckets[1]['2'].buckets[3].key).toEqual('__other__'); + } + }); }); - }); - describe('updateMissingBucket', () => { - test('correctly updates missing bucket key', () => { - const aggConfigs = getAggConfigs(nestedTerm.aggs); - const updatedResponse = updateMissingBucket( - singleTermResponse, - aggConfigs, - aggConfigs.aggs[0] as IBucketAggConfig - ); - expect( - (updatedResponse!.aggregations!['1'] as any).buckets.find( - (bucket: Record) => bucket.key === '__missing__' - ) - ).toBeDefined(); + describe(`updateMissingBucket${getTitlePostfix()}`, () => { + test('correctly updates missing bucket key', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const updatedResponse = updateMissingBucket( + enrichResponseWithSampling(singleTermResponse), + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig + ); + const topAgg = getTopAggregations(updatedResponse); + expect( + (topAgg['1'] as any).buckets.find( + (bucket: Record) => bucket.key === '__missing__' + ) + ).toBeDefined(); + }); }); - }); + } }); diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index f695dc1b1d399..68c64f67ef27f 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -20,6 +20,7 @@ import { AggGroupNames } from '../agg_groups'; import { IAggConfigs } from '../agg_configs'; import { IAggType } from '../agg_type'; import { IAggConfig } from '../agg_config'; +import { createSamplerAgg } from '../utils/sampler'; export const OTHER_BUCKET_SEPARATOR = '╰┄►'; @@ -128,6 +129,28 @@ const getOtherAggTerms = (requestAgg: Record, key: string, otherAgg .map((filter: Record) => filter.match_phrase[otherAgg.params.field.name]); }; +/** + * Helper function to handle sampling case and get the correct cursor agg from a request object + */ +const getCorrectAggCursorFromRequest = ( + requestAgg: Record, + aggConfigs: IAggConfigs +) => { + return aggConfigs.isSamplingEnabled() ? requestAgg.sampling.aggs : requestAgg; +}; + +/** + * Helper function to handle sampling case and get the correct cursor agg from a response object + */ +const getCorrectAggregationsCursorFromResponse = ( + response: estypes.SearchResponse, + aggConfigs: IAggConfigs +) => { + return aggConfigs.isSamplingEnabled() + ? (response.aggregations?.sampling as Record) + : response.aggregations; +}; + export const buildOtherBucketAgg = ( aggConfigs: IAggConfigs, aggWithOtherBucket: IAggConfig, @@ -234,7 +257,13 @@ export const buildOtherBucketAgg = ( bool: buildQueryFromFilters(filters, indexPattern), }; }; - walkBucketTree(0, response.aggregations, bucketAggs[0].id, [], ''); + walkBucketTree( + 0, + getCorrectAggregationsCursorFromResponse(response, aggConfigs), + bucketAggs[0].id, + [], + '' + ); // bail if there were no bucket results if (noAggBucketResults || exhaustiveBuckets) { @@ -242,6 +271,14 @@ export const buildOtherBucketAgg = ( } return () => { + if (aggConfigs.isSamplingEnabled()) { + return { + sampling: { + ...createSamplerAgg(aggConfigs.samplerConfig), + aggs: { 'other-filter': resultAgg }, + }, + }; + } return { 'other-filter': resultAgg, }; @@ -257,16 +294,27 @@ export const mergeOtherBucketAggResponse = ( otherFilterBuilder: (requestAgg: Record, key: string, otherAgg: IAggConfig) => Filter ): estypes.SearchResponse => { const updatedResponse = cloneDeep(response); - each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => { + const aggregationsRoot = getCorrectAggregationsCursorFromResponse(otherResponse, aggsConfig); + const updatedAggregationsRoot = getCorrectAggregationsCursorFromResponse( + updatedResponse, + aggsConfig + ); + const buckets = + 'buckets' in aggregationsRoot!['other-filter'] ? aggregationsRoot!['other-filter'].buckets : {}; + each(buckets, (bucket, key) => { if (!bucket.doc_count || key === undefined) return; const bucketKey = key.replace(new RegExp(`^${OTHER_BUCKET_SEPARATOR}`), ''); const aggResultBuckets = getAggResultBuckets( aggsConfig, - updatedResponse.aggregations, + updatedAggregationsRoot, otherAgg, bucketKey ); - const otherFilter = otherFilterBuilder(requestAgg, key, otherAgg); + const otherFilter = otherFilterBuilder( + getCorrectAggCursorFromRequest(requestAgg, aggsConfig), + key, + otherAgg + ); bucket.filters = [otherFilter]; bucket.key = '__other__'; @@ -290,7 +338,10 @@ export const updateMissingBucket = ( agg: IAggConfig ) => { const updatedResponse = cloneDeep(response); - const aggResultBuckets = getAggConfigResultMissingBuckets(updatedResponse.aggregations, agg.id); + const aggResultBuckets = getAggConfigResultMissingBuckets( + getCorrectAggregationsCursorFromResponse(updatedResponse, aggConfigs), + agg.id + ); aggResultBuckets.forEach((bucket) => { bucket.key = '__missing__'; }); diff --git a/src/plugins/data/common/search/aggs/utils/sampler.ts b/src/plugins/data/common/search/aggs/utils/sampler.ts new file mode 100644 index 0000000000000..5a6fde63b0a29 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/sampler.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export function createSamplerAgg({ + type = 'random_sampler', + probability, + seed, +}: { + type?: string; + probability: number; + seed?: number; +}) { + return { + [type]: { + probability, + seed, + }, + aggs: {}, + }; +} + +export function isSamplingEnabled(probability: number | undefined) { + return probability != null && probability !== 1; +} diff --git a/src/plugins/data/common/search/aggs/utils/time_splits.ts b/src/plugins/data/common/search/aggs/utils/time_splits.ts index c2fe8aaca0fb2..b1262683446f3 100644 --- a/src/plugins/data/common/search/aggs/utils/time_splits.ts +++ b/src/plugins/data/common/search/aggs/utils/time_splits.ts @@ -427,11 +427,11 @@ export function insertTimeShiftSplit( const timeRange = aggConfigs.timeRange; const filters: Record = {}; const timeField = aggConfigs.timeFields[0]; + const timeFilter = getTime(aggConfigs.indexPattern, timeRange, { + fieldName: timeField, + forceNow: aggConfigs.forceNow, + }) as RangeFilter; Object.entries(timeShifts).forEach(([key, shift]) => { - const timeFilter = getTime(aggConfigs.indexPattern, timeRange, { - fieldName: timeField, - forceNow: aggConfigs.forceNow, - }) as RangeFilter; if (timeFilter) { filters[key] = { range: { diff --git a/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts b/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts index c8296b3c06557..f0b55de8ffeff 100644 --- a/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts +++ b/src/plugins/data/common/search/expressions/esaggs/esaggs_fn.ts @@ -32,6 +32,8 @@ interface Arguments { metricsAtAllLevels?: boolean; partialRows?: boolean; timeFields?: string[]; + probability?: number; + samplerSeed?: number; } export type EsaggsExpressionFunctionDefinition = ExpressionFunctionDefinition< @@ -94,6 +96,21 @@ export const getEsaggsMeta: () => Omit defaultMessage: 'Provide time fields to get the resolved time ranges for the query', }), }, + probability: { + types: ['number'], + default: 1, + help: i18n.translate('data.search.functions.esaggs.probability.help', { + defaultMessage: + 'The probability that a document will be included in the aggregated data. Uses random sampler.', + }), + }, + samplerSeed: { + types: ['number'], + help: i18n.translate('data.search.functions.esaggs.samplerSeed.help', { + defaultMessage: + 'The seed to generate the random sampling of documents. Uses random sampler.', + }), + }, }, }); diff --git a/src/plugins/data/common/search/tabify/tabify.test.ts b/src/plugins/data/common/search/tabify/tabify.test.ts index d7d983fabfdaf..90ef53623c298 100644 --- a/src/plugins/data/common/search/tabify/tabify.test.ts +++ b/src/plugins/data/common/search/tabify/tabify.test.ts @@ -11,184 +11,217 @@ import type { DataView } from '@kbn/data-views-plugin/common'; import { AggConfigs, BucketAggParam, IAggConfig, IAggConfigs } from '../aggs'; import { mockAggTypesRegistry } from '../aggs/test_helpers'; import { metricOnly, threeTermBuckets } from './fixtures/fake_hierarchical_data'; +import { isSamplingEnabled } from '../aggs/utils/sampler'; describe('tabifyAggResponse Integration', () => { const typesRegistry = mockAggTypesRegistry(); - const createAggConfigs = (aggs: IAggConfig[] = []) => { - const field = { - name: '@timestamp', - }; - - const indexPattern = { - id: '1234', - title: 'logstash-*', - fields: { - getByName: () => field, - filter: () => [field], - }, - getFormatterForField: () => ({ - toJSON: () => '{}', - }), - } as unknown as DataView; - - return new AggConfigs(indexPattern, aggs, { typesRegistry }, jest.fn()); - }; - - const mockAggConfig = (agg: any): IAggConfig => agg as unknown as IAggConfig; - - test('transforms a simple response properly', () => { - const aggConfigs = createAggConfigs([{ type: 'count' } as any]); - - const resp = tabifyAggResponse(aggConfigs, metricOnly, { - metricsAtAllLevels: true, - }); - - expect(resp).toHaveProperty('rows'); - expect(resp).toHaveProperty('columns'); + for (const probability of [1, 0.5, undefined]) { + function getTitlePostfix() { + if (!isSamplingEnabled(probability)) { + return ''; + } + return ` - with sampling (probability = ${probability})`; + } - expect(resp.rows).toHaveLength(1); - expect(resp.columns).toHaveLength(1); + function enrichResponseWithSampling(response: any) { + if (!isSamplingEnabled(probability)) { + return response; + } + return { + ...response, + aggregations: { + sampling: { + ...response.aggregations, + }, + }, + }; + } - expect(resp.rows[0]).toEqual({ 'col-0-1': 1000 }); - expect(resp.columns[0]).toHaveProperty('name', aggConfigs.aggs[0].makeLabel()); + const createAggConfigs = (aggs: IAggConfig[] = []) => { + const field = { + name: '@timestamp', + }; + + const indexPattern = { + id: '1234', + title: 'logstash-*', + fields: { + getByName: () => field, + filter: () => [field], + }, + getFormatterForField: () => ({ + toJSON: () => '{}', + }), + } as unknown as DataView; + + return new AggConfigs(indexPattern, aggs, { typesRegistry, probability }, jest.fn()); + }; - expect(resp).toHaveProperty('meta.type', 'esaggs'); - expect(resp).toHaveProperty('meta.source', '1234'); - expect(resp).toHaveProperty('meta.statistics.totalCount', 1000); - }); + const mockAggConfig = (agg: any): IAggConfig => agg as unknown as IAggConfig; - describe('scaleMetricValues performance check', () => { - beforeAll(() => { - typesRegistry.get('count').params.push({ - name: 'scaleMetricValues', - default: false, - write: () => {}, - advanced: true, - } as any as BucketAggParam); - }); - test('does not call write if scaleMetricValues is not set', () => { + test(`transforms a simple response properly${getTitlePostfix()}`, () => { const aggConfigs = createAggConfigs([{ type: 'count' } as any]); - const writeMock = jest.fn(); - aggConfigs.getRequestAggs()[0].write = writeMock; - - tabifyAggResponse(aggConfigs, metricOnly, { + const resp = tabifyAggResponse(aggConfigs, enrichResponseWithSampling(metricOnly), { metricsAtAllLevels: true, }); - expect(writeMock).not.toHaveBeenCalled(); - }); - test('does call write if scaleMetricValues is set', () => { - const aggConfigs = createAggConfigs([ - { type: 'count', params: { scaleMetricValues: true } } as any, - ]); + expect(resp).toHaveProperty('rows'); + expect(resp).toHaveProperty('columns'); - const writeMock = jest.fn(() => ({})); - aggConfigs.getRequestAggs()[0].write = writeMock; - - tabifyAggResponse(aggConfigs, metricOnly, { - metricsAtAllLevels: true, - }); - expect(writeMock).toHaveBeenCalled(); - }); - }); - - describe('transforms a complex response', () => { - let esResp: typeof threeTermBuckets; - let aggConfigs: IAggConfigs; - let avg: IAggConfig; - let ext: IAggConfig; - let src: IAggConfig; - let os: IAggConfig; - - beforeEach(() => { - aggConfigs = createAggConfigs([ - mockAggConfig({ type: 'avg', schema: 'metric', params: { field: '@timestamp' } }), - mockAggConfig({ type: 'terms', schema: 'split', params: { field: '@timestamp' } }), - mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), - mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), - ]); - - [avg, ext, src, os] = aggConfigs.aggs; - - esResp = threeTermBuckets; - esResp.aggregations.agg_2.buckets[1].agg_3.buckets[0].agg_4.buckets = []; - }); + expect(resp.rows).toHaveLength(1); + expect(resp.columns).toHaveLength(1); - // check that the columns of a table are formed properly - function expectColumns(table: ReturnType, aggs: IAggConfig[]) { - expect(table.columns).toHaveLength(aggs.length); + expect(resp.rows[0]).toEqual({ 'col-0-1': 1000 }); + expect(resp.columns[0]).toHaveProperty('name', aggConfigs.aggs[0].makeLabel()); - aggs.forEach((agg, i) => { - expect(table.columns[i]).toHaveProperty('name', agg.makeLabel()); - }); - } + expect(resp).toHaveProperty('meta.type', 'esaggs'); + expect(resp).toHaveProperty('meta.source', '1234'); + expect(resp).toHaveProperty('meta.statistics.totalCount', 1000); + }); - // check that a row has expected values - function expectRow( - row: Record, - asserts: Array<(val: string | number) => void> - ) { - expect(typeof row).toBe('object'); - - asserts.forEach((assert, i: number) => { - if (row[`col-${i}`]) { - assert(row[`col-${i}`]); - } + describe(`scaleMetricValues performance check${getTitlePostfix()}`, () => { + beforeAll(() => { + typesRegistry.get('count').params.push({ + name: 'scaleMetricValues', + default: false, + write: () => {}, + advanced: true, + } as any as BucketAggParam); }); - } + test('does not call write if scaleMetricValues is not set', () => { + const aggConfigs = createAggConfigs([{ type: 'count' } as any]); - // check for two character country code - function expectCountry(val: string | number) { - expect(typeof val).toBe('string'); - expect(val).toHaveLength(2); - } + const writeMock = jest.fn(); + aggConfigs.getRequestAggs()[0].write = writeMock; - // check for an OS term - function expectExtension(val: string | number) { - expect(val).toMatch(/^(js|png|html|css|jpg)$/); - } + tabifyAggResponse(aggConfigs, enrichResponseWithSampling(metricOnly), { + metricsAtAllLevels: true, + }); + expect(writeMock).not.toHaveBeenCalled(); + }); - // check for an OS term - function expectOS(val: string | number) { - expect(val).toMatch(/^(win|mac|linux)$/); - } + test('does call write if scaleMetricValues is set', () => { + const aggConfigs = createAggConfigs([ + { type: 'count', params: { scaleMetricValues: true } } as any, + ]); - // check for something like an average bytes result - function expectAvgBytes(val: string | number) { - expect(typeof val).toBe('number'); - expect(val === 0 || val > 1000).toBeDefined(); - } + const writeMock = jest.fn(() => ({})); + aggConfigs.getRequestAggs()[0].write = writeMock; - test('for non-hierarchical vis', () => { - // the default for a non-hierarchical vis is to display - // only complete rows, and only put the metrics at the end. + tabifyAggResponse(aggConfigs, enrichResponseWithSampling(metricOnly), { + metricsAtAllLevels: true, + }); + expect(writeMock).toHaveBeenCalled(); + }); + }); - const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: false }); + describe(`transforms a complex response${getTitlePostfix()}`, () => { + let esResp: typeof threeTermBuckets; + let aggConfigs: IAggConfigs; + let avg: IAggConfig; + let ext: IAggConfig; + let src: IAggConfig; + let os: IAggConfig; + + function getTopAggregations( + rawResp: typeof threeTermBuckets + ): typeof threeTermBuckets['aggregations'] { + return !isSamplingEnabled(probability) + ? rawResp.aggregations! + : // @ts-ignore + rawResp.aggregations!.sampling!; + } + + beforeEach(() => { + aggConfigs = createAggConfigs([ + mockAggConfig({ type: 'avg', schema: 'metric', params: { field: '@timestamp' } }), + mockAggConfig({ type: 'terms', schema: 'split', params: { field: '@timestamp' } }), + mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), + mockAggConfig({ type: 'terms', schema: 'segment', params: { field: '@timestamp' } }), + ]); - expectColumns(tabbed, [ext, src, os, avg]); + [avg, ext, src, os] = aggConfigs.aggs; - tabbed.rows.forEach((row) => { - expectRow(row, [expectExtension, expectCountry, expectOS, expectAvgBytes]); + esResp = enrichResponseWithSampling(threeTermBuckets); + getTopAggregations(esResp).agg_2.buckets[1].agg_3.buckets[0].agg_4.buckets = []; }); - }); - - test('for hierarchical vis', () => { - const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: true }); - expectColumns(tabbed, [ext, avg, src, avg, os, avg]); + // check that the columns of a table are formed properly + function expectColumns(table: ReturnType, aggs: IAggConfig[]) { + expect(table.columns).toHaveLength(aggs.length); + + aggs.forEach((agg, i) => { + expect(table.columns[i]).toHaveProperty('name', agg.makeLabel()); + }); + } + + // check that a row has expected values + function expectRow( + row: Record, + asserts: Array<(val: string | number) => void> + ) { + expect(typeof row).toBe('object'); + + asserts.forEach((assert, i: number) => { + if (row[`col-${i}`]) { + assert(row[`col-${i}`]); + } + }); + } + + // check for two character country code + function expectCountry(val: string | number) { + expect(typeof val).toBe('string'); + expect(val).toHaveLength(2); + } + + // check for an OS term + function expectExtension(val: string | number) { + expect(val).toMatch(/^(js|png|html|css|jpg)$/); + } + + // check for an OS term + function expectOS(val: string | number) { + expect(val).toMatch(/^(win|mac|linux)$/); + } + + // check for something like an average bytes result + function expectAvgBytes(val: string | number) { + expect(typeof val).toBe('number'); + expect(val === 0 || val > 1000).toBeDefined(); + } + + test('for non-hierarchical vis', () => { + // the default for a non-hierarchical vis is to display + // only complete rows, and only put the metrics at the end. + + const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: false }); + + expectColumns(tabbed, [ext, src, os, avg]); + + tabbed.rows.forEach((row) => { + expectRow(row, [expectExtension, expectCountry, expectOS, expectAvgBytes]); + }); + }); - tabbed.rows.forEach((row) => { - expectRow(row, [ - expectExtension, - expectAvgBytes, - expectCountry, - expectAvgBytes, - expectOS, - expectAvgBytes, - ]); + test('for hierarchical vis', () => { + const tabbed = tabifyAggResponse(aggConfigs, esResp, { metricsAtAllLevels: true }); + + expectColumns(tabbed, [ext, avg, src, avg, os, avg]); + + tabbed.rows.forEach((row) => { + expectRow(row, [ + expectExtension, + expectAvgBytes, + expectCountry, + expectAvgBytes, + expectOS, + expectAvgBytes, + ]); + }); }); }); - }); + } }); diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index ea26afd30129a..2c332c1ad6a75 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -147,7 +147,9 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { - ...esResponse.aggregations, + ...(aggConfigs.isSamplingEnabled() + ? esResponse.aggregations.sampling + : esResponse.aggregations), doc_count: esResponse.aggregations?.doc_count || esResponse.hits?.total, }; diff --git a/src/plugins/data/public/search/expressions/esaggs.test.ts b/src/plugins/data/public/search/expressions/esaggs.test.ts index aa26ceab9ae6b..1a04e4ffeb839 100644 --- a/src/plugins/data/public/search/expressions/esaggs.test.ts +++ b/src/plugins/data/public/search/expressions/esaggs.test.ts @@ -50,6 +50,7 @@ describe('esaggs expression function - public', () => { metricsAtAllLevels: true, partialRows: false, timeFields: ['@timestamp', 'utc_time'], + probability: 1, }; beforeEach(() => { @@ -88,7 +89,7 @@ describe('esaggs expression function - public', () => { expect(startDependencies.aggs.createAggConfigs).toHaveBeenCalledWith( {}, args.aggs.map((agg) => agg.value), - { hierarchical: true, partialRows: false } + { hierarchical: true, partialRows: false, probability: 1, samplerSeed: undefined } ); }); @@ -98,6 +99,8 @@ describe('esaggs expression function - public', () => { expect(startDependencies.aggs.createAggConfigs).toHaveBeenCalledWith({}, [], { hierarchical: true, partialRows: false, + probability: 1, + samplerSeed: undefined, }); }); diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index d944dcd5c1a5d..b82401d4d8caf 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -48,7 +48,12 @@ export function getFunctionDefinition({ const aggConfigs = aggs.createAggConfigs( indexPattern, args.aggs?.map((agg) => agg.value) ?? [], - { hierarchical: args.metricsAtAllLevels, partialRows: args.partialRows } + { + hierarchical: args.metricsAtAllLevels, + partialRows: args.partialRows, + probability: args.probability, + samplerSeed: args.samplerSeed, + } ); const { handleEsaggsRequest } = await import('../../../common/search/expressions'); diff --git a/src/plugins/data/server/search/expressions/esaggs.test.ts b/src/plugins/data/server/search/expressions/esaggs.test.ts index 166f2719e356d..9954fb2457968 100644 --- a/src/plugins/data/server/search/expressions/esaggs.test.ts +++ b/src/plugins/data/server/search/expressions/esaggs.test.ts @@ -51,6 +51,7 @@ describe('esaggs expression function - server', () => { metricsAtAllLevels: true, partialRows: false, timeFields: ['@timestamp', 'utc_time'], + probability: 1, }; beforeEach(() => { diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts index 3f30dff6fd1a7..251d83201f468 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts @@ -315,7 +315,9 @@ describe('IndexPattern Data Source', () => { describe('#toExpression', () => { it('should generate an empty expression when no columns are selected', async () => { const state = FormBasedDatasource.initialize(); - expect(FormBasedDatasource.toExpression(state, 'first', indexPatterns)).toEqual(null); + expect( + FormBasedDatasource.toExpression(state, 'first', indexPatterns, 'testing-seed') + ).toEqual(null); }); it('should create a table when there is a formula without aggs', async () => { @@ -338,7 +340,9 @@ describe('IndexPattern Data Source', () => { }, }, }; - expect(FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns)).toEqual({ + expect( + FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns, 'testing-seed') + ).toEqual({ chain: [ { function: 'createTable', @@ -385,8 +389,9 @@ describe('IndexPattern Data Source', () => { }, }; - expect(FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns)) - .toMatchInlineSnapshot(` + expect( + FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns, 'testing-seed') + ).toMatchInlineSnapshot(` Object { "chain": Array [ Object { @@ -487,6 +492,12 @@ describe('IndexPattern Data Source', () => { "partialRows": Array [ false, ], + "probability": Array [ + 1, + ], + "samplerSeed": Array [ + 1889181588, + ], "timeFields": Array [ "timestamp", ], @@ -560,7 +571,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.timeFields).toEqual(['timestamp', 'another_datefield']); }); @@ -595,7 +611,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect((ast.chain[1].arguments.aggs[1] as Ast).chain[0].arguments.timeShift).toEqual(['1d']); }); @@ -802,7 +823,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const count = (ast.chain[1].arguments.aggs[1] as Ast).chain[0]; const sum = (ast.chain[1].arguments.aggs[2] as Ast).chain[0]; const average = (ast.chain[1].arguments.aggs[3] as Ast).chain[0]; @@ -866,7 +892,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.aggs[0]).toMatchInlineSnapshot(` Object { "chain": Array [ @@ -990,7 +1021,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const timeScaleCalls = ast.chain.filter((fn) => fn.function === 'lens_time_scale'); const formatCalls = ast.chain.filter((fn) => fn.function === 'lens_format_column'); expect(timeScaleCalls).toHaveLength(1); @@ -1055,7 +1091,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const filteredMetricAgg = (ast.chain[1].arguments.aggs[0] as Ast).chain[0].arguments; const metricAgg = (filteredMetricAgg.customMetric[0] as Ast).chain[0].arguments; const bucketAgg = (filteredMetricAgg.customBucket[0] as Ast).chain[0].arguments; @@ -1106,7 +1147,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const formatIndex = ast.chain.findIndex((fn) => fn.function === 'lens_format_column'); const calculationIndex = ast.chain.findIndex((fn) => fn.function === 'moving_average'); expect(calculationIndex).toBeLessThan(formatIndex); @@ -1154,7 +1200,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.metricsAtAllLevels).toEqual([false]); expect(JSON.parse(ast.chain[2].arguments.idMap[0] as string)).toEqual({ 'col-0-0': [expect.objectContaining({ id: 'bucket1' })], @@ -1193,7 +1244,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(ast.chain[1].arguments.timeFields).toEqual(['timestamp']); expect(ast.chain[1].arguments.timeFields).not.toContain('timefield'); }); @@ -1250,7 +1306,7 @@ describe('IndexPattern Data Source', () => { const optimizeMock = jest.spyOn(operationDefinitionMap.percentile, 'optimizeEsAggs'); - FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns); + FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns, 'testing-seed'); expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); @@ -1318,7 +1374,12 @@ describe('IndexPattern Data Source', () => { return { aggs: aggs.reverse(), esAggsIdMap }; }); - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(operationDefinitionMap.percentile.optimizeEsAggs).toHaveBeenCalledTimes(1); @@ -1382,7 +1443,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const idMap = JSON.parse(ast.chain[2].arguments.idMap as unknown as string); @@ -1487,7 +1553,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; // @ts-expect-error we can't isolate just the reference type expect(operationDefinitionMap.testReference.toExpression).toHaveBeenCalled(); expect(ast.chain[3]).toEqual('mock'); @@ -1520,7 +1591,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; expect(JSON.parse(ast.chain[2].arguments.idMap[0] as string)).toEqual({ 'col-0-0': [ @@ -1607,7 +1683,12 @@ describe('IndexPattern Data Source', () => { }, }; - const ast = FormBasedDatasource.toExpression(queryBaseState, 'first', indexPatterns) as Ast; + const ast = FormBasedDatasource.toExpression( + queryBaseState, + 'first', + indexPatterns, + 'testing-seed' + ) as Ast; const chainLength = ast.chain.length; expect(ast.chain[chainLength - 2].arguments.name).toEqual(['math']); expect(ast.chain[chainLength - 1].arguments.id).toEqual(['formula']); @@ -1631,6 +1712,7 @@ describe('IndexPattern Data Source', () => { }, }, currentIndexPatternId: '1', + sampling: 1, }; expect(FormBasedDatasource.insertLayer(state, 'newLayer', ['link-to-id'])).toEqual({ ...state, @@ -1640,6 +1722,7 @@ describe('IndexPattern Data Source', () => { indexPatternId: '1', columnOrder: [], columns: {}, + sampling: 1, linkToLayers: ['link-to-id'], }, }, diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx index 62c77ec5225fe..f772e432268c2 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx @@ -36,6 +36,7 @@ import type { IndexPatternField, IndexPattern, IndexPatternRef, + DatasourceLayerSettingsProps, } from '../../types'; import { changeIndexPattern, @@ -96,6 +97,7 @@ import { getStateTimeShiftWarningMessages } from './time_shift_utils'; import { getPrecisionErrorWarningMessages } from './utils'; import { DOCUMENT_FIELD_NAME } from '../../../common/constants'; import { isColumnOfType } from './operations/definitions/helpers'; +import { LayerSettingsPanel } from './layer_settings'; import { FormBasedLayer } from '../..'; export type { OperationType, GenericIndexPatternColumn } from './operations'; export { deleteColumn } from './operations'; @@ -392,8 +394,34 @@ export function getFormBasedDatasource({ return fields; }, - toExpression: (state, layerId, indexPatterns) => - toExpression(state, layerId, indexPatterns, uiSettings), + toExpression: (state, layerId, indexPatterns, searchSessionId) => + toExpression(state, layerId, indexPatterns, uiSettings, searchSessionId), + + renderLayerSettings( + domElement: Element, + props: DatasourceLayerSettingsProps + ) { + render( + + + + + + + , + domElement + ); + }, renderDataPanel(domElement: Element, props: DatasourceDataPanelProps) { const { onChangeIndexPattern, ...otherProps } = props; @@ -569,6 +597,22 @@ export function getFormBasedDatasource({ getDropProps, onDrop, + getSupportedActionsForLayer(layerId, state, _, openLayerSettings) { + if (!openLayerSettings) { + return []; + } + return [ + { + displayName: i18n.translate('xpack.lens.indexPattern.layerSettingsAction', { + defaultMessage: 'Layer settings', + }), + execute: openLayerSettings, + icon: 'gear', + isCompatible: Boolean(state.layers[layerId]), + 'data-test-subj': 'lnsLayerSettings', + }, + ]; + }, getCustomWorkspaceRenderer: ( state: FormBasedPrivateState, @@ -931,5 +975,6 @@ function blankLayer(indexPatternId: string, linkToLayers?: string[]): FormBasedL linkToLayers, columns: {}, columnOrder: [], + sampling: 1, }; } diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx index 2489659f0da55..eca0c032ee224 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.test.tsx @@ -176,6 +176,7 @@ function testInitialState(): FormBasedPrivateState { currentIndexPatternId: '1', layers: { first: { + sampling: 1, indexPatternId: '1', columnOrder: ['col1'], columns: { @@ -458,7 +459,7 @@ describe('IndexPattern Data Source suggestions', () => { }); describe('with a previous empty layer', () => { - function stateWithEmptyLayer() { + function stateWithEmptyLayer(): FormBasedPrivateState { const state = testInitialState(); return { ...state, @@ -761,6 +762,35 @@ describe('IndexPattern Data Source suggestions', () => { }) ); }); + + it('should inherit the sampling rate when generating new layer, if avaialble', () => { + const state = stateWithEmptyLayer(); + state.layers.previousLayer.sampling = 0.001; + const suggestions = getDatasourceSuggestionsForField( + state, + '1', + { + name: 'bytes', + displayName: 'bytes', + type: 'number', + aggregatable: true, + searchable: true, + }, + expectedIndexPatterns + ); + + expect(suggestions).toContainEqual( + expect.objectContaining({ + state: expect.objectContaining({ + layers: { + previousLayer: expect.objectContaining({ + sampling: 0.001, + }), + }, + }), + }) + ); + }); }); describe('suggesting extensions to non-empty tables', () => { diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts index 52008f10bcdeb..81ce81bb49053 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based_suggestions.ts @@ -529,6 +529,12 @@ function getEmptyLayerSuggestionsForField( newLayer = createNewLayerWithMetricAggregation(indexPattern, field); } + // copy the sampling rate to the new layer + // or just default to 1 + if (newLayer) { + newLayer.sampling = state.layers[layerId]?.sampling ?? 1; + } + const newLayerSuggestions = newLayer ? [ buildSuggestion({ diff --git a/x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx b/x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx new file mode 100644 index 0000000000000..7d02ac98f23a4 --- /dev/null +++ b/x-pack/plugins/lens/public/datasources/form_based/layer_settings.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFormRow, EuiRange, EuiBetaBadge } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import type { DatasourceLayerSettingsProps } from '../../types'; +import type { FormBasedPrivateState } from './types'; + +const samplingValue = [0.0001, 0.001, 0.01, 0.1, 1]; + +export function LayerSettingsPanel({ + state, + setState, + layerId, +}: DatasourceLayerSettingsProps) { + const samplingIndex = samplingValue.findIndex((v) => v === state.layers[layerId].sampling); + const currentSamplingIndex = samplingIndex > -1 ? samplingIndex : samplingValue.length - 1; + return ( + + {i18n.translate('xpack.lens.xyChart.randomSampling.label', { + defaultMessage: 'Sampling', + })}{' '} + + + } + > + { + setState({ + ...state, + layers: { + ...state.layers, + [layerId]: { + ...state.layers[layerId], + sampling: samplingValue[Number(e.currentTarget.value)], + }, + }, + }); + }} + showInput={false} + showRange={false} + showTicks + step={1} + min={0} + max={samplingValue.length - 1} + ticks={samplingValue.map((v, i) => ({ label: `${v}`, value: i }))} + /> + + ); +} diff --git a/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts b/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts index fc77aa6520bd0..365d80a3d8285 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/to_expression.ts @@ -7,6 +7,7 @@ import type { IUiSettingsClient } from '@kbn/core/public'; import { partition, uniq } from 'lodash'; +import seedrandom from 'seedrandom'; import { AggFunctionsMapping, EsaggsExpressionFunctionDefinition, @@ -52,7 +53,8 @@ const updatePositionIndex = (currentId: string, newIndex: number) => { function getExpressionForLayer( layer: FormBasedLayer, indexPattern: IndexPattern, - uiSettings: IUiSettingsClient + uiSettings: IUiSettingsClient, + searchSessionId?: string ): ExpressionAstExpression | null { const { columnOrder } = layer; if (columnOrder.length === 0 || !indexPattern) { @@ -392,6 +394,8 @@ function getExpressionForLayer( metricsAtAllLevels: false, partialRows: false, timeFields: allDateHistogramFields, + probability: layer.sampling || 1, + samplerSeed: seedrandom(searchSessionId).int32(), }).toAst(), { type: 'function', @@ -441,13 +445,15 @@ export function toExpression( state: FormBasedPrivateState, layerId: string, indexPatterns: IndexPatternMap, - uiSettings: IUiSettingsClient + uiSettings: IUiSettingsClient, + searchSessionId?: string ) { if (state.layers[layerId]) { return getExpressionForLayer( state.layers[layerId], indexPatterns[state.layers[layerId].indexPatternId], - uiSettings + uiSettings, + searchSessionId ); } diff --git a/x-pack/plugins/lens/public/datasources/form_based/types.ts b/x-pack/plugins/lens/public/datasources/form_based/types.ts index 3c695d5064e3f..0846c96d76dc8 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/types.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/types.ts @@ -54,6 +54,7 @@ export interface FormBasedLayer { linkToLayers?: string[]; // Partial columns represent the temporary invalid states incompleteColumns?: Record; + sampling?: number; } export interface FormBasedPersistedState { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss index b8ae7fa515190..7b74d0e966410 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss @@ -1,52 +1,3 @@ -@import '@elastic/eui/src/components/flyout/variables'; -@import '@elastic/eui/src/components/flyout/mixins'; - -.lnsDimensionContainer { - // Use the EuiFlyout style - @include euiFlyout; - // But with custom positioning to keep it within the sidebar contents - animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; - left: 0; - max-width: none !important; - z-index: $euiZContentMenu; - - @include euiBreakpoint('l', 'xl') { - height: 100% !important; - position: absolute; - top: 0 !important; - } - - .lnsFrameLayout__sidebar-isFullscreen & { - border-left: $euiBorderThin; // Force border regardless of theme in fullscreen - box-shadow: none; - } -} - -.lnsDimensionContainer__header { - padding: $euiSize; - - .lnsFrameLayout__sidebar-isFullscreen & { - display: none; - } -} - -.lnsDimensionContainer__content { - @include euiYScroll; - flex: 1; -} - -.lnsDimensionContainer__footer { - padding: $euiSize; - - .lnsFrameLayout__sidebar-isFullscreen & { - display: none; - } -} - -.lnsBody--overflowHidden { - overflow: hidden; -} - .lnsLayerAddButton:hover { text-decoration: none; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index 9d71e74eff473..13ba032f9b902 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -7,44 +7,12 @@ import './dimension_container.scss'; -import React, { useState, useEffect, useCallback } from 'react'; -import { - EuiFlyoutHeader, - EuiFlyoutFooter, - EuiTitle, - EuiButtonIcon, - EuiButtonEmpty, - EuiFlexGroup, - EuiFlexItem, - EuiFocusTrap, -} from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import { DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS } from '../../../utils'; - -function fromExcludedClickTarget(event: Event) { - for ( - let node: HTMLElement | null = event.target as HTMLElement; - node !== null; - node = node!.parentElement - ) { - if ( - node.classList!.contains(DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS) || - node.classList!.contains('euiBody-hasPortalContent') || - node.getAttribute('data-euiportal') === 'true' - ) { - return true; - } - } - return false; -} +import React from 'react'; +import { FlyoutContainer } from './flyout_container'; export function DimensionContainer({ - isOpen, - groupLabel, - handleClose, panel, - isFullscreen, - panelRef, + ...props }: { isOpen: boolean; handleClose: () => boolean; @@ -53,107 +21,5 @@ export function DimensionContainer({ isFullscreen: boolean; panelRef: (el: HTMLDivElement) => void; }) { - const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); - - const closeFlyout = useCallback(() => { - const canClose = handleClose(); - if (canClose) { - setFocusTrapIsEnabled(false); - } - return canClose; - }, [handleClose]); - - useEffect(() => { - document.body.classList.toggle('lnsBody--overflowHidden', isOpen); - return () => { - if (isOpen) { - setFocusTrapIsEnabled(false); - } - document.body.classList.remove('lnsBody--overflowHidden'); - }; - }, [isOpen]); - - if (!isOpen) { - return null; - } - - return ( -
- { - if (isFullscreen || fromExcludedClickTarget(event)) { - return; - } - closeFlyout(); - }} - onEscapeKey={closeFlyout} - > -
{ - if (isOpen) { - // EuiFocusTrap interferes with animating elements with absolute position: - // running this onAnimationEnd, otherwise the flyout pushes content when animating - setFocusTrapIsEnabled(true); - } - }} - > - - - - -

- - {i18n.translate('xpack.lens.configure.configurePanelTitle', { - defaultMessage: '{groupLabel}', - values: { - groupLabel, - }, - })} - -

-
-
- - - - -
-
- -
{panel}
- - - - {i18n.translate('xpack.lens.dimensionContainer.close', { - defaultMessage: 'Close', - })} - - -
-
-
- ); + return {panel}; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss new file mode 100644 index 0000000000000..b08eb6281fa0e --- /dev/null +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.scss @@ -0,0 +1,48 @@ +@import '@elastic/eui/src/components/flyout/variables'; +@import '@elastic/eui/src/components/flyout/mixins'; + +.lnsDimensionContainer { + // Use the EuiFlyout style + @include euiFlyout; + // But with custom positioning to keep it within the sidebar contents + animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; + left: 0; + max-width: none !important; + z-index: $euiZContentMenu; + + @include euiBreakpoint('l', 'xl') { + height: 100% !important; + position: absolute; + top: 0 !important; + } + + .lnsFrameLayout__sidebar-isFullscreen & { + border-left: $euiBorderThin; // Force border regardless of theme in fullscreen + box-shadow: none; + } +} + +.lnsDimensionContainer__header { + padding: $euiSize; + + .lnsFrameLayout__sidebar-isFullscreen & { + display: none; + } +} + +.lnsDimensionContainer__content { + @include euiYScroll; + flex: 1; +} + +.lnsDimensionContainer__footer { + padding: $euiSize; + + .lnsFrameLayout__sidebar-isFullscreen & { + display: none; + } +} + +.lnsBody--overflowHidden { + overflow: hidden; +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx new file mode 100644 index 0000000000000..041f59df332f4 --- /dev/null +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/flyout_container.tsx @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import './flyout_container.scss'; + +import React, { useState, useEffect, useCallback } from 'react'; +import { + EuiFlyoutHeader, + EuiFlyoutFooter, + EuiTitle, + EuiButtonIcon, + EuiButtonEmpty, + EuiFlexGroup, + EuiFlexItem, + EuiFocusTrap, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS } from '../../../utils'; + +function fromExcludedClickTarget(event: Event) { + for ( + let node: HTMLElement | null = event.target as HTMLElement; + node !== null; + node = node!.parentElement + ) { + if ( + node.classList!.contains(DONT_CLOSE_DIMENSION_CONTAINER_ON_CLICK_CLASS) || + node.classList!.contains('euiBody-hasPortalContent') || + node.getAttribute('data-euiportal') === 'true' + ) { + return true; + } + } + return false; +} + +export function FlyoutContainer({ + isOpen, + groupLabel, + handleClose, + isFullscreen, + panelRef, + children, +}: { + isOpen: boolean; + handleClose: () => boolean; + children: React.ReactElement | null; + groupLabel: string; + isFullscreen: boolean; + panelRef: (el: HTMLDivElement) => void; +}) { + const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); + + const closeFlyout = useCallback(() => { + const canClose = handleClose(); + if (canClose) { + setFocusTrapIsEnabled(false); + } + return canClose; + }, [handleClose]); + + useEffect(() => { + document.body.classList.toggle('lnsBody--overflowHidden', isOpen); + return () => { + if (isOpen) { + setFocusTrapIsEnabled(false); + } + document.body.classList.remove('lnsBody--overflowHidden'); + }; + }, [isOpen]); + + if (!isOpen) { + return null; + } + + return ( +
+ { + if (isFullscreen || fromExcludedClickTarget(event)) { + return; + } + closeFlyout(); + }} + onEscapeKey={closeFlyout} + > +
{ + if (isOpen) { + // EuiFocusTrap interferes with animating elements with absolute position: + // running this onAnimationEnd, otherwise the flyout pushes content when animating + setFocusTrapIsEnabled(true); + } + }} + > + + + + +

+ {i18n.translate('xpack.lens.configure.configurePanelTitle', { + defaultMessage: '{groupLabel}', + values: { + groupLabel, + }, + })} +

+
+
+ + + + +
+
+ +
{children}
+ + + + {i18n.translate('xpack.lens.dimensionContainer.close', { + defaultMessage: 'Close', + })} + + +
+
+
+ ); +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx index bc1c41caa650b..9c32a24eac4dd 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_actions/layer_actions.tsx @@ -104,6 +104,9 @@ const InContextMenuActions = (props: LayerActionsProps) => { closePopover={closePopover} panelPaddingSize="none" anchorPosition="downLeft" + panelProps={{ + 'data-test-subj': 'lnsLayerActionsMenu', + }} > ( initialActiveDimensionState ); + const [isPanelSettingsOpen, setPanelSettingsOpen] = useState(false); + const [hideTooltip, setHideTooltip] = useState(false); const { @@ -120,6 +123,7 @@ export function LayerPanel( }, [activeVisualization.id]); const panelRef = useRef(null); + const settingsPanelRef = useRef(null); const registerLayerRef = useCallback( (el) => registerNewLayerRef(layerId, el), [layerId, registerNewLayerRef] @@ -316,7 +320,14 @@ export function LayerPanel( ...(activeVisualization.getSupportedActionsForLayer?.( layerId, visualizationState, - updateVisualization + updateVisualization, + () => setPanelSettingsOpen(true) + ) || []), + ...(layerDatasource?.getSupportedActionsForLayer?.( + layerId, + layerDatasourceState, + (newState) => updateDatasource(datasourceId, newState), + () => setPanelSettingsOpen(true) ) || []), ...getSharedActions({ activeVisualization, @@ -332,12 +343,16 @@ export function LayerPanel( [ activeVisualization, core, + datasourceId, isOnlyLayer, isTextBasedLanguage, + layerDatasource, + layerDatasourceState, layerId, layerIndex, onCloneLayer, onRemoveLayer, + updateDatasource, updateVisualization, visualizationState, ] @@ -624,7 +639,42 @@ export function LayerPanel( })} - + {(layerDatasource?.renderLayerSettings || activeVisualization?.renderLayerSettings) && ( + (settingsPanelRef.current = el)} + isOpen={isPanelSettingsOpen} + isFullscreen={false} + groupLabel={i18n.translate('xpack.lens.editorFrame.layerSettingsTitle', { + defaultMessage: 'Layer settings', + })} + handleClose={() => { + // update the current layer settings + setPanelSettingsOpen(false); + return true; + }} + > +
+
+ {layerDatasource?.renderLayerSettings && ( + + )} + {activeVisualization?.renderLayerSettings && ( + + )} +
+
+
+ )} (panelRef.current = el)} isOpen={isDimensionPanelOpen} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts index c9b0358b81a0b..aee10196d5156 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/expression_helpers.ts @@ -11,7 +11,8 @@ import { Visualization, DatasourceMap, DatasourceLayers, IndexPatternMap } from export function getDatasourceExpressionsByLayers( datasourceMap: DatasourceMap, datasourceStates: DatasourceStates, - indexPatterns: IndexPatternMap + indexPatterns: IndexPatternMap, + searchSessionId?: string ): null | Record { const datasourceExpressions: Array<[string, Ast | string]> = []; @@ -24,7 +25,7 @@ export function getDatasourceExpressionsByLayers( const layers = datasource.getLayers(state); layers.forEach((layerId) => { - const result = datasource.toExpression(state, layerId, indexPatterns); + const result = datasource.toExpression(state, layerId, indexPatterns, searchSessionId); if (result) { datasourceExpressions.push([layerId, result]); } @@ -53,6 +54,7 @@ export function buildExpression({ title, description, indexPatterns, + searchSessionId, }: { title?: string; description?: string; @@ -62,6 +64,7 @@ export function buildExpression({ datasourceStates: DatasourceStates; datasourceLayers: DatasourceLayers; indexPatterns: IndexPatternMap; + searchSessionId?: string; }): Ast | null { if (visualization === null) { return null; @@ -70,7 +73,8 @@ export function buildExpression({ const datasourceExpressionsByLayers = getDatasourceExpressionsByLayers( datasourceMap, datasourceStates, - indexPatterns + indexPatterns, + searchSessionId ); const visualizationExpression = visualization.toExpression( diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 38f62b6e928b6..2f2003970171a 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -162,6 +162,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ const changesApplied = useLensSelector(selectChangesApplied); const triggerApply = useLensSelector(selectTriggerApplyChanges); const datasourceLayers = useLensSelector((state) => selectDatasourceLayers(state, datasourceMap)); + const searchSessionId = useLensSelector(selectSearchSessionId); const [localState, setLocalState] = useState({ expressionBuildError: undefined, @@ -317,6 +318,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ datasourceStates, datasourceLayers, indexPatterns: dataViews.indexPatterns, + searchSessionId, }); if (ast) { @@ -349,16 +351,17 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ })); } }, [ + configurationValidationError?.length, + missingRefsErrors.length, + unknownVisError, activeVisualization, visualization.state, + visualization.activeId, datasourceMap, datasourceStates, datasourceLayers, - configurationValidationError?.length, - missingRefsErrors.length, - unknownVisError, - visualization.activeId, dataViews.indexPatterns, + searchSessionId, ]); useEffect(() => { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index e01c35f3457d9..dd5d12809c94a 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -301,6 +301,10 @@ export interface Datasource { }) => T; getSelectedFields?: (state: T) => string[]; + renderLayerSettings?: ( + domElement: Element, + props: DatasourceLayerSettingsProps + ) => ((cleanupElement: Element) => void) | void; renderDataPanel: ( domElement: Element, props: DatasourceDataPanelProps @@ -360,7 +364,8 @@ export interface Datasource { toExpression: ( state: T, layerId: string, - indexPatterns: IndexPatternMap + indexPatterns: IndexPatternMap, + searchSessionId?: string ) => ExpressionAstExpression | string | null; getDatasourceSuggestionsForField: ( @@ -458,6 +463,13 @@ export interface Datasource { * Get all the used DataViews from state */ getUsedDataViews: (state: T) => string[]; + + getSupportedActionsForLayer?: ( + layerId: string, + state: T, + setState: StateSetter, + openLayerSettings?: () => void + ) => LayerAction[]; } export interface DatasourceFixAction { @@ -509,6 +521,12 @@ export interface DatasourcePublicAPI { hasDefaultTimeField: () => boolean; } +export interface DatasourceLayerSettingsProps { + layerId: string; + state: T; + setState: StateSetter; +} + export interface DatasourceDataPanelProps { state: T; dragDropContext: DragContextState; @@ -715,6 +733,11 @@ export interface VisualizationToolbarProps { state: T; } +export type VisualizationLayerSettingsProps = VisualizationConfigProps & { + setState(newState: T | ((currState: T) => T)): void; + panelRef: MutableRefObject; +}; + export type VisualizationDimensionEditorProps = VisualizationConfigProps & { groupId: string; accessor: string; @@ -1010,7 +1033,8 @@ export interface Visualization { getSupportedActionsForLayer?: ( layerId: string, state: T, - setState: StateSetter + setState: StateSetter, + openLayerSettings?: () => void ) => LayerAction[]; /** returns the type string of the given layer */ getLayerType: (layerId: string, state?: T) => LayerType | undefined; @@ -1090,6 +1114,11 @@ export interface Visualization { dropProps: GetDropPropsArgs ) => { dropTypes: DropType[]; nextLabel?: string } | undefined; + renderLayerSettings?: ( + domElement: Element, + props: VisualizationLayerSettingsProps + ) => ((cleanupElement: Element) => void) | void; + /** * Additional editor that gets rendered inside the dimension popover. * This can be used to configure dimension-specific options diff --git a/x-pack/test/functional/apps/lens/group1/index.ts b/x-pack/test/functional/apps/lens/group1/index.ts index aa2b078a50a6b..47f08a59e7341 100644 --- a/x-pack/test/functional/apps/lens/group1/index.ts +++ b/x-pack/test/functional/apps/lens/group1/index.ts @@ -79,6 +79,7 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext loadTestFile(require.resolve('./table_dashboard')); loadTestFile(require.resolve('./table')); loadTestFile(require.resolve('./text_based_languages')); + loadTestFile(require.resolve('./layer_actions')); } }); }; diff --git a/x-pack/test/functional/apps/lens/group1/layer_actions.ts b/x-pack/test/functional/apps/lens/group1/layer_actions.ts new file mode 100644 index 0000000000000..be22e4ad62511 --- /dev/null +++ b/x-pack/test/functional/apps/lens/group1/layer_actions.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const find = getService('find'); + const testSubjects = getService('testSubjects'); + + describe('lens layer actions tests', () => { + it('should allow creation of lens xy chart', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.openLayerContextMenu(); + + // should be 3 actions available + expect( + (await find.allByCssSelector('[data-test-subj=lnsLayerActionsMenu] button')).length + ).to.eql(3); + }); + + it('should open layer settings for a data layer', async () => { + // click on open layer settings + await testSubjects.click('lnsLayerSettings'); + // random sampling available + await testSubjects.existOrFail('lns-indexPattern-random-sampling-row'); + // tweak the value + await PageObjects.lens.dragRangeInput('lns-indexPattern-random-sampling', 2, 'left'); + + expect(await PageObjects.lens.getRangeInputValue('lns-indexPattern-random-sampling')).to.eql( + 2 // 0.01 + ); + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + }); + + it('should add an annotation layer and settings shoud not be available', async () => { + // configure a date histogram + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'average', + field: 'bytes', + }); + // add annotation layer + await testSubjects.click('lnsLayerAddButton'); + await testSubjects.click(`lnsLayerAddButton-annotations`); + await PageObjects.lens.openLayerContextMenu(1); + // layer settings not available + await testSubjects.missingOrFail('lnsLayerSettings'); + }); + + it('should switch to pie chart and have layer settings available', async () => { + await PageObjects.lens.switchToVisualization('pie'); + await PageObjects.lens.openLayerContextMenu(); + // layer settings still available + // open the panel + await testSubjects.click('lnsLayerSettings'); + // check the sampling value + expect(await PageObjects.lens.getRangeInputValue('lns-indexPattern-random-sampling')).to.eql( + 2 // 0.01 + ); + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + }); + + it('should switch to table and still have layer settings', async () => { + await PageObjects.lens.switchToVisualization('lnsDatatable'); + await PageObjects.lens.openLayerContextMenu(); + // layer settings still available + // open the panel + await testSubjects.click('lnsLayerSettings'); + // check the sampling value + expect(await PageObjects.lens.getRangeInputValue('lns-indexPattern-random-sampling')).to.eql( + 2 // 0.01 + ); + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index a336a0da3e4ba..8dd95aa107929 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -600,6 +600,19 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await this.waitForVisualization(); }, + async dragRangeInput(testId: string, steps: number = 1, direction: 'left' | 'right' = 'right') { + const inputEl = await testSubjects.find(testId); + await inputEl.focus(); + const browserKey = direction === 'left' ? browser.keys.LEFT : browser.keys.RIGHT; + while (steps--) { + await browser.pressKeys(browserKey); + } + }, + + async getRangeInputValue(testId: string) { + return (await testSubjects.find(testId)).getAttribute('value'); + }, + async isTopLevelAggregation() { return await testSubjects.isEuiSwitchChecked('indexPattern-nesting-switch'); }, @@ -1117,6 +1130,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ); }, + async openLayerContextMenu(index: number = 0) { + await testSubjects.click(`lnsLayerSplitButton--${index}`); + }, + async toggleColumnVisibility(dimension: string, no = 1) { await this.openDimensionEditor(dimension); const id = 'lns-table-column-hidden'; From b87e8d0a540bfdc4273515557a32d52b9a2a15c5 Mon Sep 17 00:00:00 2001 From: Kaarina Tungseth Date: Mon, 24 Oct 2022 12:29:52 -0500 Subject: [PATCH 03/15] Updates Edit data view link copy (#142883) * Updates Edit data view link copy * Updates Stack Management link text Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/components/data_view_editor_flyout_content.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx index 9c86170795961..e9b2b8de5c5f7 100644 --- a/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx +++ b/src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx @@ -335,7 +335,7 @@ const IndexPatternEditorFlyoutContentComponent = ({ })} > {i18n.translate('indexPatternEditor.goToManagementPage', { - defaultMessage: 'View on data view management page', + defaultMessage: 'Manage settings and view field details', })} )} From bbe3e232898c708ee4ad35fae1f9981c2cbf438b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 24 Oct 2022 19:41:27 +0200 Subject: [PATCH 04/15] [APM] Make `track_total_hits` and `size` required in apm alerting client (#143851) * [APM] Make `track_total_hits` and `size` required in apm alerting client * undo changes --- .../apm/server/routes/alerts/alerting_es_client.ts | 8 +++++++- .../rule_types/anomaly/register_anomaly_rule_type.ts | 1 + .../error_count/register_error_count_rule_type.ts | 7 ++++--- .../register_transaction_duration_rule_type.ts | 1 + .../register_transaction_error_rate_rule_type.ts | 3 ++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/server/routes/alerts/alerting_es_client.ts b/x-pack/plugins/apm/server/routes/alerts/alerting_es_client.ts index f0dc94daadf0b..fabe71a874beb 100644 --- a/x-pack/plugins/apm/server/routes/alerts/alerting_es_client.ts +++ b/x-pack/plugins/apm/server/routes/alerts/alerting_es_client.ts @@ -8,7 +8,13 @@ import type { ESSearchRequest, ESSearchResponse } from '@kbn/es-types'; import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; -export async function alertingEsClient({ +export type APMEventESSearchRequestParams = ESSearchRequest & { + body: { size: number; track_total_hits: boolean | number }; +}; + +export async function alertingEsClient< + TParams extends APMEventESSearchRequestParams +>({ scopedClusterClient, params, }: { diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts index 13d0f3311f186..d82a4997ffe0e 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts @@ -145,6 +145,7 @@ export function registerAnomalyRuleType({ const jobIds = mlJobs.map((job) => job.jobId); const anomalySearchParams = { body: { + track_total_hits: false, size: 0, query: { bool: { diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts index 93c88b84d1d37..58e475ced07fb 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts @@ -83,18 +83,19 @@ export function registerErrorCountRuleType({ producer: APM_SERVER_FEATURE_ID, minimumLicenseRequired: 'basic', isExportable: true, - executor: async ({ services, params }) => { + executor: async ({ services, params: ruleParams }) => { const config = await firstValueFrom(config$); - const ruleParams = params; const indices = await getApmIndices({ config, savedObjectsClient: services.savedObjectsClient, }); + const searchParams = { index: indices.error, - size: 0, body: { + track_total_hits: false, + size: 0, query: { bool: { filter: [ diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index 7a7e84414aec0..0ea099c8d4bc2 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -125,6 +125,7 @@ export function registerTransactionDurationRuleType({ const searchParams = { index, body: { + track_total_hits: false, size: 0, query: { bool: { diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts index 5e4e04d293cd7..15a5880345ffd 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts @@ -112,8 +112,9 @@ export function registerTransactionErrorRateRuleType({ const searchParams = { index, - size: 0, body: { + track_total_hits: false, + size: 0, query: { bool: { filter: [ From 9f312e8776701089c9d91ee7761b54eb263d39be Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 12:56:43 -0500 Subject: [PATCH 05/15] Update babel (main) (#143490) * Update babel * dedupe * update yarn.lock * update snapshot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Jonathan Budzenski --- package.json | 18 +- .../src/update_vscode_config.test.ts | 1 - yarn.lock | 233 +++++++----------- 3 files changed, 104 insertions(+), 148 deletions(-) diff --git a/package.json b/package.json index 8a71f04064b8a..909f468342b82 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ }, "dependencies": { "@appland/sql-parser": "^1.5.1", - "@babel/runtime": "^7.19.0", + "@babel/runtime": "^7.19.4", "@dnd-kit/core": "^3.1.1", "@dnd-kit/sortable": "^4.0.0", "@dnd-kit/utilities": "^2.0.0", @@ -673,25 +673,25 @@ "devDependencies": { "@apidevtools/swagger-parser": "^10.0.3", "@babel/cli": "^7.19.3", - "@babel/core": "^7.19.3", + "@babel/core": "^7.19.6", "@babel/eslint-parser": "^7.19.1", "@babel/eslint-plugin": "^7.19.1", - "@babel/generator": "^7.19.3", + "@babel/generator": "^7.19.6", "@babel/helper-plugin-utils": "^7.19.0", - "@babel/parser": "^7.19.3", + "@babel/parser": "^7.19.6", "@babel/plugin-proposal-class-properties": "^7.18.6", "@babel/plugin-proposal-export-namespace-from": "^7.18.9", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", - "@babel/plugin-proposal-object-rest-spread": "^7.18.9", + "@babel/plugin-proposal-object-rest-spread": "^7.19.4", "@babel/plugin-proposal-optional-chaining": "^7.18.9", "@babel/plugin-proposal-private-methods": "^7.18.6", - "@babel/plugin-transform-runtime": "^7.19.1", - "@babel/preset-env": "^7.19.3", + "@babel/plugin-transform-runtime": "^7.19.6", + "@babel/preset-env": "^7.19.4", "@babel/preset-react": "^7.18.6", "@babel/preset-typescript": "^7.18.6", "@babel/register": "^7.18.9", - "@babel/traverse": "^7.19.3", - "@babel/types": "^7.19.3", + "@babel/traverse": "^7.19.6", + "@babel/types": "^7.19.4", "@bazel/ibazel": "^0.16.2", "@bazel/typescript": "4.6.2", "@cypress/code-coverage": "^3.10.0", diff --git a/packages/kbn-managed-vscode-config/src/update_vscode_config.test.ts b/packages/kbn-managed-vscode-config/src/update_vscode_config.test.ts index f9e3aecb89b93..cfa1964dd297b 100644 --- a/packages/kbn-managed-vscode-config/src/update_vscode_config.test.ts +++ b/packages/kbn-managed-vscode-config/src/update_vscode_config.test.ts @@ -133,7 +133,6 @@ it('persists comments in the original file', () => { `); expect(newJson).toMatchInlineSnapshot(` // @managed - /** * This is a top level comment */ diff --git a/yarn.lock b/yarn.lock index f97953886c2a3..1770a163d752c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -81,10 +81,10 @@ dependencies: "@babel/highlight" "^7.18.6" -"@babel/compat-data@^7.17.7", "@babel/compat-data@^7.18.8", "@babel/compat-data@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.19.3.tgz#707b939793f867f5a73b2666e6d9a3396eb03151" - integrity sha512-prBHMK4JYYK+wDjJF1q99KK4JLL+egWS4nmNqdlMUgCExMZ+iZW0hGhyC3VEbsPjvaN0TBhW//VIFwBrk8sEiw== +"@babel/compat-data@^7.17.7", "@babel/compat-data@^7.19.3", "@babel/compat-data@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.19.4.tgz#95c86de137bf0317f3a570e1b6e996b427299747" + integrity sha512-CHIGpJcUQ5lU9KrPHTjBMhVwQG6CQjxfg36fGXl3qk/Gik1WwWachaXFuo0uCWJT/mStOKtcbFJCaVLihC1CMw== "@babel/core@7.12.9": version "7.12.9" @@ -108,21 +108,21 @@ semver "^5.4.1" source-map "^0.5.0" -"@babel/core@^7.1.0", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.19.3", "@babel/core@^7.7.5": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.19.3.tgz#2519f62a51458f43b682d61583c3810e7dcee64c" - integrity sha512-WneDJxdsjEvyKtXKsaBGbDeiyOjR5vYq4HcShxnIbG0qixpoHjI3MqeZM9NDvsojNCEBItQE4juOo/bU6e72gQ== +"@babel/core@^7.1.0", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.19.6", "@babel/core@^7.7.5": + version "7.19.6" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.19.6.tgz#7122ae4f5c5a37c0946c066149abd8e75f81540f" + integrity sha512-D2Ue4KHpc6Ys2+AxpIx1BZ8+UegLLLE2p3KJEuJRKmokHOtl49jQ5ny1773KsGLZs8MQvBidAF6yWUJxRqtKtg== dependencies: "@ampproject/remapping" "^2.1.0" "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.19.3" + "@babel/generator" "^7.19.6" "@babel/helper-compilation-targets" "^7.19.3" - "@babel/helper-module-transforms" "^7.19.0" - "@babel/helpers" "^7.19.0" - "@babel/parser" "^7.19.3" + "@babel/helper-module-transforms" "^7.19.6" + "@babel/helpers" "^7.19.4" + "@babel/parser" "^7.19.6" "@babel/template" "^7.18.10" - "@babel/traverse" "^7.19.3" - "@babel/types" "^7.19.3" + "@babel/traverse" "^7.19.6" + "@babel/types" "^7.19.4" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -145,12 +145,12 @@ dependencies: eslint-rule-composer "^0.3.0" -"@babel/generator@^7.12.11", "@babel/generator@^7.12.5", "@babel/generator@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.19.3.tgz#d7f4d1300485b4547cb6f94b27d10d237b42bf59" - integrity sha512-fqVZnmp1ncvZU757UzDheKZpfPgatqY59XtW2/j/18H7u76akb8xqvjw82f+i2UKd/ksYsSick/BCLQUUtJ/qQ== +"@babel/generator@^7.12.11", "@babel/generator@^7.12.5", "@babel/generator@^7.19.6": + version "7.19.6" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.19.6.tgz#9e481a3fe9ca6261c972645ae3904ec0f9b34a1d" + integrity sha512-oHGRUQeoX1QrKeJIKVe0hwjGqNnVYsM5Nep5zo0uE0m42sLH+Fsd2pStJ5sRM1bNyTUUoz0pe2lTeMJrb/taTA== dependencies: - "@babel/types" "^7.19.3" + "@babel/types" "^7.19.4" "@jridgewell/gen-mapping" "^0.3.2" jsesc "^2.5.1" @@ -267,19 +267,19 @@ dependencies: "@babel/types" "^7.18.6" -"@babel/helper-module-transforms@^7.12.1", "@babel/helper-module-transforms@^7.18.6", "@babel/helper-module-transforms@^7.19.0": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.19.0.tgz#309b230f04e22c58c6a2c0c0c7e50b216d350c30" - integrity sha512-3HBZ377Fe14RbLIA+ac3sY4PTgpxHVkFrESaWhoI5PuyXPBBX8+C34qblV9G89ZtycGJCmCI/Ut+VUDK4bltNQ== +"@babel/helper-module-transforms@^7.12.1", "@babel/helper-module-transforms@^7.18.6", "@babel/helper-module-transforms@^7.19.0", "@babel/helper-module-transforms@^7.19.6": + version "7.19.6" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.19.6.tgz#6c52cc3ac63b70952d33ee987cbee1c9368b533f" + integrity sha512-fCmcfQo/KYr/VXXDIyd3CBGZ6AFhPFy1TfSEJ+PilGVlQT6jcbqtHAM4C1EciRqMza7/TpOUZliuSH+U6HAhJw== dependencies: "@babel/helper-environment-visitor" "^7.18.9" "@babel/helper-module-imports" "^7.18.6" - "@babel/helper-simple-access" "^7.18.6" + "@babel/helper-simple-access" "^7.19.4" "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/helper-validator-identifier" "^7.18.6" + "@babel/helper-validator-identifier" "^7.19.1" "@babel/template" "^7.18.10" - "@babel/traverse" "^7.19.0" - "@babel/types" "^7.19.0" + "@babel/traverse" "^7.19.6" + "@babel/types" "^7.19.4" "@babel/helper-optimise-call-expression@^7.18.6": version "7.18.6" @@ -319,12 +319,12 @@ "@babel/traverse" "^7.18.9" "@babel/types" "^7.18.9" -"@babel/helper-simple-access@^7.18.6": - version "7.18.6" - resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.18.6.tgz#d6d8f51f4ac2978068df934b569f08f29788c7ea" - integrity sha512-iNpIgTgyAvDQpDj76POqg+YEt8fPxx3yaNBg3S30dxNKm2SWfYhD0TGrK/Eu9wHpUW63VQU894TsTg+GLbUa1g== +"@babel/helper-simple-access@^7.18.6", "@babel/helper-simple-access@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.19.4.tgz#be553f4951ac6352df2567f7daa19a0ee15668e7" + integrity sha512-f9Xq6WqBFqaDfbCzn2w85hwklswz5qsKlh7f08w4Y9yhJHpnNC0QemtSkK5YyOY8kPGvyiwdzZksGUhnGdaUIg== dependencies: - "@babel/types" "^7.18.6" + "@babel/types" "^7.19.4" "@babel/helper-skip-transparent-expression-wrappers@^7.18.9": version "7.18.9" @@ -340,10 +340,10 @@ dependencies: "@babel/types" "^7.18.6" -"@babel/helper-string-parser@^7.18.10": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.18.10.tgz#181f22d28ebe1b3857fa575f5c290b1aaf659b56" - integrity sha512-XtIfWmeNY3i4t7t4D2t02q50HvqHybPqW2ki1kosnvWCwuCMeo81Jf0gwr85jy/neUdg5XDdeFE/80DXiO+njw== +"@babel/helper-string-parser@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63" + integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw== "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" @@ -365,14 +365,14 @@ "@babel/traverse" "^7.18.9" "@babel/types" "^7.18.9" -"@babel/helpers@^7.12.5", "@babel/helpers@^7.19.0": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.19.0.tgz#f30534657faf246ae96551d88dd31e9d1fa1fc18" - integrity sha512-DRBCKGwIEdqY3+rPJgG/dKfQy9+08rHIAJx8q2p+HSWP87s2HCrQmaAMMyMll2kIXKCW0cO1RdQskx15Xakftg== +"@babel/helpers@^7.12.5", "@babel/helpers@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.19.4.tgz#42154945f87b8148df7203a25c31ba9a73be46c5" + integrity sha512-G+z3aOx2nfDHwX/kyVii5fJq+bgscg89/dJNWpYeKeBv3v9xX8EIabmx1k6u9LS04H7nROFVRVK+e3k0VHp+sw== dependencies: "@babel/template" "^7.18.10" - "@babel/traverse" "^7.19.0" - "@babel/types" "^7.19.0" + "@babel/traverse" "^7.19.4" + "@babel/types" "^7.19.4" "@babel/highlight@^7.10.4", "@babel/highlight@^7.18.6": version "7.18.6" @@ -383,10 +383,10 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.10.3", "@babel/parser@^7.12.11", "@babel/parser@^7.12.7", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10", "@babel/parser@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.3.tgz#8dd36d17c53ff347f9e55c328710321b49479a9a" - integrity sha512-pJ9xOlNWHiy9+FuFP09DEAFbAn4JskgRsVcc169w2xRBC3FRGuQEwjeIMMND9L2zc0iEhO/tGv4Zq+km+hxNpQ== +"@babel/parser@^7.1.0", "@babel/parser@^7.10.3", "@babel/parser@^7.12.11", "@babel/parser@^7.12.7", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10", "@babel/parser@^7.19.6": + version "7.19.6" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.6.tgz#b923430cb94f58a7eae8facbffa9efd19130e7f8" + integrity sha512-h1IUp81s2JYJ3mRkdxJgs4UvmSsRvDrx5ICSJbPvtWYv5i1nTBGcBpnog+89rAFMwvvru6E5NUHdBe01UeSzYA== "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" @@ -505,14 +505,14 @@ "@babel/plugin-syntax-object-rest-spread" "^7.8.0" "@babel/plugin-transform-parameters" "^7.12.1" -"@babel/plugin-proposal-object-rest-spread@^7.12.1", "@babel/plugin-proposal-object-rest-spread@^7.18.9": - version "7.18.9" - resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.18.9.tgz#f9434f6beb2c8cae9dfcf97d2a5941bbbf9ad4e7" - integrity sha512-kDDHQ5rflIeY5xl69CEqGEZ0KY369ehsCIEbTGb4siHG5BE9sga/T0r0OUwyZNLMmZE79E1kbsqAjwFCW4ds6Q== +"@babel/plugin-proposal-object-rest-spread@^7.12.1", "@babel/plugin-proposal-object-rest-spread@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.19.4.tgz#a8fc86e8180ff57290c91a75d83fe658189b642d" + integrity sha512-wHmj6LDxVDnL+3WhXteUBaoM1aVILZODAUjg11kHqG4cOlfgMQGxw6aCgvrXrmaJR3Bn14oZhImyCPZzRpC93Q== dependencies: - "@babel/compat-data" "^7.18.8" - "@babel/helper-compilation-targets" "^7.18.9" - "@babel/helper-plugin-utils" "^7.18.9" + "@babel/compat-data" "^7.19.4" + "@babel/helper-compilation-targets" "^7.19.3" + "@babel/helper-plugin-utils" "^7.19.0" "@babel/plugin-syntax-object-rest-spread" "^7.8.3" "@babel/plugin-transform-parameters" "^7.18.8" @@ -743,12 +743,12 @@ dependencies: "@babel/helper-plugin-utils" "^7.18.6" -"@babel/plugin-transform-block-scoping@^7.12.12", "@babel/plugin-transform-block-scoping@^7.18.9": - version "7.18.9" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.18.9.tgz#f9b7e018ac3f373c81452d6ada8bd5a18928926d" - integrity sha512-5sDIJRV1KtQVEbt/EIBwGy4T01uYIo4KRB3VUqzkhrAIOGx7AoctL9+Ux88btY0zXdDyPJ9mW+bg+v+XEkGmtw== +"@babel/plugin-transform-block-scoping@^7.12.12", "@babel/plugin-transform-block-scoping@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.19.4.tgz#315d70f68ce64426db379a3d830e7ac30be02e9b" + integrity sha512-934S2VLLlt2hRJwPf4MczaOr4hYF0z+VKPwqTNxyKX7NthTiPfhuKFWQZHXRM0vh/wo/VyXB3s4bZUNA08l+tQ== dependencies: - "@babel/helper-plugin-utils" "^7.18.9" + "@babel/helper-plugin-utils" "^7.19.0" "@babel/plugin-transform-classes@^7.12.1", "@babel/plugin-transform-classes@^7.19.0": version "7.19.0" @@ -772,12 +772,12 @@ dependencies: "@babel/helper-plugin-utils" "^7.18.9" -"@babel/plugin-transform-destructuring@^7.12.1", "@babel/plugin-transform-destructuring@^7.18.13": - version "7.18.13" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.18.13.tgz#9e03bc4a94475d62b7f4114938e6c5c33372cbf5" - integrity sha512-TodpQ29XekIsex2A+YJPj5ax2plkGa8YYY6mFjCohk/IG9IY42Rtuj1FuDeemfg2ipxIFLzPeA83SIBnlhSIow== +"@babel/plugin-transform-destructuring@^7.12.1", "@babel/plugin-transform-destructuring@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.19.4.tgz#46890722687b9b89e1369ad0bd8dc6c5a3b4319d" + integrity sha512-t0j0Hgidqf0aM86dF8U+vXYReUgJnlv4bZLsyoPnwZNrGY+7/38o8YjaELrvHeVfTZao15kjR0PVv0nju2iduA== dependencies: - "@babel/helper-plugin-utils" "^7.18.9" + "@babel/helper-plugin-utils" "^7.19.0" "@babel/plugin-transform-dotall-regex@^7.18.6", "@babel/plugin-transform-dotall-regex@^7.4.4": version "7.18.6" @@ -963,10 +963,10 @@ dependencies: "@babel/helper-plugin-utils" "^7.18.6" -"@babel/plugin-transform-runtime@^7.19.1": - version "7.19.1" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.19.1.tgz#a3df2d7312eea624c7889a2dcd37fd1dfd25b2c6" - integrity sha512-2nJjTUFIzBMP/f/miLxEK9vxwW/KUXsdvN4sR//TmuDhe6yU2h57WmIOE12Gng3MDP/xpjUV/ToZRdcf8Yj4fA== +"@babel/plugin-transform-runtime@^7.19.6": + version "7.19.6" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.19.6.tgz#9d2a9dbf4e12644d6f46e5e75bfbf02b5d6e9194" + integrity sha512-PRH37lz4JU156lYFW1p8OxE5i7d6Sl/zV58ooyr+q1J1lnQPyg5tIiXlIwNVhJaY4W3TmOtdc8jqdXQcB1v5Yw== dependencies: "@babel/helper-module-imports" "^7.18.6" "@babel/helper-plugin-utils" "^7.19.0" @@ -1035,12 +1035,12 @@ "@babel/helper-create-regexp-features-plugin" "^7.18.6" "@babel/helper-plugin-utils" "^7.18.6" -"@babel/preset-env@^7.12.11", "@babel/preset-env@^7.19.3": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.19.3.tgz#52cd19abaecb3f176a4ff9cc5e15b7bf06bec754" - integrity sha512-ziye1OTc9dGFOAXSWKUqQblYHNlBOaDl8wzqf2iKXJAltYiR3hKHUKmkt+S9PppW7RQpq4fFCrwwpIDj/f5P4w== +"@babel/preset-env@^7.12.11", "@babel/preset-env@^7.19.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/preset-env/-/preset-env-7.19.4.tgz#4c91ce2e1f994f717efb4237891c3ad2d808c94b" + integrity sha512-5QVOTXUdqTCjQuh2GGtdd7YEhoRXBMVGROAtsBeLGIbIz3obCBIfRMT1I3ZKkMgNzwkyCkftDXSSkHxnfVf4qg== dependencies: - "@babel/compat-data" "^7.19.3" + "@babel/compat-data" "^7.19.4" "@babel/helper-compilation-targets" "^7.19.3" "@babel/helper-plugin-utils" "^7.19.0" "@babel/helper-validator-option" "^7.18.6" @@ -1055,7 +1055,7 @@ "@babel/plugin-proposal-logical-assignment-operators" "^7.18.9" "@babel/plugin-proposal-nullish-coalescing-operator" "^7.18.6" "@babel/plugin-proposal-numeric-separator" "^7.18.6" - "@babel/plugin-proposal-object-rest-spread" "^7.18.9" + "@babel/plugin-proposal-object-rest-spread" "^7.19.4" "@babel/plugin-proposal-optional-catch-binding" "^7.18.6" "@babel/plugin-proposal-optional-chaining" "^7.18.9" "@babel/plugin-proposal-private-methods" "^7.18.6" @@ -1079,10 +1079,10 @@ "@babel/plugin-transform-arrow-functions" "^7.18.6" "@babel/plugin-transform-async-to-generator" "^7.18.6" "@babel/plugin-transform-block-scoped-functions" "^7.18.6" - "@babel/plugin-transform-block-scoping" "^7.18.9" + "@babel/plugin-transform-block-scoping" "^7.19.4" "@babel/plugin-transform-classes" "^7.19.0" "@babel/plugin-transform-computed-properties" "^7.18.9" - "@babel/plugin-transform-destructuring" "^7.18.13" + "@babel/plugin-transform-destructuring" "^7.19.4" "@babel/plugin-transform-dotall-regex" "^7.18.6" "@babel/plugin-transform-duplicate-keys" "^7.18.9" "@babel/plugin-transform-exponentiation-operator" "^7.18.6" @@ -1109,7 +1109,7 @@ "@babel/plugin-transform-unicode-escapes" "^7.18.10" "@babel/plugin-transform-unicode-regex" "^7.18.6" "@babel/preset-modules" "^0.1.5" - "@babel/types" "^7.19.3" + "@babel/types" "^7.19.4" babel-plugin-polyfill-corejs2 "^0.3.3" babel-plugin-polyfill-corejs3 "^0.6.0" babel-plugin-polyfill-regenerator "^0.4.1" @@ -1175,10 +1175,10 @@ core-js-pure "^3.0.0" regenerator-runtime "^0.13.4" -"@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.10.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.0", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.13", "@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.19.0", "@babel/runtime@^7.3.1", "@babel/runtime@^7.4.4", "@babel/runtime@^7.4.5", "@babel/runtime@^7.5.0", "@babel/runtime@^7.5.5", "@babel/runtime@^7.6.3", "@babel/runtime@^7.7.2", "@babel/runtime@^7.7.6", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": - version "7.19.0" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.19.0.tgz#22b11c037b094d27a8a2504ea4dcff00f50e2259" - integrity sha512-eR8Lo9hnDS7tqkO7NsV+mKvCmv5boaXFSZ70DnfhcgiEne8hv9oCEd36Klw74EtizEqLsy4YnW8UWwpBVolHZA== +"@babel/runtime@^7.0.0", "@babel/runtime@^7.1.2", "@babel/runtime@^7.10.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.0", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.13", "@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.17.8", "@babel/runtime@^7.18.3", "@babel/runtime@^7.19.4", "@babel/runtime@^7.3.1", "@babel/runtime@^7.4.4", "@babel/runtime@^7.4.5", "@babel/runtime@^7.5.0", "@babel/runtime@^7.5.5", "@babel/runtime@^7.6.3", "@babel/runtime@^7.7.2", "@babel/runtime@^7.7.6", "@babel/runtime@^7.8.4", "@babel/runtime@^7.8.7", "@babel/runtime@^7.9.2": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.19.4.tgz#a42f814502ee467d55b38dd1c256f53a7b885c78" + integrity sha512-EXpLCrk55f+cYqmHsSR+yD/0gAIMxxA9QK9lnQWzhMCvt+YmoBN7Zx94s++Kv0+unHk39vxNO8t+CMA2WSS3wA== dependencies: regenerator-runtime "^0.13.4" @@ -1191,28 +1191,28 @@ "@babel/parser" "^7.18.10" "@babel/types" "^7.18.10" -"@babel/traverse@^7.1.0", "@babel/traverse@^7.10.3", "@babel/traverse@^7.12.11", "@babel/traverse@^7.12.9", "@babel/traverse@^7.13.0", "@babel/traverse@^7.18.9", "@babel/traverse@^7.19.0", "@babel/traverse@^7.19.3", "@babel/traverse@^7.4.5": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.19.3.tgz#3a3c5348d4988ba60884e8494b0592b2f15a04b4" - integrity sha512-qh5yf6149zhq2sgIXmwjnsvmnNQC2iw70UFjp4olxucKrWd/dvlUsBI88VSLUsnMNF7/vnOiA+nk1+yLoCqROQ== +"@babel/traverse@^7.1.0", "@babel/traverse@^7.10.3", "@babel/traverse@^7.12.11", "@babel/traverse@^7.12.9", "@babel/traverse@^7.13.0", "@babel/traverse@^7.18.9", "@babel/traverse@^7.19.4", "@babel/traverse@^7.19.6", "@babel/traverse@^7.4.5": + version "7.19.6" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.19.6.tgz#7b4c865611df6d99cb131eec2e8ac71656a490dc" + integrity sha512-6l5HrUCzFM04mfbG09AagtYyR2P0B71B1wN7PfSPiksDPz2k5H9CBC1tcZpz2M8OxbKTPccByoOJ22rUKbpmQQ== dependencies: "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.19.3" + "@babel/generator" "^7.19.6" "@babel/helper-environment-visitor" "^7.18.9" "@babel/helper-function-name" "^7.19.0" "@babel/helper-hoist-variables" "^7.18.6" "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.19.3" - "@babel/types" "^7.19.3" + "@babel/parser" "^7.19.6" + "@babel/types" "^7.19.4" debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.10.3", "@babel/types@^7.12.11", "@babel/types@^7.12.7", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.19.3", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": - version "7.19.3" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.19.3.tgz#fc420e6bbe54880bce6779ffaf315f5e43ec9624" - integrity sha512-hGCaQzIY22DJlDh9CH7NOxgKkFjBk0Cw9xDO1Xmh2151ti7wiGfQ3LauXzL4HP1fmFlTX6XjpRETTpUcv7wQLw== +"@babel/types@^7.0.0", "@babel/types@^7.10.3", "@babel/types@^7.12.11", "@babel/types@^7.12.7", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.19.0", "@babel/types@^7.19.4", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": + version "7.19.4" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.19.4.tgz#0dd5c91c573a202d600490a35b33246fed8a41c7" + integrity sha512-M5LK7nAeS6+9j7hAq+b3fQs+pNfUtTGq+yFFfHnauFA8zQtLRfmuipmsKDKKLuyG+wC8ABW43A153YNawNTEtw== dependencies: - "@babel/helper-string-parser" "^7.18.10" + "@babel/helper-string-parser" "^7.19.4" "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" @@ -13992,45 +13992,7 @@ elastic-apm-http-client@11.0.1, elastic-apm-http-client@^11.0.1: semver "^6.3.0" stream-chopper "^3.0.1" -elastic-apm-node@^3.38.0: - version "3.38.0" - resolved "https://registry.yarnpkg.com/elastic-apm-node/-/elastic-apm-node-3.38.0.tgz#4d0dc9279c0e23e09b3b30aa4a9f9aeccfa59cc0" - integrity sha512-/d6YuWFtsfkVRpFD0YJ2rYJVq0rI0PGqG/C+cW1JpMZ4IOU8dA9xzUkxbT0G3B8gpHNug07Xo6bJdQa2oUaFbQ== - dependencies: - "@elastic/ecs-pino-format" "^1.2.0" - "@opentelemetry/api" "^1.1.0" - after-all-results "^2.0.0" - async-cache "^1.1.0" - async-value-promise "^1.1.1" - basic-auth "^2.0.1" - cookie "^0.5.0" - core-util-is "^1.0.2" - elastic-apm-http-client "11.0.1" - end-of-stream "^1.4.4" - error-callsites "^2.0.4" - error-stack-parser "^2.0.6" - escape-string-regexp "^4.0.0" - fast-safe-stringify "^2.0.7" - http-headers "^3.0.2" - is-native "^1.0.1" - lru-cache "^6.0.0" - measured-reporting "^1.51.1" - monitor-event-loop-delay "^1.0.0" - object-filter-sequence "^1.0.0" - object-identity-map "^1.0.2" - original-url "^1.2.3" - pino "^6.11.2" - relative-microtime "^2.0.0" - require-in-the-middle "^5.0.3" - semver "^6.3.0" - set-cookie-serde "^1.0.0" - shallow-clone-shim "^2.0.0" - source-map "^0.8.0-beta.0" - sql-summary "^1.0.1" - traverse "^0.6.6" - unicode-byte-truncate "^1.0.0" - -elastic-apm-node@^3.39.0: +elastic-apm-node@^3.38.0, elastic-apm-node@^3.39.0: version "3.39.0" resolved "https://registry.yarnpkg.com/elastic-apm-node/-/elastic-apm-node-3.39.0.tgz#51ca1dfc11e6b48b53967518461a959ac1623da1" integrity sha512-aNRLDMQreZ+u23HStmppdDNtfS7Z651MWf3wLjw72haCNpGczuXsb4EuBRfJOk0IXWXTYgX1cDy2hiy4PAxlSQ== @@ -24672,7 +24634,7 @@ require-from-string@^2.0.2: resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== -require-in-the-middle@^5.0.3, require-in-the-middle@^5.2.0: +require-in-the-middle@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-5.2.0.tgz#4b71e3cc7f59977100af9beb76bf2d056a5a6de2" integrity sha512-efCx3b+0Z69/LGJmm9Yvi4cqEdxnoGnxYxGxBghkkTTFeXRtTCmmhO0AnAfHz59k957uTSuy8WaHqOs8wbYUWg== @@ -29112,16 +29074,11 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@8.9.0: +ws@8.9.0, ws@>=8.7.0, ws@^8.2.3, ws@^8.4.2: version "8.9.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.9.0.tgz#2a994bb67144be1b53fe2d23c53c028adeb7f45e" integrity sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg== -ws@>=8.7.0, ws@^8.2.3, ws@^8.4.2: - version "8.8.0" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.0.tgz#8e71c75e2f6348dbf8d78005107297056cb77769" - integrity sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ== - ws@^7.2.3, ws@^7.3.1, ws@^7.4.6: version "7.5.6" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b" From ee6aeba68fe2026047ff34fe019ade3fa756cde8 Mon Sep 17 00:00:00 2001 From: Michael Dokolin Date: Mon, 24 Oct 2022 20:08:18 +0200 Subject: [PATCH 06/15] [Expressions] Fix the execution pipeline not to stop on a flaky subexpression (#143852) * Fix the execution pipeline not to stop on a flaky subexpression * Fix the execution pipeline not to stop on an invalid or incorrect value --- .../common/execution/execution.test.ts | 106 ++++++++++++++++++ .../expressions/common/execution/execution.ts | 103 ++++++++++------- 2 files changed, 171 insertions(+), 38 deletions(-) diff --git a/src/plugins/expressions/common/execution/execution.test.ts b/src/plugins/expressions/common/execution/execution.test.ts index a5e03084a6977..2d452b0a640ad 100644 --- a/src/plugins/expressions/common/execution/execution.test.ts +++ b/src/plugins/expressions/common/execution/execution.test.ts @@ -14,6 +14,7 @@ import { parseExpression, ExpressionAstExpression } from '../ast'; import { createUnitTestExecutor } from '../test_helpers'; import { ExpressionFunctionDefinition } from '..'; import { ExecutionContract } from './execution_contract'; +import { ExpressionValueBoxed } from '../expression_types'; beforeAll(() => { if (typeof performance === 'undefined') { @@ -744,6 +745,79 @@ describe('Execution', () => { }); }); }); + + test('continues execution when error state is gone', async () => { + testScheduler.run(({ cold, expectObservable, flush }) => { + const a = 1; + const b = 2; + const c = 3; + const d = 4; + const observable$ = cold('abcd|', { a, b, c, d }); + const flakyFn = jest + .fn() + .mockImplementationOnce((value) => value) + .mockImplementationOnce(() => { + throw new Error('Some error.'); + }) + .mockReturnValueOnce({ type: 'something' }) + .mockImplementationOnce((value) => value); + const spyFn = jest.fn((input, { arg }) => arg); + + const executor = createUnitTestExecutor(); + executor.registerFunction({ + name: 'observable', + args: {}, + help: '', + fn: () => observable$, + }); + executor.registerFunction({ + name: 'flaky', + args: {}, + help: '', + fn: (value) => flakyFn(value), + }); + executor.registerFunction({ + name: 'spy', + args: { + arg: { + help: '', + types: ['number'], + }, + }, + help: '', + fn: (input, args) => spyFn(input, args), + }); + + const result = executor.run('spy arg={observable | flaky}', null, {}); + + expectObservable(result).toBe('abcd|', { + a: { partial: true, result: a }, + b: { + partial: true, + result: { + type: 'error', + error: expect.objectContaining({ message: '[spy] > [flaky] > Some error.' }), + }, + }, + c: { + partial: true, + result: { + type: 'error', + error: expect.objectContaining({ + message: `[spy] > Can not cast 'something' to any of 'number'`, + }), + }, + }, + d: { partial: false, result: d }, + }); + + flush(); + + expect(spyFn).toHaveBeenCalledTimes(2); + expect(spyFn).toHaveBeenNthCalledWith(1, null, { arg: a }); + expect(spyFn).toHaveBeenNthCalledWith(2, null, { arg: d }); + }); + }); }); describe('when arguments are missing', () => { @@ -847,6 +921,38 @@ describe('Execution', () => { }); }); + describe('when arguments are incorrect', () => { + it('when required argument is missing and has not alias, returns error', async () => { + const incorrectArg: ExpressionFunctionDefinition< + 'incorrectArg', + unknown, + { arg: ExpressionValueBoxed<'something'> }, + unknown + > = { + name: 'incorrectArg', + args: { + arg: { + help: '', + required: true, + types: ['something'], + }, + }, + help: '', + fn: jest.fn(), + }; + const executor = createUnitTestExecutor(); + executor.registerFunction(incorrectArg); + const { result } = await lastValueFrom(executor.run('incorrectArg arg="string"', null, {})); + + expect(result).toMatchObject({ + type: 'error', + error: { + message: `[incorrectArg] > Can not cast 'string' to any of 'something'`, + }, + }); + }); + }); + describe('debug mode', () => { test('can execute expression in debug mode', async () => { const execution = createExecution('add val=1 | add val=2 | add val=3', {}, true); diff --git a/src/plugins/expressions/common/execution/execution.ts b/src/plugins/expressions/common/execution/execution.ts index 30016f36d77d6..8edf6d3227c02 100644 --- a/src/plugins/expressions/common/execution/execution.ts +++ b/src/plugins/expressions/common/execution/execution.ts @@ -352,20 +352,30 @@ export class Execution< // actually have `then` or `subscribe` methods which would be treated as a `Promise` // or an `Observable` accordingly. return this.resolveArgs(fn, currentInput, fnArgs).pipe( - tap((args) => this.execution.params.debug && Object.assign(head.debug, { args })), - switchMap((args) => this.invokeFunction(fn, currentInput, args)), - switchMap((output) => (getType(output) === 'error' ? throwError(output) : of(output))), - tap((output) => this.execution.params.debug && Object.assign(head.debug, { output })), - switchMap((output) => this.invokeChain(tail, output)), - catchError((rawError) => { - const error = createError(rawError); - error.error.message = `[${fnName}] > ${error.error.message}`; - - if (this.execution.params.debug) { - Object.assign(head.debug, { error, rawError, success: false }); - } - - return of(error); + switchMap((resolvedArgs) => { + const args$ = isExpressionValueError(resolvedArgs) + ? throwError(resolvedArgs.error) + : of(resolvedArgs); + + return args$.pipe( + tap((args) => this.execution.params.debug && Object.assign(head.debug, { args })), + switchMap((args) => this.invokeFunction(fn, currentInput, args)), + switchMap((output) => + getType(output) === 'error' ? throwError(output) : of(output) + ), + tap((output) => this.execution.params.debug && Object.assign(head.debug, { output })), + switchMap((output) => this.invokeChain(tail, output)), + catchError((rawError) => { + const error = createError(rawError); + error.error.message = `[${fnName}] > ${error.error.message}`; + + if (this.execution.params.debug) { + Object.assign(head.debug, { error, rawError, success: false }); + } + + return of(error); + }) + ); }), finalize(() => { if (this.execution.params.debug) { @@ -449,7 +459,10 @@ export class Execution< } } - throw new Error(`Can not cast '${fromTypeName}' to any of '${toTypeNames.join(', ')}'`); + throw createError({ + name: 'invalid value', + message: `Can not cast '${fromTypeName}' to any of '${toTypeNames.join(', ')}'`, + }); } validate(value: Type, argDef: ExpressionFunctionParameter): void { @@ -459,7 +472,10 @@ export class Execution< }': '${argDef.options.join("', '")}'`; if (argDef.strict) { - throw new Error(message); + throw createError({ + message, + name: 'invalid argument', + }); } this.logger?.warn(message); @@ -471,7 +487,7 @@ export class Execution< fnDef: Fn, input: unknown, argAsts: Record - ): Observable> { + ): Observable | ExpressionValueError> { return defer(() => { const { args: argDefs } = fnDef; @@ -481,7 +497,10 @@ export class Execution< (acc, argAst, argName) => { const argDef = getByAlias(argDefs, argName); if (!argDef) { - throw new Error(`Unknown argument '${argName}' passed to function '${fnDef.name}'`); + throw createError({ + name: 'unknown argument', + message: `Unknown argument '${argName}' passed to function '${fnDef.name}'`, + }); } if (argDef.deprecated && !acc[argDef.name]) { this.logger?.warn(`Argument '${argName}' is deprecated in function '${fnDef.name}'`); @@ -502,7 +521,10 @@ export class Execution< continue; } - throw new Error(`${fnDef.name} requires the "${name}" argument`); + throw createError({ + name: 'missing argument', + message: `${fnDef.name} requires the "${name}" argument`, + }); } // Create the functions to resolve the argument ASTs into values @@ -513,14 +535,17 @@ export class Execution< (subInput = input) => this.interpret(item, subInput).pipe( pluck('result'), - map((output) => { + switchMap((output) => { if (isExpressionValueError(output)) { - throw output.error; + return of(output); } - return this.cast(output, argDefs[argName].types); - }), - tap((value) => this.validate(value, argDefs[argName])) + return of(output).pipe( + map((value) => this.cast(value, argDefs[argName].types)), + tap((value) => this.validate(value, argDefs[argName])), + catchError((error) => of(error)) + ); + }) ) ) ); @@ -531,7 +556,7 @@ export class Execution< return from([{}]); } - const resolvedArgValuesObservable = combineLatest( + return combineLatest( argNames.map((argName) => { const interpretFns = resolveArgFns[argName]; @@ -542,23 +567,25 @@ export class Execution< } return argDefs[argName].resolve - ? combineLatest(interpretFns.map((fn) => fn())) + ? combineLatest(interpretFns.map((fn) => fn())).pipe( + map((values) => values.find(isExpressionValueError) ?? values) + ) : of(interpretFns); }) - ); - - return resolvedArgValuesObservable.pipe( - map((resolvedArgValues) => - mapValues( - // Return an object here because the arguments themselves might actually have a 'then' - // function which would be treated as a promise - zipObject(argNames, resolvedArgValues), - // Just return the last unless the argument definition allows multiple - (argValues, argName) => (argDefs[argName].multi ? argValues : last(argValues)) - ) + ).pipe( + map( + (values) => + values.find(isExpressionValueError) ?? + mapValues( + // Return an object here because the arguments themselves might actually have a 'then' + // function which would be treated as a promise + zipObject(argNames, values as unknown[][]), + // Just return the last unless the argument definition allows multiple + (argValues, argName) => (argDefs[argName].multi ? argValues : last(argValues)) + ) ) ); - }); + }).pipe(catchError((error) => of(error))); } interpret(ast: ExpressionAstNode, input: T): Observable> { From 04075a1c51167200a981aceababf5a2d92bea3ed Mon Sep 17 00:00:00 2001 From: Adam Demjen Date: Mon, 24 Oct 2022 14:34:32 -0400 Subject: [PATCH 07/15] [ML Inference] Reorganize ML modules (#143803) * Relocate delete and detach modules * Relocate inference errors/history modules * Relocate inference processors/history modules --- .../get_ml_inference_errors.test.ts} | 2 +- .../ml_inference/get_ml_inference_errors.ts} | 2 +- ...get_ml_inference_pipeline_history.test.ts} | 4 +-- .../get_ml_inference_pipeline_history.ts} | 2 +- .../delete_ml_inference_pipeline.test.ts | 0 .../delete_ml_inference_pipeline.ts | 2 +- .../detach_ml_inference_pipeline.test.ts | 0 .../detach_ml_inference_pipeline.ts | 4 +-- ..._ml_inference_pipeline_processors.test.ts} | 4 +-- .../get_ml_inference_pipeline_processors.ts} | 6 ++-- .../get_ml_inference_pipelines.test.ts} | 6 +--- .../get_ml_inference_pipelines.ts} | 0 .../routes/enterprise_search/indices.test.ts | 31 ++++++++++--------- .../routes/enterprise_search/indices.ts | 12 +++---- 14 files changed, 37 insertions(+), 38 deletions(-) rename x-pack/plugins/enterprise_search/server/lib/{ml_inference_pipeline/get_inference_errors.test.ts => indices/pipelines/ml_inference/get_ml_inference_errors.test.ts} (97%) rename x-pack/plugins/enterprise_search/server/lib/{ml_inference_pipeline/get_inference_errors.ts => indices/pipelines/ml_inference/get_ml_inference_errors.ts} (96%) rename x-pack/plugins/enterprise_search/server/lib/indices/{fetch_ml_inference_pipeline_history.test.ts => pipelines/ml_inference/get_ml_inference_pipeline_history.test.ts} (95%) rename x-pack/plugins/enterprise_search/server/lib/indices/{fetch_ml_inference_pipeline_history.ts => pipelines/ml_inference/get_ml_inference_pipeline_history.ts} (94%) rename x-pack/plugins/enterprise_search/server/lib/{ => indices}/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.test.ts (100%) rename x-pack/plugins/enterprise_search/server/lib/{ => indices}/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts (97%) rename x-pack/plugins/enterprise_search/server/lib/{ => indices}/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.test.ts (100%) rename x-pack/plugins/enterprise_search/server/lib/{ => indices}/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts (96%) rename x-pack/plugins/enterprise_search/server/lib/indices/{fetch_ml_inference_pipeline_processors.test.ts => pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.test.ts} (99%) rename x-pack/plugins/enterprise_search/server/lib/indices/{fetch_ml_inference_pipeline_processors.ts => pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.ts} (96%) rename x-pack/plugins/enterprise_search/server/lib/{ml_inference_pipeline/get_inference_pipelines.test.ts => pipelines/ml_inference/get_ml_inference_pipelines.test.ts} (95%) rename x-pack/plugins/enterprise_search/server/lib/{ml_inference_pipeline/get_inference_pipelines.ts => pipelines/ml_inference/get_ml_inference_pipelines.ts} (100%) diff --git a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_errors.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_errors.test.ts similarity index 97% rename from x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_errors.test.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_errors.test.ts index d9939afedaa5e..da72136402aca 100644 --- a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_errors.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_errors.test.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; -import { getMlInferenceErrors } from './get_inference_errors'; +import { getMlInferenceErrors } from './get_ml_inference_errors'; describe('getMlInferenceErrors', () => { const indexName = 'my-index'; diff --git a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_errors.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_errors.ts similarity index 96% rename from x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_errors.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_errors.ts index 09adc656d576f..1556b478de21e 100644 --- a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_errors.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_errors.ts @@ -12,7 +12,7 @@ import { import { ElasticsearchClient } from '@kbn/core/server'; -import { MlInferenceError } from '../../../common/types/pipelines'; +import { MlInferenceError } from '../../../../../common/types/pipelines'; export interface ErrorAggregationBucket extends AggregationsStringRareTermsBucketKeys { max_error_timestamp: { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_history.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history.test.ts similarity index 95% rename from x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_history.test.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history.test.ts index 61d11b50ae489..461bd0e882614 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_history.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history.test.ts @@ -11,9 +11,9 @@ import { } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { MlInferenceHistoryResponse } from '../../../common/types/pipelines'; +import { MlInferenceHistoryResponse } from '../../../../../common/types/pipelines'; -import { fetchMlInferencePipelineHistory } from './fetch_ml_inference_pipeline_history'; +import { fetchMlInferencePipelineHistory } from './get_ml_inference_pipeline_history'; const DEFAULT_RESPONSE: SearchResponse = { _shards: { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_history.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history.ts similarity index 94% rename from x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_history.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history.ts index 70cbe687590ca..f6a01ec2b3e87 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_history.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history.ts @@ -12,7 +12,7 @@ import { import { ElasticsearchClient } from '@kbn/core/server'; -import { MlInferenceHistoryResponse } from '../../../common/types/pipelines'; +import { MlInferenceHistoryResponse } from '../../../../../common/types/pipelines'; export const fetchMlInferencePipelineHistory = async ( client: ElasticsearchClient, diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.test.ts similarity index 100% rename from x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.test.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.test.ts diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts similarity index 97% rename from x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts index 19654d0b2e936..6cb74d75dd6ce 100644 --- a/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient } from '@kbn/core/server'; -import { DeleteMlInferencePipelineResponse } from '../../../../../common/types/pipelines'; +import { DeleteMlInferencePipelineResponse } from '../../../../../../common/types/pipelines'; import { detachMlInferencePipeline } from './detach_ml_inference_pipeline'; diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.test.ts similarity index 100% rename from x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.test.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.test.ts diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts similarity index 96% rename from x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts index 02d6c328a8e47..28f099fad6fa1 100644 --- a/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline.ts @@ -8,9 +8,9 @@ import { IngestPutPipelineRequest } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { DeleteMlInferencePipelineResponse } from '../../../../../common/types/pipelines'; +import { DeleteMlInferencePipelineResponse } from '../../../../../../common/types/pipelines'; -import { getInferencePipelineNameFromIndexName } from '../../../../utils/ml_inference_pipeline_utils'; +import { getInferencePipelineNameFromIndexName } from '../../../../../utils/ml_inference_pipeline_utils'; export const detachMlInferencePipeline = async ( indexName: string, diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.test.ts similarity index 99% rename from x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.test.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.test.ts index 941ef42aaa448..1a20551ab7911 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.test.ts @@ -9,7 +9,7 @@ import { errors } from '@elastic/elasticsearch'; import { ElasticsearchClient } from '@kbn/core/server'; import { MlTrainedModels } from '@kbn/ml-plugin/server'; -import { InferencePipeline, TrainedModelState } from '../../../common/types/pipelines'; +import { InferencePipeline, TrainedModelState } from '../../../../../../common/types/pipelines'; import { fetchAndAddTrainedModelData, @@ -19,7 +19,7 @@ import { fetchMlInferencePipelineProcessors, fetchPipelineProcessorInferenceData, InferencePipelineData, -} from './fetch_ml_inference_pipeline_processors'; +} from './get_ml_inference_pipeline_processors'; const mockGetPipeline = { 'my-index@ml-inference': { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.ts similarity index 96% rename from x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts rename to x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.ts index 1eabe28eb78b1..12902b896e0d8 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors.ts @@ -9,9 +9,9 @@ import { IngestGetPipelineResponse } from '@elastic/elasticsearch/lib/api/types' import { ElasticsearchClient } from '@kbn/core/server'; import { MlTrainedModels } from '@kbn/ml-plugin/server'; -import { getMlModelTypesForModelConfig } from '../../../common/ml_inference_pipeline'; -import { InferencePipeline, TrainedModelState } from '../../../common/types/pipelines'; -import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils'; +import { getMlModelTypesForModelConfig } from '../../../../../../common/ml_inference_pipeline'; +import { InferencePipeline, TrainedModelState } from '../../../../../../common/types/pipelines'; +import { getInferencePipelineNameFromIndexName } from '../../../../../utils/ml_inference_pipeline_utils'; export type InferencePipelineData = InferencePipeline & { trainedModelName: string; diff --git a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_pipelines.test.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/get_ml_inference_pipelines.test.ts similarity index 95% rename from x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_pipelines.test.ts rename to x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/get_ml_inference_pipelines.test.ts index 45953166667a5..b05a9bb15d5e8 100644 --- a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_pipelines.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/get_ml_inference_pipelines.test.ts @@ -9,11 +9,7 @@ import { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; import { MlTrainedModels } from '@kbn/ml-plugin/server'; -import { getMlInferencePipelines } from './get_inference_pipelines'; - -jest.mock('../indices/fetch_ml_inference_pipeline_processors', () => ({ - getMlModelConfigsForModelIds: jest.fn(), -})); +import { getMlInferencePipelines } from './get_ml_inference_pipelines'; describe('getMlInferencePipelines', () => { const mockClient = { diff --git a/x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_pipelines.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/get_ml_inference_pipelines.ts similarity index 100% rename from x-pack/plugins/enterprise_search/server/lib/ml_inference_pipeline/get_inference_pipelines.ts rename to x-pack/plugins/enterprise_search/server/lib/pipelines/ml_inference/get_ml_inference_pipelines.ts diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts index 1ad901330caff..52039cc48173b 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts @@ -14,12 +14,15 @@ import { SharedServices } from '@kbn/ml-plugin/server/shared_services'; import { ErrorCode } from '../../../common/types/error_codes'; -jest.mock('../../lib/indices/fetch_ml_inference_pipeline_history', () => ({ +jest.mock('../../lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history', () => ({ fetchMlInferencePipelineHistory: jest.fn(), })); -jest.mock('../../lib/indices/fetch_ml_inference_pipeline_processors', () => ({ - fetchMlInferencePipelineProcessors: jest.fn(), -})); +jest.mock( + '../../lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors', + () => ({ + fetchMlInferencePipelineProcessors: jest.fn(), + }) +); jest.mock( '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline', () => ({ @@ -33,13 +36,13 @@ jest.mock( }) ); jest.mock( - '../../lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline', + '../../lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline', () => ({ deleteMlInferencePipeline: jest.fn(), }) ); jest.mock( - '../../lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline', + '../../lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline', () => ({ detachMlInferencePipeline: jest.fn(), }) @@ -47,22 +50,22 @@ jest.mock( jest.mock('../../lib/indices/exists_index', () => ({ indexOrAliasExists: jest.fn(), })); -jest.mock('../../lib/ml_inference_pipeline/get_inference_errors', () => ({ +jest.mock('../../lib/indices/pipelines/ml_inference/get_ml_inference_errors', () => ({ getMlInferenceErrors: jest.fn(), })); -jest.mock('../../lib/ml_inference_pipeline/get_inference_pipelines', () => ({ +jest.mock('../../lib/pipelines/ml_inference/get_ml_inference_pipelines', () => ({ getMlInferencePipelines: jest.fn(), })); import { indexOrAliasExists } from '../../lib/indices/exists_index'; -import { fetchMlInferencePipelineHistory } from '../../lib/indices/fetch_ml_inference_pipeline_history'; -import { fetchMlInferencePipelineProcessors } from '../../lib/indices/fetch_ml_inference_pipeline_processors'; +import { getMlInferenceErrors } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_errors'; +import { fetchMlInferencePipelineHistory } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history'; import { attachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/attach_ml_pipeline'; import { createAndReferenceMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline'; -import { getMlInferenceErrors } from '../../lib/ml_inference_pipeline/get_inference_errors'; -import { getMlInferencePipelines } from '../../lib/ml_inference_pipeline/get_inference_pipelines'; -import { deleteMlInferencePipeline } from '../../lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline'; -import { detachMlInferencePipeline } from '../../lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline'; +import { deleteMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline'; +import { detachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline'; +import { fetchMlInferencePipelineProcessors } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors'; +import { getMlInferencePipelines } from '../../lib/pipelines/ml_inference/get_ml_inference_pipelines'; import { ElasticsearchResponseError } from '../../utils/identify_exceptions'; import { registerIndexRoutes } from './indices'; diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts index ee497ba671faf..02a7dd528f872 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts @@ -31,18 +31,18 @@ import { createIndex } from '../../lib/indices/create_index'; import { indexOrAliasExists } from '../../lib/indices/exists_index'; import { fetchIndex } from '../../lib/indices/fetch_index'; import { fetchIndices } from '../../lib/indices/fetch_indices'; -import { fetchMlInferencePipelineHistory } from '../../lib/indices/fetch_ml_inference_pipeline_history'; -import { fetchMlInferencePipelineProcessors } from '../../lib/indices/fetch_ml_inference_pipeline_processors'; import { generateApiKey } from '../../lib/indices/generate_api_key'; +import { getMlInferenceErrors } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_errors'; +import { fetchMlInferencePipelineHistory } from '../../lib/indices/pipelines/ml_inference/get_ml_inference_pipeline_history'; import { attachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/attach_ml_pipeline'; import { createAndReferenceMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline'; -import { getMlInferenceErrors } from '../../lib/ml_inference_pipeline/get_inference_errors'; -import { getMlInferencePipelines } from '../../lib/ml_inference_pipeline/get_inference_pipelines'; +import { deleteMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline'; +import { detachMlInferencePipeline } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline'; +import { fetchMlInferencePipelineProcessors } from '../../lib/indices/pipelines/ml_inference/pipeline_processors/get_ml_inference_pipeline_processors'; import { createIndexPipelineDefinitions } from '../../lib/pipelines/create_pipeline_definitions'; import { getCustomPipelines } from '../../lib/pipelines/get_custom_pipelines'; import { getPipeline } from '../../lib/pipelines/get_pipeline'; -import { deleteMlInferencePipeline } from '../../lib/pipelines/ml_inference/pipeline_processors/delete_ml_inference_pipeline'; -import { detachMlInferencePipeline } from '../../lib/pipelines/ml_inference/pipeline_processors/detach_ml_inference_pipeline'; +import { getMlInferencePipelines } from '../../lib/pipelines/ml_inference/get_ml_inference_pipelines'; import { RouteDependencies } from '../../plugin'; import { createError } from '../../utils/create_error'; import { elasticsearchErrorHandler } from '../../utils/elasticsearch_error_handler'; From 511f95a16a76eaf1609b151f3930fd66d59f2599 Mon Sep 17 00:00:00 2001 From: Lukas Olson Date: Mon, 24 Oct 2022 12:21:35 -0700 Subject: [PATCH 08/15] [KQL] Use term queries for keyword fields (#143599) --- .../kbn-es-query/src/kuery/functions/is.test.ts | 17 +++++++++++++++++ packages/kbn-es-query/src/kuery/functions/is.ts | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/kbn-es-query/src/kuery/functions/is.test.ts b/packages/kbn-es-query/src/kuery/functions/is.test.ts index f36ea0d17dc8c..bde7e956f0c43 100644 --- a/packages/kbn-es-query/src/kuery/functions/is.test.ts +++ b/packages/kbn-es-query/src/kuery/functions/is.test.ts @@ -361,6 +361,23 @@ describe('kuery functions', () => { expect(result).toEqual(expected); }); + + test('should use a term query for keyword fields', () => { + const node = nodeTypes.function.buildNode('is', 'machine.os.keyword', 'Win 7'); + const result = is.toElasticsearchQuery(node, indexPattern); + expect(result).toEqual({ + bool: { + should: [ + { + term: { + 'machine.os.keyword': 'Win 7', + }, + }, + ], + minimum_should_match: 1, + }, + }); + }); }); }); }); diff --git a/packages/kbn-es-query/src/kuery/functions/is.ts b/packages/kbn-es-query/src/kuery/functions/is.ts index f2db74857b02e..5493a3a6072e6 100644 --- a/packages/kbn-es-query/src/kuery/functions/is.ts +++ b/packages/kbn-es-query/src/kuery/functions/is.ts @@ -100,6 +100,7 @@ export function toElasticsearchQuery( } const queries = fields!.reduce((accumulator: any, field: DataViewFieldBase) => { + const isKeywordField = field.esTypes?.length === 1 && field.esTypes.includes('keyword'); const wrapWithNestedQuery = (query: any) => { // Wildcards can easily include nested and non-nested fields. There isn't a good way to let // users handle this themselves so we automatically add nested queries in this scenario. @@ -142,7 +143,7 @@ export function toElasticsearchQuery( }), ]; } else if (wildcard.isNode(valueArg)) { - const query = field.esTypes?.includes('keyword') + const query = isKeywordField ? { wildcard: { [field.name]: value, @@ -177,7 +178,7 @@ export function toElasticsearchQuery( }), ]; } else { - const queryType = type === 'phrase' ? 'match_phrase' : 'match'; + const queryType = isKeywordField ? 'term' : type === 'phrase' ? 'match_phrase' : 'match'; return [ ...accumulator, wrapWithNestedQuery({ From 670fe25673a0e0a5861b391e0cee8a47291a8bf4 Mon Sep 17 00:00:00 2001 From: Catherine Liu Date: Mon, 24 Oct 2022 12:33:36 -0700 Subject: [PATCH 09/15] [Canvas] Use data views service (#139610) * Use data views service in Canvas * Use getIndexPattern instead of title property * Fix ts errors * Remove console log statement --- .../functions/browser/escount.ts | 16 ++-- .../functions/browser/esdocs.ts | 1 + .../uis/datasources/esdocs.js | 5 +- x-pack/plugins/canvas/i18n/ui.ts | 4 +- x-pack/plugins/canvas/kibana.json | 1 + .../datasource/datasource_component.js | 9 ++- .../es_data_view_select.component.tsx} | 32 +++++--- .../es_data_view_select.tsx | 51 ++++++++++++ .../index.tsx | 4 +- .../components/es_field_select/index.tsx | 6 +- .../es_fields_select/es_fields_select.tsx | 5 +- .../es_index_select/es_index_select.tsx | 48 ----------- .../plugins/canvas/public/lib/es_service.ts | 79 ------------------- .../canvas/public/services/data_views.ts | 14 ++++ .../plugins/canvas/public/services/index.ts | 3 + .../public/services/kibana/data_views.ts | 50 ++++++++++++ .../canvas/public/services/kibana/index.ts | 3 + .../public/services/stubs/data_views.ts | 26 ++++++ .../canvas/public/services/stubs/index.ts | 3 + .../canvas/storybook/canvas_webpack.ts | 4 - .../canvas/storybook/storyshots.test.tsx | 5 -- 21 files changed, 201 insertions(+), 168 deletions(-) rename x-pack/plugins/canvas/public/components/{es_index_select/es_index_select.component.tsx => es_data_view_select/es_data_view_select.component.tsx} (50%) create mode 100644 x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.tsx rename x-pack/plugins/canvas/public/components/{es_index_select => es_data_view_select}/index.tsx (62%) delete mode 100644 x-pack/plugins/canvas/public/components/es_index_select/es_index_select.tsx delete mode 100644 x-pack/plugins/canvas/public/lib/es_service.ts create mode 100644 x-pack/plugins/canvas/public/services/data_views.ts create mode 100644 x-pack/plugins/canvas/public/services/kibana/data_views.ts create mode 100644 x-pack/plugins/canvas/public/services/stubs/data_views.ts diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts index 7bdb13cff93c0..b324d2f11a0a8 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/escount.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { lastValueFrom } from 'rxjs'; + import { ExpressionFunctionDefinition, ExpressionValueFilter, @@ -12,10 +14,8 @@ import { // @ts-expect-error untyped local import { buildESRequest } from '../../../common/lib/request/build_es_request'; - -import { searchService } from '../../../public/services'; - import { getFunctionHelp } from '../../../i18n'; +import { searchService } from '../../../public/services'; interface Arguments { index: string | null; @@ -46,6 +46,7 @@ export function escount(): ExpressionFunctionDefinition< }, index: { types: ['string'], + aliases: ['dataView'], default: '_all', help: argHelp.index, }, @@ -83,12 +84,9 @@ export function escount(): ExpressionFunctionDefinition< }, }; - return search - .search(req) - .toPromise() - .then((resp: any) => { - return resp.rawResponse.hits.total; - }); + return lastValueFrom(search.search(req)).then((resp: any) => { + return resp.rawResponse.hits.total; + }); }, }; } diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts index 83225fcafb8d8..763b1839ec3b8 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/browser/esdocs.ts @@ -70,6 +70,7 @@ export function esdocs(): ExpressionFunctionDefinition< }, index: { types: ['string'], + aliases: ['dataView'], default: '_all', help: argHelp.index, }, diff --git a/x-pack/plugins/canvas/canvas_plugin_src/uis/datasources/esdocs.js b/x-pack/plugins/canvas/canvas_plugin_src/uis/datasources/esdocs.js index 150b7c0616887..ae0af7b2297f5 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/uis/datasources/esdocs.js +++ b/x-pack/plugins/canvas/canvas_plugin_src/uis/datasources/esdocs.js @@ -20,7 +20,7 @@ import { import { getSimpleArg, setSimpleArg } from '../../../public/lib/arg_helpers'; import { ESFieldsSelect } from '../../../public/components/es_fields_select'; import { ESFieldSelect } from '../../../public/components/es_field_select'; -import { ESIndexSelect } from '../../../public/components/es_index_select'; +import { ESDataViewSelect } from '../../../public/components/es_data_view_select'; import { templateFromReactComponent } from '../../../public/lib/template_from_react_component'; import { DataSourceStrings, LUCENE_QUERY_URL } from '../../../i18n'; @@ -29,6 +29,7 @@ const { ESDocs: strings } = DataSourceStrings; const EsdocsDatasource = ({ args, updateArgs, defaultIndex }) => { const setArg = useCallback( (name, value) => { + console.log({ name, value }); updateArgs && updateArgs({ ...args, @@ -94,7 +95,7 @@ const EsdocsDatasource = ({ args, updateArgs, defaultIndex }) => { helpText={strings.getIndexLabel()} display="rowCompressed" > - setArg('index', index)} /> + setArg('index', index)} /> i18n.translate('xpack.canvas.uis.dataSources.esdocs.indexTitle', { - defaultMessage: 'Index', + defaultMessage: 'Data view', }), getIndexLabel: () => i18n.translate('xpack.canvas.uis.dataSources.esdocs.indexLabel', { - defaultMessage: 'Enter an index name or select a data view', + defaultMessage: 'Select a data view or enter an index name.', }), getQueryTitle: () => i18n.translate('xpack.canvas.uis.dataSources.esdocs.queryTitle', { diff --git a/x-pack/plugins/canvas/kibana.json b/x-pack/plugins/canvas/kibana.json index 48f4d9a6b0d58..f63c3522a8df7 100644 --- a/x-pack/plugins/canvas/kibana.json +++ b/x-pack/plugins/canvas/kibana.json @@ -14,6 +14,7 @@ "bfetch", "charts", "data", + "dataViews", "embeddable", "expressionError", "expressionImage", diff --git a/x-pack/plugins/canvas/public/components/datasource/datasource_component.js b/x-pack/plugins/canvas/public/components/datasource/datasource_component.js index 8e2176b845bfa..1a357b6722c71 100644 --- a/x-pack/plugins/canvas/public/components/datasource/datasource_component.js +++ b/x-pack/plugins/canvas/public/components/datasource/datasource_component.js @@ -20,7 +20,7 @@ import { import { isEqual } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { getDefaultIndex } from '../../lib/es_service'; +import { pluginServices } from '../../services'; import { DatasourceSelector } from './datasource_selector'; import { DatasourcePreview } from './datasource_preview'; @@ -67,7 +67,12 @@ export class DatasourceComponent extends PureComponent { state = { defaultIndex: '' }; componentDidMount() { - getDefaultIndex().then((defaultIndex) => this.setState({ defaultIndex })); + pluginServices + .getServices() + .dataViews.getDefaultDataView() + .then((defaultDataView) => { + this.setState({ defaultIndex: defaultDataView.title }); + }); } componentDidUpdate(prevProps) { diff --git a/x-pack/plugins/canvas/public/components/es_index_select/es_index_select.component.tsx b/x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.component.tsx similarity index 50% rename from x-pack/plugins/canvas/public/components/es_index_select/es_index_select.component.tsx rename to x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.component.tsx index 636b19092123a..da8b4cda4c17c 100644 --- a/x-pack/plugins/canvas/public/components/es_index_select/es_index_select.component.tsx +++ b/x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.component.tsx @@ -7,37 +7,47 @@ import React, { FocusEventHandler } from 'react'; import { EuiComboBox } from '@elastic/eui'; +import { DataView } from '@kbn/data-views-plugin/common'; -export interface ESIndexSelectProps { +type DataViewOption = Pick; + +export interface ESDataViewSelectProps { loading: boolean; value: string; - indices: string[]; - onChange: (index: string) => void; + dataViews: DataViewOption[]; + onChange: (string: string) => void; onBlur: FocusEventHandler | undefined; onFocus: FocusEventHandler | undefined; } const defaultIndex = '_all'; +const defaultOption = { value: defaultIndex, label: defaultIndex }; -export const ESIndexSelect: React.FunctionComponent = ({ +export const ESDataViewSelect: React.FunctionComponent = ({ value = defaultIndex, loading, - indices, + dataViews, onChange, onFocus, onBlur, }) => { - const selectedOption = value !== defaultIndex ? [{ label: value }] : []; - const options = indices.map((index) => ({ label: index })); + const selectedDataView = dataViews.find((view) => value === view.title) as DataView; + + const selectedOption = selectedDataView + ? { value: selectedDataView.title, label: selectedDataView.name } + : { value, label: value }; + const options = dataViews.map(({ name, title }) => ({ value: title, label: name })); return ( onChange(index?.label ?? defaultIndex)} + selectedOptions={[selectedOption]} + onChange={([view]) => { + onChange(view.value || defaultOption.value); + }} onSearchChange={(searchValue) => { // resets input when user starts typing if (searchValue) { - onChange(defaultIndex); + onChange(defaultOption.value); } }} onBlur={onBlur} @@ -46,7 +56,7 @@ export const ESIndexSelect: React.FunctionComponent = ({ options={options} singleSelection={{ asPlainText: true }} isClearable={false} - onCreateOption={(input) => onChange(input || defaultIndex)} + onCreateOption={(input) => onChange(input || defaultOption.value)} compressed /> ); diff --git a/x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.tsx b/x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.tsx new file mode 100644 index 0000000000000..83e21b8ba2e6f --- /dev/null +++ b/x-pack/plugins/canvas/public/components/es_data_view_select/es_data_view_select.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataView } from '@kbn/data-views-plugin/common'; +import { sortBy } from 'lodash'; +import React, { FC, useRef, useState } from 'react'; +import useEffectOnce from 'react-use/lib/useEffectOnce'; +import { useDataViewsService } from '../../services'; +import { + ESDataViewSelect as Component, + ESDataViewSelectProps as Props, +} from './es_data_view_select.component'; + +type ESDataViewSelectProps = Omit; + +export const ESDataViewSelect: FC = (props) => { + const { value, onChange } = props; + + const [dataViews, setDataViews] = useState>>([]); + const [loading, setLoading] = useState(true); + const mounted = useRef(true); + const { getDataViews } = useDataViewsService(); + + useEffectOnce(() => { + getDataViews().then((newDataViews) => { + if (!mounted.current) { + return; + } + + if (!newDataViews) { + newDataViews = []; + } + + setLoading(false); + setDataViews(sortBy(newDataViews, 'name')); + if (!value && newDataViews.length) { + onChange(newDataViews[0].title); + } + }); + + return () => { + mounted.current = false; + }; + }); + + return ; +}; diff --git a/x-pack/plugins/canvas/public/components/es_index_select/index.tsx b/x-pack/plugins/canvas/public/components/es_data_view_select/index.tsx similarity index 62% rename from x-pack/plugins/canvas/public/components/es_index_select/index.tsx rename to x-pack/plugins/canvas/public/components/es_data_view_select/index.tsx index 29a19e8770606..870a6b3aff1aa 100644 --- a/x-pack/plugins/canvas/public/components/es_index_select/index.tsx +++ b/x-pack/plugins/canvas/public/components/es_data_view_select/index.tsx @@ -5,5 +5,5 @@ * 2.0. */ -export { ESIndexSelect } from './es_index_select'; -export { ESIndexSelect as ESIndexSelectComponent } from './es_index_select.component'; +export { ESDataViewSelect } from './es_data_view_select'; +export { ESDataViewSelect as ESDataViewSelectComponent } from './es_data_view_select.component'; diff --git a/x-pack/plugins/canvas/public/components/es_field_select/index.tsx b/x-pack/plugins/canvas/public/components/es_field_select/index.tsx index 8c0baea681731..653eec22d77d9 100644 --- a/x-pack/plugins/canvas/public/components/es_field_select/index.tsx +++ b/x-pack/plugins/canvas/public/components/es_field_select/index.tsx @@ -6,7 +6,7 @@ */ import React, { useState, useEffect, useRef } from 'react'; -import { getFields } from '../../lib/es_service'; +import { useDataViewsService } from '../../services'; import { ESFieldSelect as Component, ESFieldSelectProps as Props } from './es_field_select'; type ESFieldSelectProps = Omit; @@ -15,15 +15,17 @@ export const ESFieldSelect: React.FunctionComponent = (props const { index, value, onChange } = props; const [fields, setFields] = useState([]); const loadingFields = useRef(false); + const { getFields } = useDataViewsService(); useEffect(() => { loadingFields.current = true; + getFields(index) .then((newFields) => setFields(newFields || [])) .finally(() => { loadingFields.current = false; }); - }, [index]); + }, [index, getFields]); useEffect(() => { if (!loadingFields.current && value && !fields.includes(value)) { diff --git a/x-pack/plugins/canvas/public/components/es_fields_select/es_fields_select.tsx b/x-pack/plugins/canvas/public/components/es_fields_select/es_fields_select.tsx index 2f94b154edab2..c929203f9e094 100644 --- a/x-pack/plugins/canvas/public/components/es_fields_select/es_fields_select.tsx +++ b/x-pack/plugins/canvas/public/components/es_fields_select/es_fields_select.tsx @@ -8,7 +8,7 @@ import React, { useState, useEffect, useRef } from 'react'; import { isEqual } from 'lodash'; import usePrevious from 'react-use/lib/usePrevious'; -import { getFields } from '../../lib/es_service'; +import { useDataViewsService } from '../../services'; import { ESFieldsSelect as Component, ESFieldsSelectProps as Props, @@ -21,6 +21,7 @@ export const ESFieldsSelect: React.FunctionComponent = (pro const [fields, setFields] = useState([]); const prevIndex = usePrevious(index); const mounted = useRef(true); + const { getFields } = useDataViewsService(); useEffect(() => { if (prevIndex !== index) { @@ -36,7 +37,7 @@ export const ESFieldsSelect: React.FunctionComponent = (pro } }); } - }, [fields, index, onChange, prevIndex, selected]); + }, [fields, index, onChange, prevIndex, selected, getFields]); useEffect( () => () => { diff --git a/x-pack/plugins/canvas/public/components/es_index_select/es_index_select.tsx b/x-pack/plugins/canvas/public/components/es_index_select/es_index_select.tsx deleted file mode 100644 index 7d2e87902d2d1..0000000000000 --- a/x-pack/plugins/canvas/public/components/es_index_select/es_index_select.tsx +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { useRef, useState } from 'react'; -import useEffectOnce from 'react-use/lib/useEffectOnce'; -import { getIndices } from '../../lib/es_service'; -import { - ESIndexSelect as Component, - ESIndexSelectProps as Props, -} from './es_index_select.component'; - -type ESIndexSelectProps = Omit; - -export const ESIndexSelect: React.FunctionComponent = (props) => { - const { value, onChange } = props; - - const [indices, setIndices] = useState([]); - const [loading, setLoading] = useState(true); - const mounted = useRef(true); - - useEffectOnce(() => { - getIndices().then((newIndices) => { - if (!mounted.current) { - return; - } - - if (!newIndices) { - newIndices = []; - } - - setLoading(false); - setIndices(newIndices.sort()); - if (!value && newIndices.length) { - onChange(newIndices[0]); - } - }); - - return () => { - mounted.current = false; - }; - }); - - return ; -}; diff --git a/x-pack/plugins/canvas/public/lib/es_service.ts b/x-pack/plugins/canvas/public/lib/es_service.ts deleted file mode 100644 index 9d558243c9421..0000000000000 --- a/x-pack/plugins/canvas/public/lib/es_service.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// TODO - clint: convert to service abstraction - -import { API_ROUTE } from '../../common/lib/constants'; -import { fetch } from '../../common/lib/fetch'; -import { ErrorStrings } from '../../i18n'; -import { pluginServices } from '../services'; - -const { esService: strings } = ErrorStrings; - -const getApiPath = function () { - const platformService = pluginServices.getServices().platform; - const basePath = platformService.getBasePath(); - return basePath + API_ROUTE; -}; - -const getSavedObjectsClient = function () { - const platformService = pluginServices.getServices().platform; - return platformService.getSavedObjectsClient(); -}; - -const getAdvancedSettings = function () { - const platformService = pluginServices.getServices().platform; - return platformService.getUISettings(); -}; - -export const getFields = (index = '_all') => { - return fetch - .get(`${getApiPath()}/es_fields?index=${index}`) - .then(({ data: mapping }: { data: object }) => - Object.keys(mapping) - .filter((field) => !field.startsWith('_')) // filters out meta fields - .sort() - ) - .catch((err: Error) => { - const notifyService = pluginServices.getServices().notify; - notifyService.error(err, { - title: strings.getFieldsFetchErrorMessage(index), - }); - }); -}; - -export const getIndices = () => - getSavedObjectsClient() - .find<{ title: string }>({ - type: 'index-pattern', - fields: ['title'], - searchFields: ['title'], - perPage: 1000, - }) - .then((resp) => { - return resp.savedObjects.map((savedObject) => { - return savedObject.attributes.title; - }); - }) - .catch((err: Error) => { - const notifyService = pluginServices.getServices().notify; - notifyService.error(err, { title: strings.getIndicesFetchErrorMessage() }); - }); - -export const getDefaultIndex = () => { - const defaultIndexId = getAdvancedSettings().get('defaultIndex'); - - return defaultIndexId - ? getSavedObjectsClient() - .get<{ title: string }>('index-pattern', defaultIndexId) - .then((defaultIndex) => defaultIndex.attributes.title) - .catch((err) => { - const notifyService = pluginServices.getServices().notify; - notifyService.error(err, { title: strings.getDefaultIndexFetchErrorMessage() }); - }) - : Promise.resolve(''); -}; diff --git a/x-pack/plugins/canvas/public/services/data_views.ts b/x-pack/plugins/canvas/public/services/data_views.ts new file mode 100644 index 0000000000000..86faa87bfaa59 --- /dev/null +++ b/x-pack/plugins/canvas/public/services/data_views.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataView } from '@kbn/data-views-plugin/common'; + +export interface CanvasDataViewsService { + getFields: (index: string) => Promise; + getDataViews: () => Promise>>; + getDefaultDataView: () => Promise | undefined>; +} diff --git a/x-pack/plugins/canvas/public/services/index.ts b/x-pack/plugins/canvas/public/services/index.ts index 2c01378851ee5..a6ebc865fbd3f 100644 --- a/x-pack/plugins/canvas/public/services/index.ts +++ b/x-pack/plugins/canvas/public/services/index.ts @@ -10,6 +10,7 @@ export * from './legacy'; import { PluginServices } from '@kbn/presentation-util-plugin/public'; import { CanvasCustomElementService } from './custom_element'; +import { CanvasDataViewsService } from './data_views'; import { CanvasEmbeddablesService } from './embeddables'; import { CanvasExpressionsService } from './expressions'; import { CanvasFiltersService } from './filters'; @@ -23,6 +24,7 @@ import { CanvasWorkpadService } from './workpad'; export interface CanvasPluginServices { customElement: CanvasCustomElementService; + dataViews: CanvasDataViewsService; embeddables: CanvasEmbeddablesService; expressions: CanvasExpressionsService; filters: CanvasFiltersService; @@ -39,6 +41,7 @@ export const pluginServices = new PluginServices(); export const useCustomElementService = () => (() => pluginServices.getHooks().customElement.useService())(); +export const useDataViewsService = () => (() => pluginServices.getHooks().dataViews.useService())(); export const useEmbeddablesService = () => (() => pluginServices.getHooks().embeddables.useService())(); export const useExpressionsService = () => diff --git a/x-pack/plugins/canvas/public/services/kibana/data_views.ts b/x-pack/plugins/canvas/public/services/kibana/data_views.ts new file mode 100644 index 0000000000000..99e95c2e9dc4f --- /dev/null +++ b/x-pack/plugins/canvas/public/services/kibana/data_views.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataView } from '@kbn/data-views-plugin/public'; +import { KibanaPluginServiceFactory } from '@kbn/presentation-util-plugin/public'; +import { ErrorStrings } from '../../../i18n'; +import { CanvasStartDeps } from '../../plugin'; +import { CanvasDataViewsService } from '../data_views'; +import { CanvasNotifyService } from '../notify'; + +const { esService: strings } = ErrorStrings; + +export type DataViewsServiceFactory = KibanaPluginServiceFactory< + CanvasDataViewsService, + CanvasStartDeps, + { + notify: CanvasNotifyService; + } +>; + +export const dataViewsServiceFactory: DataViewsServiceFactory = ({ startPlugins }, { notify }) => ({ + getDataViews: async () => { + try { + const dataViews = await startPlugins.data.dataViews.getIdsWithTitle(); + return dataViews.map(({ id, name, title }) => ({ id, name, title } as DataView)); + } catch (e) { + notify.error(e, { title: strings.getIndicesFetchErrorMessage() }); + } + + return []; + }, + getFields: async (dataViewTitle: string) => { + const dataView = await startPlugins.data.dataViews.create({ title: dataViewTitle }); + + return dataView.fields + .filter((field) => !field.name.startsWith('_')) + .map((field) => field.name); + }, + getDefaultDataView: async () => { + const dataView = await startPlugins.data.dataViews.getDefaultDataView(); + + return dataView + ? { id: dataView.id, name: dataView.name, title: dataView.getIndexPattern() } + : undefined; + }, +}); diff --git a/x-pack/plugins/canvas/public/services/kibana/index.ts b/x-pack/plugins/canvas/public/services/kibana/index.ts index cd5e704296405..0d4dd20a52fef 100644 --- a/x-pack/plugins/canvas/public/services/kibana/index.ts +++ b/x-pack/plugins/canvas/public/services/kibana/index.ts @@ -15,6 +15,7 @@ import { import { CanvasPluginServices } from '..'; import { CanvasStartDeps } from '../../plugin'; import { customElementServiceFactory } from './custom_element'; +import { dataViewsServiceFactory } from './data_views'; import { embeddablesServiceFactory } from './embeddables'; import { expressionsServiceFactory } from './expressions'; import { labsServiceFactory } from './labs'; @@ -27,6 +28,7 @@ import { workpadServiceFactory } from './workpad'; import { filtersServiceFactory } from './filters'; export { customElementServiceFactory } from './custom_element'; +export { dataViewsServiceFactory } from './data_views'; export { embeddablesServiceFactory } from './embeddables'; export { expressionsServiceFactory } from './expressions'; export { filtersServiceFactory } from './filters'; @@ -42,6 +44,7 @@ export const pluginServiceProviders: PluginServiceProviders< KibanaPluginServiceParams > = { customElement: new PluginServiceProvider(customElementServiceFactory), + dataViews: new PluginServiceProvider(dataViewsServiceFactory, ['notify']), embeddables: new PluginServiceProvider(embeddablesServiceFactory), expressions: new PluginServiceProvider(expressionsServiceFactory, ['filters', 'notify']), filters: new PluginServiceProvider(filtersServiceFactory), diff --git a/x-pack/plugins/canvas/public/services/stubs/data_views.ts b/x-pack/plugins/canvas/public/services/stubs/data_views.ts new file mode 100644 index 0000000000000..1b1227dba4d3f --- /dev/null +++ b/x-pack/plugins/canvas/public/services/stubs/data_views.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginServiceFactory } from '@kbn/presentation-util-plugin/public'; +import { CanvasDataViewsService } from '../data_views'; + +type DataViewsServiceFactory = PluginServiceFactory; + +export const dataViewsServiceFactory: DataViewsServiceFactory = () => ({ + getDataViews: () => + Promise.resolve([ + { id: 'dataview1', title: 'dataview1', name: 'Data view 1' }, + { id: 'dataview2', title: 'dataview2', name: 'Data view 2' }, + ]), + getFields: () => Promise.resolve(['field1', 'field2']), + getDefaultDataView: () => + Promise.resolve({ + id: 'defaultDataViewId', + title: 'defaultDataView', + name: 'Default data view', + }), +}); diff --git a/x-pack/plugins/canvas/public/services/stubs/index.ts b/x-pack/plugins/canvas/public/services/stubs/index.ts index bb47f0506ce89..7a6abd470c004 100644 --- a/x-pack/plugins/canvas/public/services/stubs/index.ts +++ b/x-pack/plugins/canvas/public/services/stubs/index.ts @@ -15,6 +15,7 @@ import { import { CanvasPluginServices } from '..'; import { customElementServiceFactory } from './custom_element'; +import { dataViewsServiceFactory } from './data_views'; import { embeddablesServiceFactory } from './embeddables'; import { expressionsServiceFactory } from './expressions'; import { labsServiceFactory } from './labs'; @@ -27,6 +28,7 @@ import { workpadServiceFactory } from './workpad'; import { filtersServiceFactory } from './filters'; export { customElementServiceFactory } from './custom_element'; +export { dataViewsServiceFactory } from './data_views'; export { expressionsServiceFactory } from './expressions'; export { filtersServiceFactory } from './filters'; export { labsServiceFactory } from './labs'; @@ -39,6 +41,7 @@ export { workpadServiceFactory } from './workpad'; export const pluginServiceProviders: PluginServiceProviders = { customElement: new PluginServiceProvider(customElementServiceFactory), + dataViews: new PluginServiceProvider(dataViewsServiceFactory), embeddables: new PluginServiceProvider(embeddablesServiceFactory), expressions: new PluginServiceProvider(expressionsServiceFactory, ['filters', 'notify']), filters: new PluginServiceProvider(filtersServiceFactory), diff --git a/x-pack/plugins/canvas/storybook/canvas_webpack.ts b/x-pack/plugins/canvas/storybook/canvas_webpack.ts index 502e175b91b06..946b6c5b78cec 100644 --- a/x-pack/plugins/canvas/storybook/canvas_webpack.ts +++ b/x-pack/plugins/canvas/storybook/canvas_webpack.ts @@ -56,10 +56,6 @@ export const canvasWebpack = { resolve: { alias: { 'src/plugins': resolve(KIBANA_ROOT, 'src/plugins'), - '../../lib/es_service': resolve( - KIBANA_ROOT, - 'x-pack/plugins/canvas/storybook/__mocks__/es_service.ts' - ), }, }, }; diff --git a/x-pack/plugins/canvas/storybook/storyshots.test.tsx b/x-pack/plugins/canvas/storybook/storyshots.test.tsx index 39dde64283ba3..d880a31f446d3 100644 --- a/x-pack/plugins/canvas/storybook/storyshots.test.tsx +++ b/x-pack/plugins/canvas/storybook/storyshots.test.tsx @@ -85,11 +85,6 @@ if (!fs.existsSync(cssDir)) { fs.mkdirSync(cssDir, { recursive: true }); } -// Mock index for datasource stories -jest.mock('../public/lib/es_service', () => ({ - getDefaultIndex: () => Promise.resolve('test index'), -})); - addSerializer(styleSheetSerializer); const emotionSerializer = createSerializer({ From 15ef4a0bccb7df6f8c0f643e16fb08b17a906288 Mon Sep 17 00:00:00 2001 From: Andrew Tate Date: Mon, 24 Oct 2022 14:54:15 -0500 Subject: [PATCH 10/15] [Lens] Metric trendlines design changes (#143781) --- .../datasources/form_based/form_based.test.ts | 75 +++- .../datasources/form_based/form_based.tsx | 32 +- .../text_based/text_based_languages.test.ts | 31 +- .../text_based/text_based_languages.tsx | 20 +- .../workspace_panel/chart_switch.test.tsx | 4 +- .../lens/public/mocks/datasource_mock.ts | 4 +- .../state_management/lens_slice.test.ts | 70 ++- .../public/state_management/lens_slice.ts | 44 +- x-pack/plugins/lens/public/types.ts | 4 +- .../metric/dimension_editor.test.tsx | 411 ++++++++++-------- .../metric/dimension_editor.tsx | 365 +++++++++------- .../visualizations/metric/visualization.tsx | 17 +- 12 files changed, 682 insertions(+), 395 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts index 251d83201f468..2881511d704ac 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts @@ -1748,12 +1748,74 @@ describe('IndexPattern Data Source', () => { currentIndexPatternId: '1', }; expect(FormBasedDatasource.removeLayer(state, 'first')).toEqual({ - ...state, + removedLayerIds: ['first'], + newState: { + ...state, + layers: { + second: { + indexPatternId: '2', + columnOrder: [], + columns: {}, + }, + }, + }, + }); + }); + + it('should remove linked layers', () => { + const state = { layers: { + first: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, second: { indexPatternId: '2', columnOrder: [], columns: {}, + linkToLayers: ['first'], + }, + }, + currentIndexPatternId: '1', + }; + expect(FormBasedDatasource.removeLayer(state, 'first')).toEqual({ + removedLayerIds: ['first', 'second'], + newState: { + ...state, + layers: {}, + }, + }); + }); + }); + + describe('#clearLayer', () => { + it('should clear a layer', () => { + const state = { + layers: { + first: { + indexPatternId: '1', + columnOrder: ['some', 'order'], + columns: { + some: {} as GenericIndexPatternColumn, + columns: {} as GenericIndexPatternColumn, + }, + linkToLayers: ['some-layer'], + }, + }, + currentIndexPatternId: '1', + }; + expect(FormBasedDatasource.clearLayer(state, 'first')).toEqual({ + removedLayerIds: [], + newState: { + ...state, + layers: { + first: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + linkToLayers: ['some-layer'], + }, }, }, }); @@ -1776,9 +1838,14 @@ describe('IndexPattern Data Source', () => { }, currentIndexPatternId: '1', }; - expect(FormBasedDatasource.removeLayer(state, 'first')).toEqual({ - ...state, - layers: {}, + expect(FormBasedDatasource.clearLayer(state, 'first')).toEqual({ + removedLayerIds: ['second'], + newState: { + ...state, + layers: { + first: state.layers.first, + }, + }, }); }); }); diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx index f772e432268c2..b863c69d7f7a6 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx @@ -220,27 +220,47 @@ export function getFormBasedDatasource({ removeLayer(state: FormBasedPrivateState, layerId: string) { const newLayers = { ...state.layers }; delete newLayers[layerId]; + const removedLayerIds: string[] = [layerId]; // delete layers linked to this layer Object.keys(newLayers).forEach((id) => { const linkedLayers = newLayers[id]?.linkToLayers; if (linkedLayers && linkedLayers.includes(layerId)) { delete newLayers[id]; + removedLayerIds.push(id); } }); return { - ...state, - layers: newLayers, + removedLayerIds, + newState: { + ...state, + layers: newLayers, + }, }; }, clearLayer(state: FormBasedPrivateState, layerId: string) { + const newLayers = { ...state.layers }; + + const removedLayerIds: string[] = []; + // delete layers linked to this layer + Object.keys(newLayers).forEach((id) => { + const linkedLayers = newLayers[id]?.linkToLayers; + if (linkedLayers && linkedLayers.includes(layerId)) { + delete newLayers[id]; + removedLayerIds.push(id); + } + }); + return { - ...state, - layers: { - ...state.layers, - [layerId]: blankLayer(state.currentIndexPatternId, state.layers[layerId].linkToLayers), + removedLayerIds, + newState: { + ...state, + layers: { + ...newLayers, + [layerId]: blankLayer(state.currentIndexPatternId, state.layers[layerId].linkToLayers), + }, }, }; }, diff --git a/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.test.ts b/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.test.ts index cef1bfa96b8a5..25bab766558e3 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.test.ts +++ b/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.test.ts @@ -217,21 +217,24 @@ describe('IndexPattern Data Source', () => { describe('#removeLayer', () => { it('should remove a layer', () => { expect(TextBasedDatasource.removeLayer(baseState, 'a')).toEqual({ - ...baseState, - layers: { - a: { - columns: [], - allColumns: [ - { - columnId: 'col1', - fieldName: 'Test 1', - meta: { - type: 'number', + removedLayerIds: ['a'], + newState: { + ...baseState, + layers: { + a: { + columns: [], + allColumns: [ + { + columnId: 'col1', + fieldName: 'Test 1', + meta: { + type: 'number', + }, }, - }, - ], - query: { sql: 'SELECT * FROM foo' }, - index: 'foo', + ], + query: { sql: 'SELECT * FROM foo' }, + index: 'foo', + }, }, }, }); diff --git a/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.tsx b/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.tsx index afe6368477cc9..368036e3ebd50 100644 --- a/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.tsx +++ b/x-pack/plugins/lens/public/datasources/text_based/text_based_languages.tsx @@ -281,18 +281,24 @@ export function getTextBasedDatasource({ }; return { - ...state, - layers: newLayers, - fieldList: state.fieldList, + removedLayerIds: [layerId], + newState: { + ...state, + layers: newLayers, + fieldList: state.fieldList, + }, }; }, clearLayer(state: TextBasedPrivateState, layerId: string) { return { - ...state, - layers: { - ...state.layers, - [layerId]: { ...state.layers[layerId], columns: [] }, + removedLayerIds: [], + newState: { + ...state, + layers: { + ...state.layers, + [layerId]: { ...state.layers[layerId], columns: [] }, + }, }, }; }, diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx index e6b762911d4dc..276b56cc69fff 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/chart_switch.test.tsx @@ -494,8 +494,8 @@ describe('chart_switch', () => { switchTo('visB', instance); expect(datasourceMap.testDatasource.removeLayer).toHaveBeenCalledWith({}, 'a'); - expect(datasourceMap.testDatasource.removeLayer).toHaveBeenCalledWith(undefined, 'b'); - expect(datasourceMap.testDatasource.removeLayer).toHaveBeenCalledWith(undefined, 'c'); + expect(datasourceMap.testDatasource.removeLayer).toHaveBeenCalledWith({}, 'b'); + expect(datasourceMap.testDatasource.removeLayer).toHaveBeenCalledWith({}, 'c'); expect(visualizationMap.visB.getSuggestions).toHaveBeenCalledWith( expect.objectContaining({ keptLayerIds: ['a'], diff --git a/x-pack/plugins/lens/public/mocks/datasource_mock.ts b/x-pack/plugins/lens/public/mocks/datasource_mock.ts index cfb93882559ef..986a753a1a4a4 100644 --- a/x-pack/plugins/lens/public/mocks/datasource_mock.ts +++ b/x-pack/plugins/lens/public/mocks/datasource_mock.ts @@ -26,7 +26,7 @@ export function createMockDatasource(id: string): DatasourceMock { return { id: 'testDatasource', - clearLayer: jest.fn((state, _layerId) => state), + clearLayer: jest.fn((state, _layerId) => ({ newState: state, removedLayerIds: [] })), getDatasourceSuggestionsForField: jest.fn((_state, _item, filterFn, _indexPatterns) => []), getDatasourceSuggestionsForVisualizeField: jest.fn( (_state, _indexpatternId, _fieldName, _indexPatterns) => [] @@ -44,7 +44,7 @@ export function createMockDatasource(id: string): DatasourceMock { renderLayerPanel: jest.fn(), toExpression: jest.fn((_frame, _state, _indexPatterns) => null), insertLayer: jest.fn((_state, _newLayerId) => ({})), - removeLayer: jest.fn((_state, _layerId) => {}), + removeLayer: jest.fn((state, layerId) => ({ newState: state, removedLayerIds: [layerId] })), cloneLayer: jest.fn((_state, _layerId, _newLayerId, getNewId) => {}), removeColumn: jest.fn((props) => {}), getLayers: jest.fn((_state) => []), diff --git a/x-pack/plugins/lens/public/state_management/lens_slice.test.ts b/x-pack/plugins/lens/public/state_management/lens_slice.test.ts index 934fc0854b6e6..df9d42015632c 100644 --- a/x-pack/plugins/lens/public/state_management/lens_slice.test.ts +++ b/x-pack/plugins/lens/public/state_management/lens_slice.test.ts @@ -39,6 +39,7 @@ describe('lensSlice', () => { let store: EnhancedStore<{ lens: LensAppState }>; beforeEach(() => { store = makeLensStore({}).store; + jest.clearAllMocks(); }); const customQuery = { query: 'custom' } as Query; @@ -275,17 +276,21 @@ describe('lensSlice', () => { return { id: datasourceId, getPublicAPI: () => ({ - datasourceId: 'testDatasource', + datasourceId, getOperationForColumnId: jest.fn(), getTableSpec: jest.fn(), }), getLayers: () => ['layer1'], - clearLayer: (layerIds: unknown, layerId: string) => - (layerIds as string[]).map((id: string) => + clearLayer: (layerIds: unknown, layerId: string) => ({ + removedLayerIds: [], + newState: (layerIds as string[]).map((id: string) => id === layerId ? `${datasourceId}_clear_${layerId}` : id ), - removeLayer: (layerIds: unknown, layerId: string) => - (layerIds as string[]).filter((id: string) => id !== layerId), + }), + removeLayer: (layerIds: unknown, layerId: string) => ({ + newState: (layerIds as string[]).filter((id: string) => id !== layerId), + removedLayerIds: [layerId], + }), insertLayer: (layerIds: unknown, layerId: string, layersToLinkTo: string[]) => [ ...(layerIds as string[]), layerId, @@ -317,8 +322,9 @@ describe('lensSlice', () => { (layerIds as string[]).map((id: string) => id === layerId ? `vis_clear_${layerId}` : id ), - removeLayer: (layerIds: unknown, layerId: string) => - (layerIds as string[]).filter((id: string) => id !== layerId), + removeLayer: jest.fn((layerIds: unknown, layerId: string) => + (layerIds as string[]).filter((id: string) => id !== layerId) + ), getLayerIds: (layerIds: unknown) => layerIds as string[], getLayersToLinkTo: (state, newLayerId) => ['linked-layer-id'], appendLayer: (layerIds: unknown, layerId: string) => [...(layerIds as string[]), layerId], @@ -482,6 +488,54 @@ describe('lensSlice', () => { expect(state.datasourceStates.testDatasource2.state).toEqual(['layer2']); expect(state.stagedPreview).not.toBeDefined(); }); + + it('removeLayer: should remove all layers from visualization that were removed by datasource', () => { + const removedLayerId = 'other-removed-layer'; + + const testDatasource3 = testDatasource('testDatasource3'); + testDatasource3.removeLayer = (layerIds: unknown, layerId: string) => ({ + newState: (layerIds as string[]).filter((id: string) => id !== layerId), + removedLayerIds: [layerId, removedLayerId], + }); + + const localStore = makeLensStore({ + preloadedState: { + activeDatasourceId: 'testDatasource', + datasourceStates: { + ...datasourceStates, + testDatasource3: { + isLoading: false, + state: [], + }, + }, + visualization: { + activeId: activeVisId, + state: ['layer1', 'layer2'], + }, + stagedPreview: { + visualization: { + activeId: activeVisId, + state: ['layer1', 'layer2'], + }, + datasourceStates, + }, + }, + storeDeps: mockStoreDeps({ + visualizationMap: visualizationMap as unknown as VisualizationMap, + datasourceMap: { ...datasourceMap, testDatasource3 } as unknown as DatasourceMap, + }), + }).store; + + localStore.dispatch( + removeOrClearLayer({ + visualizationId: 'testVis', + layerId: 'layer1', + layerIds: ['layer1', 'layer2'], + }) + ); + + expect(visualizationMap[activeVisId].removeLayer).toHaveBeenCalledTimes(2); + }); }); describe('removing a dimension', () => { @@ -546,8 +600,6 @@ describe('lensSlice', () => { datasourceMap: datasourceMap as unknown as DatasourceMap, }), }).store; - - jest.clearAllMocks(); }); it('removes a dimension', () => { diff --git a/x-pack/plugins/lens/public/state_management/lens_slice.ts b/x-pack/plugins/lens/public/state_management/lens_slice.ts index b90c5bc965a6b..3f3490906f88c 100644 --- a/x-pack/plugins/lens/public/state_management/lens_slice.ts +++ b/x-pack/plugins/lens/public/state_management/lens_slice.ts @@ -7,7 +7,7 @@ import { createAction, createReducer, current, PayloadAction } from '@reduxjs/toolkit'; import { VisualizeFieldContext } from '@kbn/ui-actions-plugin/public'; -import { mapValues } from 'lodash'; +import { mapValues, uniq } from 'lodash'; import { Query } from '@kbn/es-query'; import { History } from 'history'; import { LensEmbeddableInput } from '..'; @@ -400,16 +400,23 @@ export const makeLensReducer = (storeDeps: LensStoreDeps) => { layerIds.length ) === 'clear'; + let removedLayerIds: string[] = []; + state.datasourceStates = mapValues( state.datasourceStates, (datasourceState, datasourceId) => { const datasource = datasourceMap[datasourceId!]; + + const { newState, removedLayerIds: removedLayerIdsForThisDatasource } = isOnlyLayer + ? datasource.clearLayer(datasourceState.state, layerId) + : datasource.removeLayer(datasourceState.state, layerId); + + removedLayerIds = [...removedLayerIds, ...removedLayerIdsForThisDatasource]; + return { ...datasourceState, ...(datasourceId === state.activeDatasourceId && { - state: isOnlyLayer - ? datasource.clearLayer(datasourceState.state, layerId) - : datasource.removeLayer(datasourceState.state, layerId), + state: newState, }), }; } @@ -419,10 +426,22 @@ export const makeLensReducer = (storeDeps: LensStoreDeps) => { const currentDataViewsId = activeDataSource.getUsedDataView( state.datasourceStates[state.activeDatasourceId!].state ); - state.visualization.state = - isOnlyLayer || !activeVisualization.removeLayer - ? activeVisualization.clearLayer(state.visualization.state, layerId, currentDataViewsId) - : activeVisualization.removeLayer(state.visualization.state, layerId); + + if (isOnlyLayer || !activeVisualization.removeLayer) { + state.visualization.state = activeVisualization.clearLayer( + state.visualization.state, + layerId, + currentDataViewsId + ); + } + + uniq(removedLayerIds).forEach( + (removedId) => + (state.visualization.state = activeVisualization.removeLayer?.( + state.visualization.state, + removedId + )) + ); }, [changeIndexPattern.type]: ( state, @@ -977,9 +996,12 @@ export const makeLensReducer = (storeDeps: LensStoreDeps) => { ); }) ?? []; if (layerDatasourceId) { - state.datasourceStates[layerDatasourceId].state = datasourceMap[ - layerDatasourceId - ].removeLayer(current(state).datasourceStates[layerDatasourceId].state, layerId); + const { newState } = datasourceMap[layerDatasourceId].removeLayer( + current(state).datasourceStates[layerDatasourceId].state, + layerId + ); + state.datasourceStates[layerDatasourceId].state = newState; + // TODO - call removeLayer for any extra (linked) layers removed by the datasource } }); }, diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index dd5d12809c94a..94a0589f78b08 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -265,8 +265,8 @@ export interface Datasource { insertLayer: (state: T, newLayerId: string, linkToLayers?: string[]) => T; createEmptyLayer: (indexPatternId: string) => T; - removeLayer: (state: T, layerId: string) => T; - clearLayer: (state: T, layerId: string) => T; + removeLayer: (state: T, layerId: string) => { newState: T; removedLayerIds: string[] }; + clearLayer: (state: T, layerId: string) => { newState: T; removedLayerIds: string[] }; cloneLayer: ( state: T, layerId: string, diff --git a/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx b/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx index 9826e60a83bc9..b45eef4ec379e 100644 --- a/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx +++ b/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.test.tsx @@ -12,7 +12,11 @@ import { OperationDescriptor, VisualizationDimensionEditorProps } from '../../ty import { CustomPaletteParams, PaletteOutput, PaletteRegistry } from '@kbn/coloring'; import { MetricVisualizationState } from './visualization'; -import { DimensionEditor, SupportingVisType } from './dimension_editor'; +import { + DimensionEditor, + DimensionEditorAdditionalSection, + SupportingVisType, +} from './dimension_editor'; import { HTMLAttributes, mount, ReactWrapper, shallow } from 'enzyme'; import { CollapseSetting } from '../../shared_components/collapse_setting'; import { EuiButtonGroup, EuiColorPicker, PropsOf } from '@elastic/eui'; @@ -154,42 +158,6 @@ describe('dimension editor', () => { this.colorPicker.props().onChange!(color, {} as EuiColorPickerOutput); }); } - - private get supportingVisButtonGroup() { - return this._wrapper.find( - 'EuiButtonGroup[data-test-subj="lnsMetric_supporting_visualization_buttons"]' - ) as unknown as ReactWrapper>; - } - - public get currentSupportingVis() { - return this.supportingVisButtonGroup - .props() - .idSelected?.split('--')[1] as SupportingVisType; - } - - public isDisabled(type: SupportingVisType) { - return this.supportingVisButtonGroup.props().options.find(({ id }) => id.includes(type)) - ?.isDisabled; - } - - public setSupportingVis(type: SupportingVisType) { - this.supportingVisButtonGroup.props().onChange(`some-id--${type}`); - } - - private get progressDirectionControl() { - return this._wrapper.find( - 'EuiButtonGroup[data-test-subj="lnsMetric_progress_direction_buttons"]' - ) as unknown as ReactWrapper>; - } - - public get progressDirectionShowing() { - return this.progressDirectionControl.exists(); - } - - public setProgressDirection(direction: LayoutDirection) { - this.progressDirectionControl.props().onChange(direction); - this._wrapper.update(); - } } const mockSetState = jest.fn(); @@ -266,144 +234,6 @@ describe('dimension editor', () => { `); }); }); - - describe('supporting visualizations', () => { - const stateWOTrend = { - ...metricAccessorState, - trendlineLayerId: undefined, - }; - - describe('reflecting visualization state', () => { - it('should select the correct button', () => { - expect( - getHarnessWithState({ ...stateWOTrend, showBar: false, maxAccessor: undefined }) - .currentSupportingVis - ).toBe('none'); - expect( - getHarnessWithState({ ...stateWOTrend, showBar: true }).currentSupportingVis - ).toBe('bar'); - expect( - getHarnessWithState(metricAccessorState).currentSupportingVis - ).toBe('trendline'); - }); - - it('should disable bar when no max dimension', () => { - expect( - getHarnessWithState({ - ...stateWOTrend, - showBar: false, - maxAccessor: 'something', - }).isDisabled('bar') - ).toBeFalsy(); - expect( - getHarnessWithState({ - ...stateWOTrend, - showBar: false, - maxAccessor: undefined, - }).isDisabled('bar') - ).toBeTruthy(); - }); - - it('should disable trendline when no default time field', () => { - expect( - getHarnessWithState(stateWOTrend, { - hasDefaultTimeField: () => false, - getOperationForColumnId: (id) => ({} as OperationDescriptor), - } as DatasourcePublicAPI).isDisabled('trendline') - ).toBeTruthy(); - expect( - getHarnessWithState(stateWOTrend, { - hasDefaultTimeField: () => true, - getOperationForColumnId: (id) => ({} as OperationDescriptor), - } as DatasourcePublicAPI).isDisabled('trendline') - ).toBeFalsy(); - }); - }); - - it('should disable trendline when a metric dimension has a reduced time range', () => { - expect( - getHarnessWithState(stateWOTrend, { - hasDefaultTimeField: () => true, - getOperationForColumnId: (id) => - ({ hasReducedTimeRange: id === stateWOTrend.metricAccessor } as OperationDescriptor), - } as DatasourcePublicAPI).isDisabled('trendline') - ).toBeTruthy(); - expect( - getHarnessWithState(stateWOTrend, { - hasDefaultTimeField: () => true, - getOperationForColumnId: (id) => - ({ - hasReducedTimeRange: id === stateWOTrend.secondaryMetricAccessor, - } as OperationDescriptor), - } as DatasourcePublicAPI).isDisabled('trendline') - ).toBeTruthy(); - }); - - describe('responding to buttons', () => { - it('enables trendline', () => { - getHarnessWithState(stateWOTrend).setSupportingVis('trendline'); - - expect(mockSetState).toHaveBeenCalledWith({ ...stateWOTrend, showBar: false }); - expect(props.addLayer).toHaveBeenCalledWith('metricTrendline'); - - expectCalledBefore(mockSetState, props.addLayer as jest.Mock); - }); - - it('enables bar', () => { - getHarnessWithState(metricAccessorState).setSupportingVis('bar'); - - expect(mockSetState).toHaveBeenCalledWith({ ...metricAccessorState, showBar: true }); - expect(props.removeLayer).toHaveBeenCalledWith(metricAccessorState.trendlineLayerId); - - expectCalledBefore(mockSetState, props.removeLayer as jest.Mock); - }); - - it('selects none from bar', () => { - getHarnessWithState(stateWOTrend).setSupportingVis('none'); - - expect(mockSetState).toHaveBeenCalledWith({ ...stateWOTrend, showBar: false }); - expect(props.removeLayer).not.toHaveBeenCalled(); - }); - - it('selects none from trendline', () => { - getHarnessWithState(metricAccessorState).setSupportingVis('none'); - - expect(mockSetState).toHaveBeenCalledWith({ ...metricAccessorState, showBar: false }); - expect(props.removeLayer).toHaveBeenCalledWith(metricAccessorState.trendlineLayerId); - - expectCalledBefore(mockSetState, props.removeLayer as jest.Mock); - }); - }); - - describe('progress bar direction controls', () => { - it('hides direction controls if bar not showing', () => { - expect( - getHarnessWithState({ ...stateWOTrend, showBar: false }).progressDirectionShowing - ).toBeFalsy(); - }); - - it('toggles progress direction', () => { - const harness = getHarnessWithState(metricAccessorState); - - expect(harness.progressDirectionShowing).toBeTruthy(); - expect(harness.currentState.progressDirection).toBe('vertical'); - - harness.setProgressDirection('horizontal'); - harness.setProgressDirection('vertical'); - harness.setProgressDirection('horizontal'); - - expect(mockSetState).toHaveBeenCalledTimes(3); - expect(mockSetState.mock.calls.map((args) => args[0].progressDirection)) - .toMatchInlineSnapshot(` - Array [ - "horizontal", - "vertical", - "horizontal", - ] - `); - }); - }); - }); }); describe('secondary metric dimension', () => { @@ -628,4 +458,235 @@ describe('dimension editor', () => { `); }); }); + + describe('additional section', () => { + const accessor = 'primary-metric-col-id'; + const metricAccessorState = { ...fullState, metricAccessor: accessor }; + + class Harness { + public _wrapper; + + constructor( + wrapper: ReactWrapper> + ) { + this._wrapper = wrapper; + } + + private get rootComponent() { + return this._wrapper.find(DimensionEditorAdditionalSection); + } + + public get currentState() { + return this.rootComponent.props().state; + } + + private get supportingVisButtonGroup() { + return this._wrapper.find( + 'EuiButtonGroup[data-test-subj="lnsMetric_supporting_visualization_buttons"]' + ) as unknown as ReactWrapper>; + } + + public get currentSupportingVis() { + return this.supportingVisButtonGroup + .props() + .idSelected?.split('--')[1] as SupportingVisType; + } + + public isDisabled(type: SupportingVisType) { + return this.supportingVisButtonGroup.props().options.find(({ id }) => id.includes(type)) + ?.isDisabled; + } + + public setSupportingVis(type: SupportingVisType) { + this.supportingVisButtonGroup.props().onChange(`some-id--${type}`); + } + + private get progressDirectionControl() { + return this._wrapper.find( + 'EuiButtonGroup[data-test-subj="lnsMetric_progress_direction_buttons"]' + ) as unknown as ReactWrapper>; + } + + public get progressDirectionShowing() { + return this.progressDirectionControl.exists(); + } + + public setProgressDirection(direction: LayoutDirection) { + this.progressDirectionControl.props().onChange(direction); + this._wrapper.update(); + } + } + + const mockSetState = jest.fn(); + + const getHarnessWithState = (state: MetricVisualizationState, datasource = props.datasource) => + new Harness( + mountWithIntl( + + ) + ); + + it.each([ + { name: 'secondary metric', accessor: metricAccessorState.secondaryMetricAccessor }, + { name: 'max', accessor: metricAccessorState.maxAccessor }, + { name: 'break down by', accessor: metricAccessorState.breakdownByAccessor }, + ])('doesnt show for the following dimension: %s', ({ accessor: testAccessor }) => { + expect( + shallow( + + ).isEmptyRender() + ).toBeTruthy(); + }); + + describe('supporting visualizations', () => { + const stateWOTrend = { + ...metricAccessorState, + trendlineLayerId: undefined, + }; + + describe('reflecting visualization state', () => { + it('should select the correct button', () => { + expect( + getHarnessWithState({ ...stateWOTrend, showBar: false, maxAccessor: undefined }) + .currentSupportingVis + ).toBe('none'); + expect( + getHarnessWithState({ ...stateWOTrend, showBar: true }).currentSupportingVis + ).toBe('bar'); + expect( + getHarnessWithState(metricAccessorState).currentSupportingVis + ).toBe('trendline'); + }); + + it('should disable bar when no max dimension', () => { + expect( + getHarnessWithState({ + ...stateWOTrend, + showBar: false, + maxAccessor: 'something', + }).isDisabled('bar') + ).toBeFalsy(); + expect( + getHarnessWithState({ + ...stateWOTrend, + showBar: false, + maxAccessor: undefined, + }).isDisabled('bar') + ).toBeTruthy(); + }); + + it('should disable trendline when no default time field', () => { + expect( + getHarnessWithState(stateWOTrend, { + hasDefaultTimeField: () => false, + getOperationForColumnId: (id) => ({} as OperationDescriptor), + } as DatasourcePublicAPI).isDisabled('trendline') + ).toBeTruthy(); + expect( + getHarnessWithState(stateWOTrend, { + hasDefaultTimeField: () => true, + getOperationForColumnId: (id) => ({} as OperationDescriptor), + } as DatasourcePublicAPI).isDisabled('trendline') + ).toBeFalsy(); + }); + }); + + it('should disable trendline when a metric dimension has a reduced time range', () => { + expect( + getHarnessWithState(stateWOTrend, { + hasDefaultTimeField: () => true, + getOperationForColumnId: (id) => + ({ + hasReducedTimeRange: id === stateWOTrend.metricAccessor, + } as OperationDescriptor), + } as DatasourcePublicAPI).isDisabled('trendline') + ).toBeTruthy(); + expect( + getHarnessWithState(stateWOTrend, { + hasDefaultTimeField: () => true, + getOperationForColumnId: (id) => + ({ + hasReducedTimeRange: id === stateWOTrend.secondaryMetricAccessor, + } as OperationDescriptor), + } as DatasourcePublicAPI).isDisabled('trendline') + ).toBeTruthy(); + }); + + describe('responding to buttons', () => { + it('enables trendline', () => { + getHarnessWithState(stateWOTrend).setSupportingVis('trendline'); + + expect(mockSetState).toHaveBeenCalledWith({ ...stateWOTrend, showBar: false }); + expect(props.addLayer).toHaveBeenCalledWith('metricTrendline'); + + expectCalledBefore(mockSetState, props.addLayer as jest.Mock); + }); + + it('enables bar', () => { + getHarnessWithState(metricAccessorState).setSupportingVis('bar'); + + expect(mockSetState).toHaveBeenCalledWith({ ...metricAccessorState, showBar: true }); + expect(props.removeLayer).toHaveBeenCalledWith(metricAccessorState.trendlineLayerId); + + expectCalledBefore(mockSetState, props.removeLayer as jest.Mock); + }); + + it('selects none from bar', () => { + getHarnessWithState(stateWOTrend).setSupportingVis('none'); + + expect(mockSetState).toHaveBeenCalledWith({ ...stateWOTrend, showBar: false }); + expect(props.removeLayer).not.toHaveBeenCalled(); + }); + + it('selects none from trendline', () => { + getHarnessWithState(metricAccessorState).setSupportingVis('none'); + + expect(mockSetState).toHaveBeenCalledWith({ ...metricAccessorState, showBar: false }); + expect(props.removeLayer).toHaveBeenCalledWith(metricAccessorState.trendlineLayerId); + + expectCalledBefore(mockSetState, props.removeLayer as jest.Mock); + }); + }); + + describe('progress bar direction controls', () => { + it('hides direction controls if bar not showing', () => { + expect( + getHarnessWithState({ ...stateWOTrend, showBar: false }).progressDirectionShowing + ).toBeFalsy(); + }); + + it('toggles progress direction', () => { + const harness = getHarnessWithState(metricAccessorState); + + expect(harness.progressDirectionShowing).toBeTruthy(); + expect(harness.currentState.progressDirection).toBe('vertical'); + + harness.setProgressDirection('horizontal'); + harness.setProgressDirection('vertical'); + harness.setProgressDirection('horizontal'); + + expect(mockSetState).toHaveBeenCalledTimes(3); + expect(mockSetState.mock.calls.map((args) => args[0].progressDirection)) + .toMatchInlineSnapshot(` + Array [ + "horizontal", + "vertical", + "horizontal", + ] + `); + }); + }); + }); + }); }); diff --git a/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.tsx b/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.tsx index 4ca35b060e023..6571e1bb2c7d6 100644 --- a/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/visualizations/metric/dimension_editor.tsx @@ -17,6 +17,8 @@ import { EuiColorPicker, euiPaletteColorBlind, EuiSpacer, + EuiText, + useEuiTheme, } from '@elastic/eui'; import { LayoutDirection } from '@elastic/charts'; import React, { useCallback, useState } from 'react'; @@ -30,6 +32,7 @@ import { } from '@kbn/coloring'; import { getDataBoundsForPalette } from '@kbn/expression-metric-vis-plugin/public'; import { getColumnByAccessor } from '@kbn/visualizations-plugin/common/utils'; +import { css } from '@emotion/react'; import { isNumericFieldForDatatable } from '../../../common/expressions/datatable/utils'; import { applyPaletteParams, @@ -263,170 +266,8 @@ function PrimaryMetricEditor(props: SubProps) { const togglePalette = () => setIsPaletteOpen(!isPaletteOpen); - const supportingVisLabel = i18n.translate('xpack.lens.metric.supportingVis.label', { - defaultMessage: 'Supporting visualization', - }); - - const hasDefaultTimeField = props.datasource?.hasDefaultTimeField(); - const metricHasReducedTimeRange = Boolean( - state.metricAccessor && - props.datasource?.getOperationForColumnId(state.metricAccessor)?.hasReducedTimeRange - ); - const secondaryMetricHasReducedTimeRange = Boolean( - state.secondaryMetricAccessor && - props.datasource?.getOperationForColumnId(state.secondaryMetricAccessor)?.hasReducedTimeRange - ); - - const supportingVisHelpTexts: string[] = []; - - const supportsTrendline = - hasDefaultTimeField && !metricHasReducedTimeRange && !secondaryMetricHasReducedTimeRange; - - if (!supportsTrendline) { - supportingVisHelpTexts.push( - !hasDefaultTimeField - ? i18n.translate('xpack.lens.metric.supportingVis.needDefaultTimeField', { - defaultMessage: 'Use a data view with a default time field to enable trend lines.', - }) - : metricHasReducedTimeRange - ? i18n.translate('xpack.lens.metric.supportingVis.metricHasReducedTimeRange', { - defaultMessage: - 'Remove the reduced time range on this dimension to enable trend lines.', - }) - : secondaryMetricHasReducedTimeRange - ? i18n.translate('xpack.lens.metric.supportingVis.secondaryMetricHasReducedTimeRange', { - defaultMessage: - 'Remove the reduced time range on the secondary metric dimension to enable trend lines.', - }) - : '' - ); - } - - if (!state.maxAccessor) { - supportingVisHelpTexts.push( - i18n.translate('xpack.lens.metric.summportingVis.needMaxDimension', { - defaultMessage: 'Add a maximum dimension to enable the progress bar.', - }) - ); - } - - const buttonIdPrefix = `${idPrefix}--`; - return ( <> - ( -
{text}
- ))} - > - { - const supportingVisualizationType = id.split('--')[1] as SupportingVisType; - - switch (supportingVisualizationType) { - case 'trendline': - setState({ - ...state, - showBar: false, - }); - props.addLayer('metricTrendline'); - break; - case 'bar': - setState({ - ...state, - showBar: true, - }); - if (state.trendlineLayerId) props.removeLayer(state.trendlineLayerId); - break; - case 'none': - setState({ - ...state, - showBar: false, - }); - if (state.trendlineLayerId) props.removeLayer(state.trendlineLayerId); - break; - } - }} - /> -
- {showingBar(state) && ( - - { - const newDirection = id.replace(idPrefix, '') as LayoutDirection; - setState({ - ...state, - progressDirection: newDirection, - }); - }} - /> - - )} ); } + +export function DimensionEditorAdditionalSection({ + state, + datasource, + setState, + addLayer, + removeLayer, + accessor, +}: VisualizationDimensionEditorProps) { + const { euiTheme } = useEuiTheme(); + + if (accessor !== state.metricAccessor) { + return null; + } + + const idPrefix = htmlIdGenerator()(); + + const hasDefaultTimeField = datasource?.hasDefaultTimeField(); + const metricHasReducedTimeRange = Boolean( + state.metricAccessor && + datasource?.getOperationForColumnId(state.metricAccessor)?.hasReducedTimeRange + ); + const secondaryMetricHasReducedTimeRange = Boolean( + state.secondaryMetricAccessor && + datasource?.getOperationForColumnId(state.secondaryMetricAccessor)?.hasReducedTimeRange + ); + + const supportingVisHelpTexts: string[] = []; + + const supportsTrendline = + hasDefaultTimeField && !metricHasReducedTimeRange && !secondaryMetricHasReducedTimeRange; + + if (!supportsTrendline) { + supportingVisHelpTexts.push( + !hasDefaultTimeField + ? i18n.translate('xpack.lens.metric.supportingVis.needDefaultTimeField', { + defaultMessage: + 'Line visualizations require use of a data view with a default time field.', + }) + : metricHasReducedTimeRange + ? i18n.translate('xpack.lens.metric.supportingVis.metricHasReducedTimeRange', { + defaultMessage: + 'Line visualizations cannot be used when a reduced time range is applied to the primary metric.', + }) + : secondaryMetricHasReducedTimeRange + ? i18n.translate('xpack.lens.metric.supportingVis.secondaryMetricHasReducedTimeRange', { + defaultMessage: + 'Line visualizations cannot be used when a reduced time range is applied to the secondary metric.', + }) + : '' + ); + } + + if (!state.maxAccessor) { + supportingVisHelpTexts.push( + i18n.translate('xpack.lens.metric.summportingVis.needMaxDimension', { + defaultMessage: 'Bar visualizations require a maximum value to be defined.', + }) + ); + } + + const buttonIdPrefix = `${idPrefix}--`; + + return ( +
+ +

+ {i18n.translate('xpack.lens.metric.supportingVis.label', { + defaultMessage: 'Supporting visualization', + })} +

+
+ + <> + ( +

{text}

+ ))} + > + { + const supportingVisualizationType = id.split('--')[1] as SupportingVisType; + + switch (supportingVisualizationType) { + case 'trendline': + setState({ + ...state, + showBar: false, + }); + addLayer('metricTrendline'); + break; + case 'bar': + setState({ + ...state, + showBar: true, + }); + if (state.trendlineLayerId) removeLayer(state.trendlineLayerId); + break; + case 'none': + setState({ + ...state, + showBar: false, + }); + if (state.trendlineLayerId) removeLayer(state.trendlineLayerId); + break; + } + }} + /> +
+ {showingBar(state) && ( + + { + const newDirection = id.replace(idPrefix, '') as LayoutDirection; + setState({ + ...state, + progressDirection: newDirection, + }); + }} + /> + + )} + +
+ ); +} diff --git a/x-pack/plugins/lens/public/visualizations/metric/visualization.tsx b/x-pack/plugins/lens/public/visualizations/metric/visualization.tsx index 123821a62aa69..eac08b22c9ea4 100644 --- a/x-pack/plugins/lens/public/visualizations/metric/visualization.tsx +++ b/x-pack/plugins/lens/public/visualizations/metric/visualization.tsx @@ -30,7 +30,7 @@ import { Suggestion, } from '../../types'; import { GROUP_ID, LENS_METRIC_ID } from './constants'; -import { DimensionEditor } from './dimension_editor'; +import { DimensionEditor, DimensionEditorAdditionalSection } from './dimension_editor'; import { Toolbar } from './toolbar'; import { generateId } from '../../id_generator'; import { FormatSelectorOptions } from '../../datasources/form_based/dimension_panel/format_selector'; @@ -454,6 +454,10 @@ export const getMetricVisualization = ({ return newState; }, + getRemoveOperation(state, layerId) { + return layerId === state.trendlineLayerId ? 'remove' : 'clear'; + }, + getLayersToLinkTo(state, newLayerId: string): string[] { return newLayerId === state.trendlineLayerId ? [state.layerId] : []; }, @@ -617,6 +621,17 @@ export const getMetricVisualization = ({ ); }, + renderDimensionEditorAdditionalSection(domElement, props) { + render( + + + + + , + domElement + ); + }, + getErrorMessages(state) { // Is it possible to break it? return undefined; From 7d10edcc8fd20a5bdb2f24c4063cf29d8449e9aa Mon Sep 17 00:00:00 2001 From: Kurt Date: Mon, 24 Oct 2022 16:01:43 -0400 Subject: [PATCH 11/15] Removing docs about multi-tenancy (#143698) * Removing docs about multi-tenancy * Removing link to multi tenancy --- docs/spaces/index.asciidoc | 2 -- docs/user/security/authorization/index.asciidoc | 16 ---------------- 2 files changed, 18 deletions(-) diff --git a/docs/spaces/index.asciidoc b/docs/spaces/index.asciidoc index 00e50a41b6ce3..9b20d1c23719b 100644 --- a/docs/spaces/index.asciidoc +++ b/docs/spaces/index.asciidoc @@ -19,8 +19,6 @@ image::images/change-space.png["Change current space menu"] The `kibana_admin` role or equivalent is required to manage **Spaces**. -TIP: Looking to support multiple tenants? Refer to <> for more information. - [float] [[spaces-managing]] === View, create, and delete spaces diff --git a/docs/user/security/authorization/index.asciidoc b/docs/user/security/authorization/index.asciidoc index b478e8b7c38d5..7127ebf84ae9b 100644 --- a/docs/user/security/authorization/index.asciidoc +++ b/docs/user/security/authorization/index.asciidoc @@ -6,22 +6,6 @@ The Elastic Stack comes with the `kibana_admin` {ref}/built-in-roles.html[built- When you assign a user multiple roles, the user receives a union of the roles’ privileges. Therefore, assigning the `kibana_admin` role in addition to a custom role that grants {kib} privileges is ineffective because `kibana_admin` has access to all the features in all spaces. -[[xpack-security-multiple-tenants]] -==== Supporting multiple tenants - -There are two approaches to supporting multi-tenancy in {kib}: - -1. *Recommended:* Create a space and a limited role for each tenant, and configure each user with the appropriate role. See -<> for more details. -2. deprecated:[7.13.0,"In 8.0 and later, the `kibana.index` setting will no longer be supported."] Set up separate {kib} instances to work -with a single {es} cluster by changing the `kibana.index` setting in your `kibana.yml` file. -+ -NOTE: When using multiple {kib} instances this way, you cannot use the `kibana_admin` role to grant access. You must create custom roles -that authorize the user for each specific instance. - -Whichever approach you use, be careful when granting cluster privileges and index privileges. Both of these approaches share the same {es} -cluster, and {kib} spaces do not prevent you from granting users of two different tenants access to the same index. - [role="xpack"] [[kibana-role-management]] === {kib} role management From ee326591a3f8edff34ec978da781cc5c121348bf Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 24 Oct 2022 15:10:42 -0600 Subject: [PATCH 12/15] [Maps] Add ability to invert color ramp and size (#143307) * convert color forms to TS * switch for setting invert * reverse colors * convert size components to TS * invert size switch * invert mb size expression * invert size legend * invert ordinal legend * invert colors in color picker * update jest snapshot * add unit tests for inverting color ramp creation * review feedback Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../style_property_descriptor_types.ts | 2 + .../classes/styles/color_palettes.test.ts | 59 +++++++- .../public/classes/styles/color_palettes.ts | 30 ++-- .../components/ranged_style_legend_row.tsx | 6 +- .../components/legend/heatmap_legend.tsx | 1 + .../classes/styles/heatmap/heatmap_style.tsx | 3 +- .../components/color/color_map_select.js | 22 ++- ...c_color_form.js => dynamic_color_form.tsx} | 140 +++++++++++++----- ...ic_color_form.js => static_color_form.tsx} | 18 ++- .../color/vector_style_color_editor.tsx | 8 +- .../components/legend/marker_size_legend.tsx | 14 +- .../components/legend/ordinal_legend.tsx | 4 + .../components/size/dynamic_size_form.js | 66 --------- .../components/size/dynamic_size_form.tsx | 109 ++++++++++++++ ...ge_selector.js => size_range_selector.tsx} | 20 +-- ...atic_size_form.js => static_size_form.tsx} | 18 ++- .../size/vector_style_size_editor.tsx | 8 +- .../properties/dynamic_color_property.tsx | 18 ++- .../dynamic_size_property.test.tsx.snap | 1 + .../dynamic_size_property.tsx | 16 +- 20 files changed, 393 insertions(+), 170 deletions(-) rename x-pack/plugins/maps/public/classes/styles/vector/components/color/{dynamic_color_form.js => dynamic_color_form.tsx} (50%) rename x-pack/plugins/maps/public/classes/styles/vector/components/color/{static_color_form.js => static_color_form.tsx} (61%) delete mode 100644 x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.js create mode 100644 x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.tsx rename x-pack/plugins/maps/public/classes/styles/vector/components/size/{size_range_selector.js => size_range_selector.tsx} (74%) rename x-pack/plugins/maps/public/classes/styles/vector/components/size/{static_size_form.js => static_size_form.tsx} (69%) diff --git a/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts index 78e9393e5755d..ad04d467741c4 100644 --- a/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts @@ -78,6 +78,7 @@ export type ColorDynamicOptions = { customColorRamp?: OrdinalColorStop[]; useCustomColorRamp?: boolean; dataMappingFunction?: DATA_MAPPING_FUNCTION; + invert?: boolean; // category color properties colorCategory?: string; // TODO move color category palettes to constants and make ENUM type @@ -176,6 +177,7 @@ export type SizeDynamicOptions = { maxSize: number; field?: StylePropertyField; fieldMetaOptions: FieldMetaOptions; + invert?: boolean; }; export type SizeStaticOptions = { diff --git a/x-pack/plugins/maps/public/classes/styles/color_palettes.test.ts b/x-pack/plugins/maps/public/classes/styles/color_palettes.test.ts index 6c98cdc476502..d5a6ea694b3e0 100644 --- a/x-pack/plugins/maps/public/classes/styles/color_palettes.test.ts +++ b/x-pack/plugins/maps/public/classes/styles/color_palettes.test.ts @@ -13,7 +13,7 @@ import { } from './color_palettes'; describe('getColorPalette', () => { - it('Should create RGB color ramp', () => { + test('Should create RGB color ramp', () => { expect(getColorPalette('Blues')).toEqual([ '#ecf1f7', '#d9e3ef', @@ -28,14 +28,14 @@ describe('getColorPalette', () => { }); describe('getColorRampCenterColor', () => { - it('Should get center color from color ramp', () => { + test('Should get center color from color ramp', () => { expect(getColorRampCenterColor('Blues')).toBe('#9eb9d8'); }); }); describe('getOrdinalMbColorRampStops', () => { - it('Should create color stops for custom range', () => { - expect(getOrdinalMbColorRampStops('Blues', 0, 1000)).toEqual([ + test('Should create color stops', () => { + expect(getOrdinalMbColorRampStops('Blues', 0, 1000, false)).toEqual([ 0, '#ecf1f7', 125, @@ -55,13 +55,34 @@ describe('getOrdinalMbColorRampStops', () => { ]); }); - it('Should snap to end of color stops for identical range', () => { - expect(getOrdinalMbColorRampStops('Blues', 23, 23)).toEqual([23, '#6092c0']); + test('Should create inverted color stops', () => { + expect(getOrdinalMbColorRampStops('Blues', 0, 1000, true)).toEqual([ + 0, + '#6092c0', + 125, + '#769fc8', + 250, + '#8bacd0', + 375, + '#9eb9d8', + 500, + '#b2c7df', + 625, + '#c5d5e7', + 750, + '#d9e3ef', + 875, + '#ecf1f7', + ]); + }); + + test('Should snap to end of color stops for identical range', () => { + expect(getOrdinalMbColorRampStops('Blues', 23, 23, false)).toEqual([23, '#6092c0']); }); }); describe('getPercentilesMbColorRampStops', () => { - it('Should create color stops for custom range', () => { + test('Should create color stops', () => { const percentiles = [ { percentile: '50.0', value: 5567.83 }, { percentile: '75.0', value: 8069 }, @@ -69,7 +90,7 @@ describe('getPercentilesMbColorRampStops', () => { { percentile: '95.0', value: 11145.5 }, { percentile: '99.0', value: 16958.18 }, ]; - expect(getPercentilesMbColorRampStops('Blues', percentiles)).toEqual([ + expect(getPercentilesMbColorRampStops('Blues', percentiles, false)).toEqual([ 5567.83, '#e0e8f2', 8069, @@ -82,4 +103,26 @@ describe('getPercentilesMbColorRampStops', () => { '#6092c0', ]); }); + + test('Should create inverted color stops', () => { + const percentiles = [ + { percentile: '50.0', value: 5567.83 }, + { percentile: '75.0', value: 8069 }, + { percentile: '90.0', value: 9581.13 }, + { percentile: '95.0', value: 11145.5 }, + { percentile: '99.0', value: 16958.18 }, + ]; + expect(getPercentilesMbColorRampStops('Blues', percentiles, true)).toEqual([ + 5567.83, + '#6092c0', + 8069, + '#82a7cd', + 9581.13, + '#a2bcd9', + 11145.5, + '#c2d2e6', + 16958.18, + '#e0e8f2', + ]); + }); }); diff --git a/x-pack/plugins/maps/public/classes/styles/color_palettes.ts b/x-pack/plugins/maps/public/classes/styles/color_palettes.ts index 6fa68047acbed..45cd1d988fe6b 100644 --- a/x-pack/plugins/maps/public/classes/styles/color_palettes.ts +++ b/x-pack/plugins/maps/public/classes/styles/color_palettes.ts @@ -153,7 +153,7 @@ export function getColorPalette(colorPaletteId: string): string[] { const colorPalette = COLOR_PALETTES.find(({ value }: COLOR_PALETTE) => { return value === colorPaletteId; }); - return colorPalette ? (colorPalette.palette as string[]) : []; + return colorPalette ? [...(colorPalette.palette as string[])] : []; } export function getColorRampCenterColor(colorPaletteId: string): string | null { @@ -169,7 +169,8 @@ export function getColorRampCenterColor(colorPaletteId: string): string | null { export function getOrdinalMbColorRampStops( colorPaletteId: string | null, min: number, - max: number + max: number, + invert: boolean ): Array | null { if (!colorPaletteId) { return null; @@ -180,6 +181,10 @@ export function getOrdinalMbColorRampStops( } const palette = getColorPalette(colorPaletteId); + if (invert) { + palette.reverse(); + } + if (palette.length === 0) { return null; } @@ -203,7 +208,8 @@ export function getOrdinalMbColorRampStops( // [ stop_input_1: number, stop_output_1: color, stop_input_n: number, stop_output_n: color ] export function getPercentilesMbColorRampStops( colorPaletteId: string | null, - percentiles: PercentilesFieldMeta + percentiles: PercentilesFieldMeta, + invert: boolean ): Array | null { if (!colorPaletteId) { return null; @@ -213,13 +219,17 @@ export function getPercentilesMbColorRampStops( return value === colorPaletteId; }); - return paletteObject - ? paletteObject - .getPalette(percentiles.length) - .reduce((accu: Array, stopColor: string, idx: number) => { - return [...accu, percentiles[idx].value, stopColor]; - }, []) - : null; + if (!paletteObject) { + return null; + } + + const palette = paletteObject.getPalette(percentiles.length); + if (invert) { + palette.reverse(); + } + return palette.reduce((accu: Array, stopColor: string, idx: number) => { + return [...accu, percentiles[idx].value, stopColor]; + }, []); } export function getLinearGradient(colorStrings: string[]): string { diff --git a/x-pack/plugins/maps/public/classes/styles/components/ranged_style_legend_row.tsx b/x-pack/plugins/maps/public/classes/styles/components/ranged_style_legend_row.tsx index b6a0f901b3db6..cf753da8dba0e 100644 --- a/x-pack/plugins/maps/public/classes/styles/components/ranged_style_legend_row.tsx +++ b/x-pack/plugins/maps/public/classes/styles/components/ranged_style_legend_row.tsx @@ -15,6 +15,7 @@ interface Props { maxLabel: string | number; propertyLabel: string; fieldLabel: string; + invert: boolean; } export function RangedStyleLegendRow({ @@ -23,6 +24,7 @@ export function RangedStyleLegendRow({ maxLabel, propertyLabel, fieldLabel, + invert, }: Props) { return (
@@ -41,12 +43,12 @@ export function RangedStyleLegendRow({ - {minLabel} + {invert ? maxLabel : minLabel} - {maxLabel} + {invert ? minLabel : maxLabel} diff --git a/x-pack/plugins/maps/public/classes/styles/heatmap/components/legend/heatmap_legend.tsx b/x-pack/plugins/maps/public/classes/styles/heatmap/components/legend/heatmap_legend.tsx index 5459c55ed95c8..390e48408d747 100644 --- a/x-pack/plugins/maps/public/classes/styles/heatmap/components/legend/heatmap_legend.tsx +++ b/x-pack/plugins/maps/public/classes/styles/heatmap/components/legend/heatmap_legend.tsx @@ -58,6 +58,7 @@ export class HeatmapLegend extends Component { })} propertyLabel={HEATMAP_COLOR_RAMP_LABEL} fieldLabel={this.state.label} + invert={false} /> ); } diff --git a/x-pack/plugins/maps/public/classes/styles/heatmap/heatmap_style.tsx b/x-pack/plugins/maps/public/classes/styles/heatmap/heatmap_style.tsx index a2abbad8cddf0..f4413eea15ac8 100644 --- a/x-pack/plugins/maps/public/classes/styles/heatmap/heatmap_style.tsx +++ b/x-pack/plugins/maps/public/classes/styles/heatmap/heatmap_style.tsx @@ -96,7 +96,8 @@ export class HeatmapStyle implements IStyle { const colorStops = getOrdinalMbColorRampStops( this._descriptor.colorRampName, MIN_RANGE, - MAX_RANGE + MAX_RANGE, + false ); if (colorStops) { mbMap.setPaintProperty(layerId, 'heatmap-color', [ diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js b/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js index 4549367e53470..1409444e0742f 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js @@ -129,16 +129,26 @@ export class ColorMapSelect extends Component { ); } + _getColorPalettes() { + if (this.props.colorMapType === COLOR_MAP_TYPE.CATEGORICAL) { + return CATEGORICAL_COLOR_PALETTES; + } + + return this.props.invert + ? NUMERICAL_COLOR_PALETTES.map((paletteProps) => { + return { + ...paletteProps, + palette: [...paletteProps.palette].reverse(), + }; + }) + : NUMERICAL_COLOR_PALETTES; + } + _renderColorMapSelections() { if (this.props.isCustomOnly) { return null; } - const palettes = - this.props.colorMapType === COLOR_MAP_TYPE.ORDINAL - ? NUMERICAL_COLOR_PALETTES - : CATEGORICAL_COLOR_PALETTES; - const palettesWithCustom = [ { value: CUSTOM_COLOR_MAP, @@ -153,7 +163,7 @@ export class ColorMapSelect extends Component { type: 'text', 'data-test-subj': `colorMapSelectOption_${CUSTOM_COLOR_MAP}`, }, - ...palettes, + ...this._getColorPalettes(), ]; const toggle = this.props.showColorMapTypeToggle ? ( diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js b/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.tsx similarity index 50% rename from x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js rename to x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.tsx index a2dba66d3956b..fd0f367d2954d 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.tsx @@ -6,12 +6,40 @@ */ import _ from 'lodash'; -import React, { Fragment } from 'react'; +import React, { ChangeEvent, ReactNode } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSpacer, + EuiSwitch, + EuiSwitchEvent, +} from '@elastic/eui'; import { FieldSelect } from '../field_select'; +// @ts-expect-error import { ColorMapSelect } from './color_map_select'; -import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { OtherCategoryColorPicker } from './other_category_color_picker'; -import { CATEGORICAL_DATA_TYPES, COLOR_MAP_TYPE } from '../../../../../../common/constants'; +import { + CategoryColorStop, + ColorDynamicOptions, + OrdinalColorStop, +} from '../../../../../../common/descriptor_types'; +import { + CATEGORICAL_DATA_TYPES, + COLOR_MAP_TYPE, + VECTOR_STYLES, +} from '../../../../../../common/constants'; +import { StyleField } from '../../style_fields_helper'; +import { DynamicColorProperty } from '../../properties/dynamic_color_property'; + +interface Props { + fields: StyleField[]; + onDynamicStyleChange: (propertyName: VECTOR_STYLES, options: ColorDynamicOptions) => void; + staticDynamicSelect?: ReactNode; + styleProperty: DynamicColorProperty; + swatches: string[]; +} export function DynamicColorForm({ fields, @@ -19,10 +47,20 @@ export function DynamicColorForm({ staticDynamicSelect, styleProperty, swatches, -}) { +}: Props) { const styleOptions = styleProperty.getOptions(); - const onColorMapSelect = ({ color, customColorMap, type, useCustomColorMap }) => { + const onColorMapSelect = ({ + color, + customColorMap, + type, + useCustomColorMap, + }: { + color?: null | string; + customColorMap?: OrdinalColorStop[] | CategoryColorStop[]; + type: COLOR_MAP_TYPE; + useCustomColorMap: boolean; + }) => { const newColorOptions = { ...styleOptions, type, @@ -30,7 +68,7 @@ export function DynamicColorForm({ if (type === COLOR_MAP_TYPE.ORDINAL) { newColorOptions.useCustomColorRamp = useCustomColorMap; if (customColorMap) { - newColorOptions.customColorRamp = customColorMap; + newColorOptions.customColorRamp = customColorMap as OrdinalColorStop[]; } if (color) { newColorOptions.color = color; @@ -38,7 +76,7 @@ export function DynamicColorForm({ } else { newColorOptions.useCustomColorPalette = useCustomColorMap; if (customColorMap) { - newColorOptions.customColorPalette = customColorMap; + newColorOptions.customColorPalette = customColorMap as CategoryColorStop[]; } if (color) { newColorOptions.colorCategory = color; @@ -48,7 +86,11 @@ export function DynamicColorForm({ onDynamicStyleChange(styleProperty.getStyleName(), newColorOptions); }; - const onFieldChange = async ({ field }) => { + const onFieldChange = ({ field }: { field: StyleField | null }) => { + if (!field) { + return; + } + const { name, origin, type: fieldType } = field; const defaultColorMapType = CATEGORICAL_DATA_TYPES.includes(fieldType) ? COLOR_MAP_TYPE.CATEGORICAL @@ -60,21 +102,28 @@ export function DynamicColorForm({ }); }; - const onColorMapTypeChange = async (e) => { - const colorMapType = e.target.value; + const onColorMapTypeChange = (e: ChangeEvent) => { + const colorMapType = e.target.value as COLOR_MAP_TYPE; onDynamicStyleChange(styleProperty.getStyleName(), { ...styleOptions, type: colorMapType, }); }; - const onOtherCategoryColorChange = (color) => { + const onOtherCategoryColorChange = (color: string) => { onDynamicStyleChange(styleProperty.getStyleName(), { ...styleOptions, otherCategoryColor: color, }); }; + const onInvertChange = (event: EuiSwitchEvent) => { + onDynamicStyleChange(styleProperty.getStyleName(), { + ...styleOptions, + invert: event.target.checked, + }); + }; + const getField = () => { const fieldName = styleProperty.getFieldName(); if (!fieldName) { @@ -92,10 +141,11 @@ export function DynamicColorForm({ return null; } + const invert = styleOptions.invert === undefined ? false : styleOptions.invert; const showColorMapTypeToggle = !CATEGORICAL_DATA_TYPES.includes(field.type); - if (styleProperty.isOrdinal()) { - return ( + return styleProperty.isOrdinal() ? ( + <> - ); - } else if (styleProperty.isCategorical()) { - return ( - <> - - - - ); - } + {!!styleOptions.useCustomColorRamp ? null : ( + + + + )} + + ) : ( + <> + + + + ); }; return ( - + <> - {staticDynamicSelect} + {staticDynamicSelect ? staticDynamicSelect : null} {renderColorMapSelect()} - + ); } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/color/static_color_form.js b/x-pack/plugins/maps/public/classes/styles/vector/components/color/static_color_form.tsx similarity index 61% rename from x-pack/plugins/maps/public/classes/styles/vector/components/color/static_color_form.js rename to x-pack/plugins/maps/public/classes/styles/vector/components/color/static_color_form.tsx index 02716e7994c4d..20f854916a636 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/color/static_color_form.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/color/static_color_form.tsx @@ -5,24 +5,34 @@ * 2.0. */ -import React from 'react'; +import React, { ReactNode } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { VECTOR_STYLES } from '../../../../../../common/constants'; +import { ColorStaticOptions } from '../../../../../../common/descriptor_types'; import { MbValidatedColorPicker } from './mb_validated_color_picker'; +import { StaticColorProperty } from '../../properties/static_color_property'; + +interface Props { + onStaticStyleChange: (propertyName: VECTOR_STYLES, options: ColorStaticOptions) => void; + staticDynamicSelect?: ReactNode; + styleProperty: StaticColorProperty; + swatches: string[]; +} export function StaticColorForm({ onStaticStyleChange, staticDynamicSelect, styleProperty, swatches, -}) { - const onColorChange = (color) => { +}: Props) { + const onColorChange = (color: string) => { onStaticStyleChange(styleProperty.getStyleName(), { color }); }; return ( - {staticDynamicSelect} + {staticDynamicSelect ? staticDynamicSelect : null} , 'children'> & { @@ -21,9 +21,9 @@ type ColorEditorProps = Omit, 'ch export function VectorStyleColorEditor(props: ColorEditorProps) { const colorForm = props.styleProperty.isDynamic() ? ( - + ) : ( - + ); return ( diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/legend/marker_size_legend.tsx b/x-pack/plugins/maps/public/classes/styles/vector/components/legend/marker_size_legend.tsx index c54a42b529a20..11deb6e55942e 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/legend/marker_size_legend.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/legend/marker_size_legend.tsx @@ -91,6 +91,7 @@ export class MarkerSizeLegend extends Component { if (!fieldMeta || !options) { return null; } + const invert = options.invert === undefined ? false : options.invert; const circleStyle = { fillOpacity: 0, @@ -136,14 +137,18 @@ export class MarkerSizeLegend extends Component { // Markers interpolated by area instead of radius to be more consistent with how the human eye+brain perceive shapes // and their visual relevance // This function mirrors output of maplibre expression created from DynamicSizeProperty.getMbSizeExpression - const value = Math.pow(percentage * Math.sqrt(fieldMeta!.delta), 2) + fieldMeta!.min; + const scaledWidth = Math.pow(percentage * Math.sqrt(fieldMeta!.delta), 2); + const value = invert ? fieldMeta!.max - scaledWidth : scaledWidth + fieldMeta!.min; return fieldMeta!.delta > 3 ? Math.round(value) : value; } const markers = []; if (fieldMeta.delta > 0) { - const smallestMarker = makeMarker(options.minSize, this._formatValue(fieldMeta.min)); + const smallestMarker = makeMarker( + options.minSize, + this._formatValue(invert ? fieldMeta.max : fieldMeta.min) + ); markers.push(smallestMarker); const markerDelta = options.maxSize - options.minSize; @@ -156,7 +161,10 @@ export class MarkerSizeLegend extends Component { } } - const largestMarker = makeMarker(options.maxSize, this._formatValue(fieldMeta.max)); + const largestMarker = makeMarker( + options.maxSize, + this._formatValue(invert ? fieldMeta.min : fieldMeta.max) + ); markers.push(largestMarker); return ( diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/legend/ordinal_legend.tsx b/x-pack/plugins/maps/public/classes/styles/vector/components/legend/ordinal_legend.tsx index 2578a908bb68c..048d1bf12a218 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/legend/ordinal_legend.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/legend/ordinal_legend.tsx @@ -138,6 +138,9 @@ export class OrdinalLegend extends Component { this.props.style.isFieldMetaEnabled() && fieldMeta.isMaxOutsideStdRange ? `> ${max}` : max; } + const options = this.props.style.getOptions(); + const invert = options.invert === undefined ? false : options.invert; + return ( { maxLabel={maxLabel} propertyLabel={this.props.style.getDisplayStyleName()} fieldLabel={this.state.label} + invert={invert} /> ); } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.js b/x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.js deleted file mode 100644 index a32ab43f6e070..0000000000000 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { Fragment } from 'react'; -import { FieldSelect } from '../field_select'; -import { SizeRangeSelector } from './size_range_selector'; -import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; - -export function DynamicSizeForm({ - fields, - onDynamicStyleChange, - staticDynamicSelect, - styleProperty, -}) { - const styleOptions = styleProperty.getOptions(); - - const onFieldChange = ({ field }) => { - onDynamicStyleChange(styleProperty.getStyleName(), { ...styleOptions, field }); - }; - - const onSizeRangeChange = ({ minSize, maxSize }) => { - onDynamicStyleChange(styleProperty.getStyleName(), { - ...styleOptions, - minSize, - maxSize, - }); - }; - - let sizeRange; - if (styleOptions.field && styleOptions.field.name) { - sizeRange = ( - - ); - } - - return ( - - - - {staticDynamicSelect} - - - - - - - {sizeRange} - - ); -} diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.tsx b/x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.tsx new file mode 100644 index 0000000000000..8efe05bc2644c --- /dev/null +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/size/dynamic_size_form.tsx @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { ReactNode } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSpacer, + EuiSwitch, + EuiSwitchEvent, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FieldSelect } from '../field_select'; +import { SizeRangeSelector } from './size_range_selector'; +import { SizeDynamicOptions } from '../../../../../../common/descriptor_types'; +import { VECTOR_STYLES } from '../../../../../../common/constants'; +import { DynamicSizeProperty } from '../../properties/dynamic_size_property'; +import { StyleField } from '../../style_fields_helper'; + +interface Props { + fields: StyleField[]; + onDynamicStyleChange: (propertyName: VECTOR_STYLES, options: SizeDynamicOptions) => void; + staticDynamicSelect?: ReactNode; + styleProperty: DynamicSizeProperty; +} + +export function DynamicSizeForm({ + fields, + onDynamicStyleChange, + staticDynamicSelect, + styleProperty, +}: Props) { + const styleOptions = styleProperty.getOptions(); + + const onFieldChange = ({ field }: { field: StyleField | null }) => { + if (field) { + onDynamicStyleChange(styleProperty.getStyleName(), { + ...styleOptions, + field: { name: field.name, origin: field.origin }, + }); + } + }; + + const onSizeRangeChange = ({ minSize, maxSize }: { minSize: number; maxSize: number }) => { + onDynamicStyleChange(styleProperty.getStyleName(), { + ...styleOptions, + minSize, + maxSize, + }); + }; + + const onInvertChange = (event: EuiSwitchEvent) => { + onDynamicStyleChange(styleProperty.getStyleName(), { + ...styleOptions, + invert: event.target.checked, + }); + }; + + let sizeRange; + if (styleOptions.field && styleOptions.field.name) { + sizeRange = ( + <> + + + + + + ); + } + + return ( + <> + + + {staticDynamicSelect} + + + + + + + {sizeRange} + + ); +} diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/size/size_range_selector.js b/x-pack/plugins/maps/public/classes/styles/vector/components/size/size_range_selector.tsx similarity index 74% rename from x-pack/plugins/maps/public/classes/styles/vector/components/size/size_range_selector.js rename to x-pack/plugins/maps/public/classes/styles/vector/components/size/size_range_selector.tsx index b47e22fcd318c..59bdfa3be5938 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/size/size_range_selector.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/size/size_range_selector.tsx @@ -6,13 +6,19 @@ */ import React from 'react'; -import PropTypes from 'prop-types'; import { ValidatedDualRange } from '@kbn/kibana-react-plugin/public'; -import { MIN_SIZE, MAX_SIZE } from '../../vector_style_defaults'; +import { EuiDualRangeProps } from '@elastic/eui/src/components/form/range/dual_range'; import { i18n } from '@kbn/i18n'; +import { MIN_SIZE, MAX_SIZE } from '../../vector_style_defaults'; -export function SizeRangeSelector({ minSize, maxSize, onChange, ...rest }) { - const onSizeChange = ([min, max]) => { +interface Props extends Omit { + minSize: number; + maxSize: number; + onChange: ({ maxSize, minSize }: { maxSize: number; minSize: number }) => void; +} + +export function SizeRangeSelector({ minSize, maxSize, onChange, ...rest }: Props) { + const onSizeChange = ([min, max]: [string, string]) => { onChange({ minSize: Math.max(MIN_SIZE, parseInt(min, 10)), maxSize: Math.min(MAX_SIZE, parseInt(max, 10)), @@ -37,9 +43,3 @@ export function SizeRangeSelector({ minSize, maxSize, onChange, ...rest }) { /> ); } - -SizeRangeSelector.propTypes = { - minSize: PropTypes.number.isRequired, - maxSize: PropTypes.number.isRequired, - onChange: PropTypes.func.isRequired, -}; diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/size/static_size_form.js b/x-pack/plugins/maps/public/classes/styles/vector/components/size/static_size_form.tsx similarity index 69% rename from x-pack/plugins/maps/public/classes/styles/vector/components/size/static_size_form.js rename to x-pack/plugins/maps/public/classes/styles/vector/components/size/static_size_form.tsx index 64ef17b67ed13..a3a7ad7e50bda 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/size/static_size_form.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/size/static_size_form.tsx @@ -5,13 +5,23 @@ * 2.0. */ -import React from 'react'; -import { ValidatedRange } from '../../../../../components/validated_range'; +import React, { ReactNode } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +// @ts-expect-error +import { ValidatedRange } from '../../../../../components/validated_range'; +import { SizeStaticOptions } from '../../../../../../common/descriptor_types'; +import { VECTOR_STYLES } from '../../../../../../common/constants'; +import { StaticSizeProperty } from '../../properties/static_size_property'; + +interface Props { + onStaticStyleChange: (propertyName: VECTOR_STYLES, options: SizeStaticOptions) => void; + staticDynamicSelect?: ReactNode; + styleProperty: StaticSizeProperty; +} -export function StaticSizeForm({ onStaticStyleChange, staticDynamicSelect, styleProperty }) { - const onSizeChange = (size) => { +export function StaticSizeForm({ onStaticStyleChange, staticDynamicSelect, styleProperty }: Props) { + const onSizeChange = (size: number) => { onStaticStyleChange(styleProperty.getStyleName(), { size }); }; diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/size/vector_style_size_editor.tsx b/x-pack/plugins/maps/public/classes/styles/vector/components/size/vector_style_size_editor.tsx index b34ff87618f33..7c4aac0f32e25 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/size/vector_style_size_editor.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/size/vector_style_size_editor.tsx @@ -8,19 +8,19 @@ import React from 'react'; import { Props, StylePropEditor } from '../style_prop_editor'; -// @ts-expect-error import { DynamicSizeForm } from './dynamic_size_form'; -// @ts-expect-error import { StaticSizeForm } from './static_size_form'; import { SizeDynamicOptions, SizeStaticOptions } from '../../../../../../common/descriptor_types'; +import { DynamicSizeProperty } from '../../properties/dynamic_size_property'; +import { StaticSizeProperty } from '../../properties/static_size_property'; type SizeEditorProps = Omit, 'children'>; export function VectorStyleSizeEditor(props: SizeEditorProps) { const sizeForm = props.styleProperty.isDynamic() ? ( - + ) : ( - + ); return {sizeForm}; diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.tsx b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.tsx index cbf5a4fe868e7..091c4c4e36b2e 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_color_property.tsx @@ -142,6 +142,7 @@ export class DynamicColorProperty extends DynamicStyleProperty | null = null; let getValuePrefix: ((i: number, isNext: boolean) => string) | null = null; if (this._options.useCustomColorRamp) { @@ -364,7 +368,8 @@ export class DynamicColorProperty extends DynamicStyleProperty } + invert={false} maxLabel="100_format" minLabel="0_format" propertyLabel="Border width" diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_size_property/dynamic_size_property.tsx b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_size_property/dynamic_size_property.tsx index 83ac50c7b4eaa..13d93dffaaec0 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_size_property/dynamic_size_property.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_size_property/dynamic_size_property.tsx @@ -112,17 +112,25 @@ export class DynamicSizeProperty extends DynamicStyleProperty Date: Mon, 24 Oct 2022 22:16:27 +0100 Subject: [PATCH 13/15] skip flaky suite (#142222) --- test/functional/apps/discover/group1/_discover.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/apps/discover/group1/_discover.ts b/test/functional/apps/discover/group1/_discover.ts index d6035d0a28a6e..4b5137fadeb5c 100644 --- a/test/functional/apps/discover/group1/_discover.ts +++ b/test/functional/apps/discover/group1/_discover.ts @@ -26,7 +26,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { defaultIndex: 'logstash-*', }; - describe('discover test', function describeIndexTests() { + // FLAKY: https://github.com/elastic/kibana/issues/142222 + describe.skip('discover test', function describeIndexTests() { before(async function () { log.debug('load kibana index with default index pattern'); await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); From f40227427078bbc3aaf8a718efceb6d189c204eb Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 24 Oct 2022 22:19:12 +0100 Subject: [PATCH 14/15] skip flaky suite (#143907, #143908) --- .../lens/public/datasources/form_based/form_based.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts index 2881511d704ac..7c575241ba30b 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts +++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.test.ts @@ -1789,7 +1789,9 @@ describe('IndexPattern Data Source', () => { }); }); - describe('#clearLayer', () => { + // FLAKY: https://github.com/elastic/kibana/issues/143908 + // FLAKY: https://github.com/elastic/kibana/issues/143907 + describe.skip('#clearLayer', () => { it('should clear a layer', () => { const state = { layers: { From 21c7f5e074db54fd1c79f35ef5979b61ef0aa0df Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Mon, 24 Oct 2022 14:56:36 -0700 Subject: [PATCH 15/15] [Security Solution][Alerts] Refactor rule execution logic integration tests (#142679) * Separate rule execution logic tests and move bulk of the tests to preview for speed * Remove bad dependency * Update unit test snapshot * Fix flaky test * Fix another flaky test * Fix more imports * Remove superfluous return type --- .buildkite/ftr_configs.yml | 1 + .../signals/executors/threshold.ts | 2 +- .../get_threshold_signal_history.test.ts.snap | 4 +- .../get_threshold_signal_history.test.ts | 8 +- .../threshold/get_threshold_signal_history.ts | 13 +- .../common/config.ts | 1 + .../group1/create_new_terms.ts | 428 +---- .../group1/create_threat_matching.ts | 1421 --------------- .../group1/generating_signals.ts | 1554 ----------------- .../security_and_spaces/group1/index.ts | 3 - .../group1/preview_rules.ts | 4 +- .../rule_execution_logic/README.md | 11 + .../rule_execution_logic/config.ts | 18 + .../rule_execution_logic/eql.ts | 606 +++++++ .../rule_execution_logic/index.ts | 21 + .../machine_learning.ts} | 141 +- .../rule_execution_logic/new_terms.ts | 385 ++++ .../rule_execution_logic/query.ts | 426 +++++ .../rule_execution_logic/saved_query.ts | 85 + .../rule_execution_logic/threat_match.ts | 1306 ++++++++++++++ .../rule_execution_logic/threshold.ts | 385 ++++ .../utils/get_preview_alerts.ts | 45 + .../utils/index.ts | 4 + .../utils/machine_learning_setup.ts | 54 + .../utils/preview_rule.ts | 50 + .../preview_rule_with_exception_entries.ts | 63 + .../alerts/8.0.0/data.json.gz | Bin 9231 -> 9203 bytes .../alerts/8.0.0/mappings.json.gz | Bin 9711 -> 9609 bytes .../alerts/8.1.0/data.json.gz | Bin 9500 -> 9456 bytes .../alerts/8.1.0/mappings.json.gz | Bin 9739 -> 9616 bytes .../tests/basic/search_strategy.ts | 3 +- 31 files changed, 3542 insertions(+), 3500 deletions(-) delete mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts delete mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/README.md create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/eql.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/index.ts rename x-pack/test/detection_engine_api_integration/security_and_spaces/{group1/create_ml.ts => rule_execution_logic/machine_learning.ts} (69%) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/saved_query.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threat_match.ts create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threshold.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/get_preview_alerts.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/machine_learning_setup.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/preview_rule.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/preview_rule_with_exception_entries.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index aa35797d1f986..cc625a09fadd0 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -142,6 +142,7 @@ enabled: - x-pack/test/detection_engine_api_integration/security_and_spaces/group8/config.ts - x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts + - x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts - x-pack/test/encrypted_saved_objects_api_integration/config.ts - x-pack/test/endpoint_api_integration_no_ingest/config.ts - x-pack/test/examples/config.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 7a148e80282dc..515caf5dcd5e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -92,7 +92,7 @@ export const thresholdExecutor = async ({ : await getThresholdSignalHistory({ from: tuple.from.toISOString(), to: tuple.to.toISOString(), - ruleId: ruleParams.ruleId, + frameworkRuleId: completeRule.alertId, bucketByFields: ruleParams.threshold.field, ruleDataReader, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/__snapshots__/get_threshold_signal_history.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/__snapshots__/get_threshold_signal_history.test.ts.snap index 005ce07afcac4..bb9e29f1f5b52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/__snapshots__/get_threshold_signal_history.test.ts.snap +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/__snapshots__/get_threshold_signal_history.test.ts.snap @@ -17,7 +17,7 @@ Object { }, Object { "term": Object { - "signal.rule.rule_id": "threshold-rule", + "kibana.alert.rule.uuid": "threshold-rule", }, }, Object { @@ -91,7 +91,7 @@ Object { }, Object { "term": Object { - "signal.rule.rule_id": "threshold-rule", + "kibana.alert.rule.uuid": "threshold-rule", }, }, Object { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.test.ts index d9db2972a89d3..0e896ad49cd82 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.test.ts @@ -12,10 +12,10 @@ describe('buildPreviousThresholdAlertRequest', () => { const bucketByFields: string[] = []; const to = 'now'; const from = 'now-6m'; - const ruleId = 'threshold-rule'; + const frameworkRuleId = 'threshold-rule'; expect( - buildPreviousThresholdAlertRequest({ from, to, ruleId, bucketByFields }) + buildPreviousThresholdAlertRequest({ from, to, frameworkRuleId, bucketByFields }) ).toMatchSnapshot(); }); @@ -23,10 +23,10 @@ describe('buildPreviousThresholdAlertRequest', () => { const bucketByFields: string[] = ['host.name', 'user.name']; const to = 'now'; const from = 'now-6m'; - const ruleId = 'threshold-rule'; + const frameworkRuleId = 'threshold-rule'; expect( - buildPreviousThresholdAlertRequest({ from, to, ruleId, bucketByFields }) + buildPreviousThresholdAlertRequest({ from, to, frameworkRuleId, bucketByFields }) ).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts index 997e6c213f3b1..4826cd574a906 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/get_threshold_signal_history.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { IRuleDataReader } from '@kbn/rule-registry-plugin/server'; +import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; import type { ThresholdSignalHistory } from '../types'; import { buildThresholdSignalHistory } from './build_signal_history'; import { createErrorsFromShard } from '../utils'; @@ -14,7 +15,7 @@ import { createErrorsFromShard } from '../utils'; interface GetThresholdSignalHistoryParams { from: string; to: string; - ruleId: string; + frameworkRuleId: string; bucketByFields: string[]; ruleDataReader: IRuleDataReader; } @@ -22,7 +23,7 @@ interface GetThresholdSignalHistoryParams { export const getThresholdSignalHistory = async ({ from, to, - ruleId, + frameworkRuleId, bucketByFields, ruleDataReader, }: GetThresholdSignalHistoryParams): Promise<{ @@ -32,7 +33,7 @@ export const getThresholdSignalHistory = async ({ const request = buildPreviousThresholdAlertRequest({ from, to, - ruleId, + frameworkRuleId, bucketByFields, }); @@ -48,12 +49,12 @@ export const getThresholdSignalHistory = async ({ export const buildPreviousThresholdAlertRequest = ({ from, to, - ruleId, + frameworkRuleId, bucketByFields, }: { from: string; to: string; - ruleId: string; + frameworkRuleId: string; bucketByFields: string[]; }): estypes.SearchRequest => { return { @@ -80,7 +81,7 @@ export const buildPreviousThresholdAlertRequest = ({ }, { term: { - 'signal.rule.rule_id': ruleId, + [ALERT_RULE_UUID]: frameworkRuleId, }, }, // We might find a signal that was generated on the interval for old data... make sure to exclude those. diff --git a/x-pack/test/detection_engine_api_integration/common/config.ts b/x-pack/test/detection_engine_api_integration/common/config.ts index a95cb937d4cd9..fbbe7fd62a7a8 100644 --- a/x-pack/test/detection_engine_api_integration/common/config.ts +++ b/x-pack/test/detection_engine_api_integration/common/config.ts @@ -77,6 +77,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', ])}`, + '--xpack.task_manager.poll_interval=1000', ...(ssl ? [ `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts index bfb369e0091b5..095ce3766918d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts @@ -5,78 +5,26 @@ * 2.0. */ -import { orderBy } from 'lodash'; import expect from '@kbn/expect'; import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; -import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/rule_monitoring'; -import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks'; -import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { - createRule, - createRuleWithExceptionEntries, - createSignalsIndex, - deleteAllAlerts, - deleteSignalsIndex, - getOpenSignals, - getSignalsByIds, - waitForRuleSuccessOrStatus, - waitForSignalsToBePresent, -} from '../../utils'; +import { deleteAllAlerts } from '../../utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); const log = getService('log'); - const es = getService('es'); /** * Specific api integration tests for threat matching rule type */ describe('create_new_terms', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - beforeEach(async () => { - await createSignalsIndex(supertest, log); - }); - afterEach(async () => { - await deleteSignalsIndex(supertest, log); await deleteAllAlerts(supertest, log); }); - it('should create a single rule with a rule_id and validate it ran successfully', async () => { - const ruleResponse = await createRule( - supertest, - log, - getCreateNewTermsRulesSchemaMock('rule-1', true) - ); - - await waitForRuleSuccessOrStatus( - supertest, - log, - ruleResponse.id, - RuleExecutionStatus.succeeded - ); - - const { body: rule } = await supertest - .get(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .query({ id: ruleResponse.id }) - .expect(200); - - expect(rule?.execution_summary?.last_execution.status).to.eql('succeeded'); - }); - it('should not be able to create a new terms rule with too small history window', async () => { const rule = { ...getCreateNewTermsRulesSchemaMock('rule-1'), @@ -92,379 +40,5 @@ export default ({ getService }: FtrProviderContext) => { "params invalid: History window size is smaller than rule interval + additional lookback, 'historyWindowStart' must be earlier than 'from'" ); }); - - const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { - if (!alert) { - return undefined; - } - const { - 'kibana.version': version, - 'kibana.alert.rule.execution.uuid': execUuid, - 'kibana.alert.rule.uuid': uuid, - '@timestamp': timestamp, - 'kibana.alert.rule.created_at': createdAt, - 'kibana.alert.rule.updated_at': updatedAt, - 'kibana.alert.uuid': alertUuid, - ...restOfAlert - } = alert; - return restOfAlert; - }; - - // This test also tests that alerts are NOT created for terms that are not new: the host name - // suricata-sensor-san-francisco appears in a document at 2019-02-19T20:42:08.230Z, but also appears - // in earlier documents so is not new. An alert should not be generated for that term. - it('should generate 1 alert with 1 selected field', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['host.name'], - from: '2019-02-19T20:42:00.000Z', - history_window_start: '2019-01-19T20:42:00.000Z', - }; - - const createdRule = await createRule(supertest, log, rule); - - await waitForRuleSuccessOrStatus( - supertest, - log, - createdRule.id, - RuleExecutionStatus.succeeded - ); - - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(1); - expect(removeRandomValuedProperties(signalsOpen.hits.hits[0]._source)).eql({ - 'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'], - 'kibana.alert.rule.category': 'New Terms Rule', - 'kibana.alert.rule.consumer': 'siem', - 'kibana.alert.rule.name': 'Query with a rule id', - 'kibana.alert.rule.producer': 'siem', - 'kibana.alert.rule.rule_type_id': 'siem.newTermsRule', - 'kibana.space_ids': ['default'], - 'kibana.alert.rule.tags': [], - agent: { - ephemeral_id: '7cc2091a-72f1-4c63-843b-fdeb622f9c69', - hostname: 'zeek-newyork-sha-aa8df15', - id: '4b4462ef-93d2-409c-87a6-299d942e5047', - type: 'auditbeat', - version: '8.0.0', - }, - cloud: { instance: { id: '139865230' }, provider: 'digitalocean', region: 'nyc1' }, - ecs: { version: '1.0.0-beta2' }, - host: { - architecture: 'x86_64', - hostname: 'zeek-newyork-sha-aa8df15', - id: '3729d06ce9964aa98549f41cbd99334d', - ip: ['157.230.208.30', '10.10.0.6', 'fe80::24ce:f7ff:fede:a571'], - mac: ['26:ce:f7:de:a5:71'], - name: 'zeek-newyork-sha-aa8df15', - os: { - codename: 'cosmic', - family: 'debian', - kernel: '4.18.0-10-generic', - name: 'Ubuntu', - platform: 'ubuntu', - version: '18.10 (Cosmic Cuttlefish)', - }, - }, - message: - 'Login by user root (UID: 0) on pts/0 (PID: 20638) from 8.42.77.171 (IP: 8.42.77.171)', - process: { pid: 20638 }, - service: { type: 'system' }, - source: { ip: '8.42.77.171' }, - user: { id: 0, name: 'root', terminal: 'pts/0' }, - 'event.action': 'user_login', - 'event.category': 'authentication', - 'event.dataset': 'login', - 'event.kind': 'signal', - 'event.module': 'system', - 'event.origin': '/var/log/wtmp', - 'event.outcome': 'success', - 'event.type': 'authentication_success', - 'kibana.alert.original_time': '2019-02-19T20:42:08.230Z', - 'kibana.alert.ancestors': [ - { - id: 'x07wJ2oB9v5HJNSHhyxi', - type: 'event', - index: 'auditbeat-8.0.0-2019.02.19-000001', - depth: 0, - }, - ], - 'kibana.alert.status': 'active', - 'kibana.alert.workflow_status': 'open', - 'kibana.alert.depth': 1, - 'kibana.alert.reason': - 'authentication event with source 8.42.77.171 by root on zeek-newyork-sha-aa8df15 created high alert Query with a rule id.', - 'kibana.alert.severity': 'high', - 'kibana.alert.risk_score': 55, - 'kibana.alert.rule.parameters': { - description: 'Detecting root and admin users', - risk_score: 55, - severity: 'high', - author: [], - false_positives: [], - from: '2019-02-19T20:42:00.000Z', - rule_id: 'rule-1', - max_signals: 100, - risk_score_mapping: [], - severity_mapping: [], - threat: [], - to: 'now', - references: [], - version: 1, - exceptions_list: [], - immutable: false, - related_integrations: [], - required_fields: [], - setup: '', - type: 'new_terms', - query: '*', - new_terms_fields: ['host.name'], - history_window_start: '2019-01-19T20:42:00.000Z', - index: ['auditbeat-*'], - language: 'kuery', - }, - 'kibana.alert.rule.actions': [], - 'kibana.alert.rule.author': [], - 'kibana.alert.rule.created_by': 'elastic', - 'kibana.alert.rule.description': 'Detecting root and admin users', - 'kibana.alert.rule.enabled': true, - 'kibana.alert.rule.exceptions_list': [], - 'kibana.alert.rule.false_positives': [], - 'kibana.alert.rule.from': '2019-02-19T20:42:00.000Z', - 'kibana.alert.rule.immutable': false, - 'kibana.alert.rule.indices': ['auditbeat-*'], - 'kibana.alert.rule.interval': '5m', - 'kibana.alert.rule.max_signals': 100, - 'kibana.alert.rule.references': [], - 'kibana.alert.rule.risk_score_mapping': [], - 'kibana.alert.rule.rule_id': 'rule-1', - 'kibana.alert.rule.severity_mapping': [], - 'kibana.alert.rule.threat': [], - 'kibana.alert.rule.to': 'now', - 'kibana.alert.rule.type': 'new_terms', - 'kibana.alert.rule.updated_by': 'elastic', - 'kibana.alert.rule.version': 1, - 'kibana.alert.rule.risk_score': 55, - 'kibana.alert.rule.severity': 'high', - 'kibana.alert.original_event.action': 'user_login', - 'kibana.alert.original_event.category': 'authentication', - 'kibana.alert.original_event.dataset': 'login', - 'kibana.alert.original_event.kind': 'event', - 'kibana.alert.original_event.module': 'system', - 'kibana.alert.original_event.origin': '/var/log/wtmp', - 'kibana.alert.original_event.outcome': 'success', - 'kibana.alert.original_event.type': 'authentication_success', - }); - }); - - it('should generate 3 alerts when 1 document has 3 new values', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['host.ip'], - from: '2019-02-19T20:42:00.000Z', - history_window_start: '2019-01-19T20:42:00.000Z', - }; - - const createdRule = await createRule(supertest, log, rule); - - await waitForRuleSuccessOrStatus( - supertest, - log, - createdRule.id, - RuleExecutionStatus.succeeded - ); - - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(3); - const signalsOrderedByHostIp = orderBy( - signalsOpen.hits.hits, - '_source.kibana.alert.new_terms', - 'asc' - ); - expect(signalsOrderedByHostIp[0]._source?.['kibana.alert.new_terms']).eql(['10.10.0.6']); - expect(signalsOrderedByHostIp[1]._source?.['kibana.alert.new_terms']).eql(['157.230.208.30']); - expect(signalsOrderedByHostIp[2]._source?.['kibana.alert.new_terms']).eql([ - 'fe80::24ce:f7ff:fede:a571', - ]); - }); - - it('should generate alerts for every term when history window is small', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['host.name'], - from: '2019-02-19T20:42:00.000Z', - // Set the history_window_start close to 'from' so we should alert on all terms in the time range - history_window_start: '2019-02-19T20:41:59.000Z', - }; - - const createdRule = await createRule(supertest, log, rule); - - await waitForRuleSuccessOrStatus( - supertest, - log, - createdRule.id, - RuleExecutionStatus.succeeded - ); - - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(5); - const hostNames = signalsOpen.hits.hits - .map((signal) => signal._source?.['kibana.alert.new_terms']) - .sort(); - expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); - expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); - expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); - expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); - expect(hostNames[4]).eql(['zeek-sensor-san-francisco']); - }); - - describe('timestamp override and fallback', () => { - before(async () => { - await esArchiver.load( - 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' - ); - await esArchiver.load( - 'x-pack/test/functional/es_archives/security_solution/timestamp_override_3' - ); - }); - after(async () => { - await esArchiver.unload( - 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' - ); - await esArchiver.unload( - 'x-pack/test/functional/es_archives/security_solution/timestamp_override_3' - ); - }); - - it('should generate the correct alerts', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - // myfakeindex-3 does not have event.ingested mapped so we can test if the runtime field - // 'kibana.combined_timestamp' handles unmapped fields properly - index: ['timestamp-fallback-test', 'myfakeindex-3'], - new_terms_fields: ['host.name'], - from: '2020-12-16T16:00:00.000Z', - // Set the history_window_start close to 'from' so we should alert on all terms in the time range - history_window_start: '2020-12-16T15:59:00.000Z', - timestamp_override: 'event.ingested', - }; - - const createdRule = await createRule(supertest, log, rule); - - await waitForSignalsToBePresent(supertest, log, 2, [createdRule.id]); - - const signalsOpen = await getSignalsByIds(supertest, log, [createdRule.id]); - expect(signalsOpen.hits.hits.length).eql(2); - const hostNames = signalsOpen.hits.hits - .map((signal) => signal._source?.['kibana.alert.new_terms']) - .sort(); - expect(hostNames[0]).eql(['host-3']); - expect(hostNames[1]).eql(['host-4']); - }); - }); - - it('should apply exceptions', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['host.name'], - from: '2019-02-19T20:42:00.000Z', - // Set the history_window_start close to 'from' so we should alert on all terms in the time range - history_window_start: '2019-02-19T20:41:59.000Z', - }; - const createdRule = await createRuleWithExceptionEntries(supertest, log, rule, [ - [ - { - field: 'host.name', - operator: 'included', - type: 'match', - value: 'zeek-sensor-san-francisco', - }, - ], - ]); - - await waitForRuleSuccessOrStatus( - supertest, - log, - createdRule.id, - RuleExecutionStatus.succeeded - ); - - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(4); - const hostNames = signalsOpen.hits.hits - .map((signal) => signal._source?.['kibana.alert.new_terms']) - .sort(); - expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); - expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); - expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); - expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); - }); - - it('should work for max signals > 100', async () => { - const maxSignals = 200; - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['process.pid'], - from: '2018-02-19T20:42:00.000Z', - // Set the history_window_start close to 'from' so we should alert on all terms in the time range - history_window_start: '2018-02-19T20:41:59.000Z', - max_signals: maxSignals, - }; - - const createdRule = await createRule(supertest, log, rule); - - await waitForRuleSuccessOrStatus( - supertest, - log, - createdRule.id, - RuleExecutionStatus.succeeded - ); - - const signalsOpen = await getOpenSignals( - supertest, - log, - es, - createdRule, - RuleExecutionStatus.succeeded, - maxSignals - ); - expect(signalsOpen.hits.hits.length).eql(maxSignals); - const processPids = signalsOpen.hits.hits - .map((signal) => signal._source?.['kibana.alert.new_terms']) - .sort(); - expect(processPids[0]).eql([1]); - }); - - describe('alerts should be be enriched', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - it('should be enriched with host risk score', async () => { - const rule: NewTermsRuleCreateProps = { - ...getCreateNewTermsRulesSchemaMock('rule-1', true), - new_terms_fields: ['host.name'], - from: '2019-02-19T20:42:00.000Z', - history_window_start: '2019-01-19T20:42:00.000Z', - }; - - const createdRule = await createRule(supertest, log, rule); - - await waitForRuleSuccessOrStatus( - supertest, - log, - createdRule.id, - RuleExecutionStatus.succeeded - ); - - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); - expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23); - }); - }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts deleted file mode 100644 index b18f716d17d4e..0000000000000 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_threat_matching.ts +++ /dev/null @@ -1,1421 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { get, isEqual } from 'lodash'; -import expect from '@kbn/expect'; -import { - ALERT_REASON, - ALERT_RULE_UUID, - ALERT_STATUS, - ALERT_RULE_NAMESPACE, - ALERT_RULE_UPDATED_AT, - ALERT_UUID, - ALERT_WORKFLOW_STATUS, - SPACE_IDS, - VERSION, -} from '@kbn/rule-data-utils'; -import { flattenWithPrefix } from '@kbn/securitysolution-rules'; - -import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; -import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/rule_monitoring'; -import { RuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; -import { - getCreateThreatMatchRulesSchemaMock, - getThreatMatchingSchemaPartialMock, -} from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks'; -import { ENRICHMENT_TYPES } from '@kbn/security-solution-plugin/common/cti/constants'; -import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; -import { - ALERT_ANCESTORS, - ALERT_DEPTH, - ALERT_ORIGINAL_EVENT_ACTION, - ALERT_ORIGINAL_EVENT_CATEGORY, - ALERT_ORIGINAL_EVENT_MODULE, - ALERT_ORIGINAL_TIME, -} from '@kbn/security-solution-plugin/common/field_maps/field_names'; -import { - createRule, - createSignalsIndex, - deleteAllAlerts, - deleteSignalsIndex, - getSignalsByIds, - removeServerGeneratedProperties, - waitForRuleSuccessOrStatus, - waitForSignalsToBePresent, -} from '../../utils'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -const format = (value: unknown): string => JSON.stringify(value, null, 2); - -// Asserts that each expected value is included in the subject, independent of -// ordering. Uses _.isEqual for value comparison. -const assertContains = (subject: unknown[], expected: unknown[]) => - expected.forEach((expectedValue) => - expect(subject.some((value) => isEqual(value, expectedValue))).to.eql( - true, - `expected ${format(subject)} to contain ${format(expectedValue)}` - ) - ); - -// eslint-disable-next-line import/no-default-export -export default ({ getService }: FtrProviderContext) => { - const esArchiver = getService('esArchiver'); - const supertest = getService('supertest'); - const log = getService('log'); - - /** - * Specific api integration tests for threat matching rule type - */ - describe('create_threat_matching', () => { - describe('creating threat match rule', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - beforeEach(async () => { - await createSignalsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest, log); - await deleteAllAlerts(supertest, log); - }); - - it('should create a single rule with a rule_id', async () => { - const ruleResponse = await createRule( - supertest, - log, - getCreateThreatMatchRulesSchemaMock() - ); - const bodyToCompare = removeServerGeneratedProperties(ruleResponse); - expect(bodyToCompare).to.eql(getThreatMatchingSchemaPartialMock()); - }); - - it('should create a single rule with a rule_id and validate it ran successfully', async () => { - const ruleResponse = await createRule( - supertest, - log, - getCreateThreatMatchRulesSchemaMock('rule-1', true) - ); - - await waitForRuleSuccessOrStatus( - supertest, - log, - ruleResponse.id, - RuleExecutionStatus.succeeded - ); - - const { body: rule } = await supertest - .get(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .query({ id: ruleResponse.id }) - .expect(200); - - const bodyToCompare = removeServerGeneratedProperties(ruleResponse); - expect(bodyToCompare).to.eql(getThreatMatchingSchemaPartialMock(true)); - - // TODO: https://github.com/elastic/kibana/pull/121644 clean up, make type-safe - expect(rule?.execution_summary?.last_execution.status).to.eql('succeeded'); - }); - }); - - describe('tests with auditbeat data', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - beforeEach(async () => { - await deleteAllAlerts(supertest, log); - await createSignalsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest, log); - await deleteAllAlerts(supertest, log); - }); - - it('should be able to execute and get 10 signals when doing a specific query', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip - threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity - threat_mapping: [ - // We match host.name against host.name - { - entries: [ - { - field: 'host.name', - value: 'host.name', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const createdRule = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, createdRule.id); - await waitForSignalsToBePresent(supertest, log, 10, [createdRule.id]); - const signalsOpen = await getSignalsByIds(supertest, log, [createdRule.id]); - expect(signalsOpen.hits.hits.length).equal(10); - const fullSource = signalsOpen.hits.hits.find( - (signal) => - (signal._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id === '7yJ-B2kBR346wHgnhlMn' - ); - const fullSignal = fullSource?._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - expect(fullSignal).eql({ - ...fullSignal, - '@timestamp': fullSignal['@timestamp'], - agent: { - ephemeral_id: '1b4978a0-48be-49b1-ac96-323425b389ab', - hostname: 'zeek-sensor-amsterdam', - id: 'e52588e6-7aa3-4c89-a2c4-d6bc5c286db1', - type: 'auditbeat', - version: '8.0.0', - }, - auditd: { - data: { - hostname: '46.101.47.213', - op: 'PAM:bad_ident', - terminal: 'ssh', - }, - message_type: 'user_err', - result: 'fail', - sequence: 2267, - session: 'unset', - summary: { - actor: { - primary: 'unset', - secondary: 'root', - }, - how: '/usr/sbin/sshd', - object: { - primary: 'ssh', - secondary: '46.101.47.213', - type: 'user-session', - }, - }, - }, - cloud: { - instance: { - id: '133551048', - }, - provider: 'digitalocean', - region: 'ams3', - }, - ecs: { - version: '1.0.0-beta2', - }, - ...flattenWithPrefix('event', { - action: 'error', - category: 'user-login', - module: 'auditd', - kind: 'signal', - }), - host: { - architecture: 'x86_64', - containerized: false, - hostname: 'zeek-sensor-amsterdam', - id: '2ce8b1e7d69e4a1d9c6bcddc473da9d9', - name: 'zeek-sensor-amsterdam', - os: { - codename: 'bionic', - family: 'debian', - kernel: '4.15.0-45-generic', - name: 'Ubuntu', - platform: 'ubuntu', - version: '18.04.2 LTS (Bionic Beaver)', - }, - }, - network: { - direction: 'incoming', - }, - process: { - executable: '/usr/sbin/sshd', - pid: 32739, - }, - service: { - type: 'auditd', - }, - source: { - ip: '46.101.47.213', - }, - user: { - audit: { - id: 'unset', - }, - id: '0', - name: 'root', - }, - [ALERT_ANCESTORS]: [ - { - id: '7yJ-B2kBR346wHgnhlMn', - type: 'event', - index: 'auditbeat-8.0.0-2019.02.19-000001', - depth: 0, - }, - ], - [ALERT_DEPTH]: 1, - [ALERT_ORIGINAL_EVENT_ACTION]: 'error', - [ALERT_ORIGINAL_EVENT_CATEGORY]: 'user-login', - [ALERT_ORIGINAL_EVENT_MODULE]: 'auditd', - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_REASON]: - 'user-login event with source 46.101.47.213 by root on zeek-sensor-amsterdam created high alert Query with a rule id.', - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_STATUS]: 'active', - [ALERT_UUID]: fullSignal[ALERT_UUID], - [ALERT_WORKFLOW_STATUS]: 'open', - [SPACE_IDS]: ['default'], - [VERSION]: fullSignal[VERSION], - threat: { - enrichments: get(fullSignal, 'threat.enrichments'), - }, - ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { - actions: [], - author: [], - category: 'Indicator Match Rule', - consumer: 'siem', - created_at: createdRule.created_at, - created_by: 'elastic', - description: 'Detecting root and admin users', - enabled: true, - exceptions_list: [], - false_positives: [], - from: '1900-01-01T00:00:00.000Z', - immutable: false, - interval: '5m', - max_signals: 100, - name: 'Query with a rule id', - producer: 'siem', - references: [], - risk_score: 55, - risk_score_mapping: [], - rule_id: createdRule.rule_id, - rule_type_id: 'siem.indicatorRule', - severity: 'high', - severity_mapping: [], - tags: [], - threat: [], - to: 'now', - type: 'threat_match', - updated_at: fullSignal[ALERT_RULE_UPDATED_AT], - updated_by: 'elastic', - uuid: createdRule.id, - version: 1, - }), - }); - }); - - it('should return 0 matches if the mapping does not match against anything in the mapping', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip - threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity - threat_mapping: [ - // We match host.name against host.name - { - entries: [ - { - field: 'host.name', - value: 'invalid.mapping.value', // invalid mapping value - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const ruleResponse = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, ruleResponse.id); - const signalsOpen = await getSignalsByIds(supertest, log, [ruleResponse.id]); - expect(signalsOpen.hits.hits.length).equal(0); - }); - - it('should return 0 signals when using an AND and one of the clauses does not have data', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip - threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity - threat_mapping: [ - { - entries: [ - { - field: 'source.ip', - value: 'source.ip', - type: 'mapping', - }, - { - field: 'source.ip', - value: 'destination.ip', // All records from the threat query do NOT have destination.ip, so those records that do not should drop this entire AND clause. - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const ruleResponse = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, ruleResponse.id); - const signalsOpen = await getSignalsByIds(supertest, log, [ruleResponse.id]); - expect(signalsOpen.hits.hits.length).equal(0); - }); - - it('should return 0 signals when using an AND and one of the clauses has a made up value that does not exist', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - type: 'threat_match', - index: ['auditbeat-*'], - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip - threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity - threat_mapping: [ - { - entries: [ - { - field: 'source.ip', - value: 'source.ip', - type: 'mapping', - }, - { - field: 'source.ip', - value: 'made.up.non.existent.field', // made up field should not match - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const ruleResponse = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, ruleResponse.id); - const signalsOpen = await getSignalsByIds(supertest, log, [ruleResponse.id]); - expect(signalsOpen.hits.hits.length).equal(0); - }); - - describe('timeout behavior', () => { - // Flaky - it.skip('will return an error if a rule execution exceeds the rule interval', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a short interval', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: '*:*', // broad query to take more time - threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity - threat_mapping: [ - { - entries: [ - { - field: 'host.name', - value: 'host.name', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - concurrent_searches: 1, - interval: '1s', // short interval - items_per_search: 1, // iterate only 1 threat item per loop to ensure we're slow - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id, RuleExecutionStatus.failed); - - const { body } = await supertest - .post(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .query({ id }) - .expect(200); - - // TODO: https://github.com/elastic/kibana/pull/121644 clean up, make type-safe - expect(body?.execution_summary?.last_execution.message).to.contain( - 'execution has exceeded its allotted interval' - ); - }); - }); - - describe('indicator enrichment: threat-first search', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/filebeat/threat_intel'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/filebeat/threat_intel'); - }); - - it('enriches signals with the single indicator that matched', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', // narrow events down to 2 with a destination.ip - threat_indicator_path: 'threat.indicator', - threat_query: 'threat.indicator.domain: 159.89.119.67', // narrow things down to indicators with a domain - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.domain', - field: 'destination.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(2); - - const { hits } = signalsOpen.hits; - const threats = hits.map((hit) => hit._source?.threat); - expect(threats).to.eql([ - { - enrichments: [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - type: 'url', - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ], - }, - { - enrichments: [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - type: 'url', - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ], - }, - ]); - }); - - it('enriches signals with multiple indicators if several matched', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: 'NOT source.port:35326', // specify query to have signals more than treat indicators, but only 1 will match - threat_indicator_path: 'threat.indicator', - threat_query: 'threat.indicator.ip: *', - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.ip', - field: 'source.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(1); - - const { hits } = signalsOpen.hits; - const [threat] = hits.map((hit) => hit._source?.threat) as Array<{ - enrichments: unknown[]; - }>; - - assertContains(threat.enrichments, [ - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: '45.115.45.3', - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - provider: 'other_provider', - type: 'ip', - }, - - matched: { - atomic: '45.115.45.3', - id: '978787', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - }); - - it('adds a single indicator that matched multiple fields', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: 'NOT source.port:35326', // specify query to have signals more than treat indicators, but only 1 will match - threat_indicator_path: 'threat.indicator', - threat_query: 'threat.indicator.port: 57324 or threat.indicator.ip:45.115.45.3', // narrow our query to a single indicator - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.port', - field: 'source.port', - type: 'mapping', - }, - ], - }, - { - entries: [ - { - value: 'threat.indicator.ip', - field: 'source.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(1); - - const { hits } = signalsOpen.hits; - const [threat] = hits.map((hit) => hit._source?.threat) as Array<{ - enrichments: unknown[]; - }>; - - assertContains(threat.enrichments, [ - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: '45.115.45.3', - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - // We do not merge matched indicators during enrichment, so in - // certain circumstances a given indicator document could appear - // multiple times in an enriched alert (albeit with different - // threat.indicator.matched data). That's the case with the - // first and third indicators matched, here. - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - - matched: { - atomic: 57324, - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.port', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - provider: 'other_provider', - type: 'ip', - }, - matched: { - atomic: '45.115.45.3', - id: '978787', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - }); - - it('generates multiple signals with multiple matches', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - threat_language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', // narrow our query to a single record that matches two indicators - threat_indicator_path: 'threat.indicator', - threat_query: '*:*', - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.port', - field: 'source.port', - type: 'mapping', - }, - { - value: 'threat.indicator.ip', - field: 'source.ip', - type: 'mapping', - }, - ], - }, - { - entries: [ - { - value: 'threat.indicator.domain', - field: 'destination.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(2); - - const { hits } = signalsOpen.hits; - const threats = hits.map((hit) => hit._source?.threat) as Array<{ - enrichments: unknown[]; - }>; - - assertContains(threats[0].enrichments, [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - type: 'url', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: '45.115.45.3', - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: 57324, - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.port', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - - assertContains(threats[1].enrichments, [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - type: 'url', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - }); - }); - - describe('indicator enrichment: event-first search', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/filebeat/threat_intel'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/filebeat/threat_intel'); - }); - - it('enriches signals with the single indicator that matched', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: 'destination.ip:159.89.119.67', - threat_indicator_path: 'threat.indicator', - threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.domain', - field: 'destination.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(2); - - const { hits } = signalsOpen.hits; - const threats = hits.map((hit) => hit._source?.threat); - expect(threats).to.eql([ - { - enrichments: [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - type: 'url', - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ], - }, - { - enrichments: [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - type: 'url', - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ], - }, - ]); - }); - - it('enriches signals with multiple indicators if several matched', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: 'source.port: 57324', // narrow our query to a single record that matches two indicators - threat_indicator_path: 'threat.indicator', - threat_query: 'threat.indicator.ip: *', - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.ip', - field: 'source.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(1); - - const { hits } = signalsOpen.hits; - const [threat] = hits.map((hit) => hit._source?.threat) as Array<{ - enrichments: unknown[]; - }>; - - assertContains(threat.enrichments, [ - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: '45.115.45.3', - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - provider: 'other_provider', - type: 'ip', - }, - - matched: { - atomic: '45.115.45.3', - id: '978787', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - }); - - it('adds a single indicator that matched multiple fields', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: 'source.port: 57324', // narrow our query to a single record that matches two indicators - threat_indicator_path: 'threat.indicator', - threat_query: 'threat.indicator.ip: *', - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.port', - field: 'source.port', - type: 'mapping', - }, - ], - }, - { - entries: [ - { - value: 'threat.indicator.ip', - field: 'source.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(1); - - const { hits } = signalsOpen.hits; - const [threat] = hits.map((hit) => hit._source?.threat) as Array<{ - enrichments: unknown[]; - }>; - - assertContains(threat.enrichments, [ - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: '45.115.45.3', - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - // We do not merge matched indicators during enrichment, so in - // certain circumstances a given indicator document could appear - // multiple times in an enriched alert (albeit with different - // threat.indicator.matched data). That's the case with the - // first and third indicators matched, here. - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - - matched: { - atomic: 57324, - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.port', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - provider: 'other_provider', - type: 'ip', - }, - matched: { - atomic: '45.115.45.3', - id: '978787', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - }); - - it('generates multiple signals with multiple matches', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - threat_language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '(source.port:57324 and source.ip:45.115.45.3) or destination.ip:159.89.119.67', // narrow our query to a single record that matches two indicators - threat_indicator_path: 'threat.indicator', - threat_query: '*:*', - threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module - threat_mapping: [ - { - entries: [ - { - value: 'threat.indicator.port', - field: 'source.port', - type: 'mapping', - }, - { - value: 'threat.indicator.ip', - field: 'source.ip', - type: 'mapping', - }, - ], - }, - { - entries: [ - { - value: 'threat.indicator.domain', - field: 'destination.ip', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).equal(2); - - const { hits } = signalsOpen.hits; - const threats = hits.map((hit) => hit._source?.threat) as Array<{ - enrichments: unknown[]; - }>; - - assertContains(threats[0].enrichments, [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - type: 'url', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: '45.115.45.3', - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - { - feed: {}, - indicator: { - description: 'this should match auditbeat/hosts on both port and ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - port: 57324, - provider: 'geenensp', - type: 'url', - }, - matched: { - atomic: 57324, - id: '978785', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'source.port', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - - assertContains(threats[1].enrichments, [ - { - feed: {}, - indicator: { - description: "domain should match the auditbeat hosts' data's source.ip", - domain: '159.89.119.67', - first_seen: '2021-01-26T11:09:04.000Z', - provider: 'geenensp', - type: 'url', - url: { - full: 'http://159.89.119.67:59600/bin.sh', - scheme: 'http', - }, - }, - matched: { - atomic: '159.89.119.67', - id: '978783', - index: 'filebeat-8.0.0-2021.01.26-000001', - field: 'destination.ip', - type: ENRICHMENT_TYPES.IndicatorMatchRule, - }, - }, - ]); - }); - }); - - describe('alerts should be be enriched', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - it('should be enriched with host risk score', async () => { - const rule: RuleCreateProps = { - description: 'Detecting root and admin users', - name: 'Query with a rule id', - severity: 'high', - index: ['auditbeat-*'], - type: 'threat_match', - risk_score: 55, - language: 'kuery', - rule_id: 'rule-1', - from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip - threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity - threat_mapping: [ - // We match host.name against host.name - { - entries: [ - { - field: 'host.name', - value: 'host.name', - type: 'mapping', - }, - ], - }, - ], - threat_filters: [], - }; - - const createdRule = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, createdRule.id); - await waitForSignalsToBePresent(supertest, log, 10, [createdRule.id]); - const signalsOpen = await getSignalsByIds(supertest, log, [createdRule.id]); - expect(signalsOpen.hits.hits.length).equal(10); - const fullSource = signalsOpen.hits.hits.find( - (signal) => - (signal._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id === '7yJ-B2kBR346wHgnhlMn' - ); - const fullSignal = fullSource?._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - - expect(fullSignal?.host?.risk?.calculated_level).to.eql('Critical'); - expect(fullSignal?.host?.risk?.calculated_score_norm).to.eql(70); - }); - }); - }); - }); -}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts deleted file mode 100644 index 60e4cef77c896..0000000000000 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/generating_signals.ts +++ /dev/null @@ -1,1554 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { - ALERT_REASON, - ALERT_RISK_SCORE, - ALERT_RULE_NAME, - ALERT_RULE_PARAMETERS, - ALERT_RULE_RULE_ID, - ALERT_RULE_RULE_NAME_OVERRIDE, - ALERT_RULE_UUID, - ALERT_SEVERITY, - ALERT_WORKFLOW_STATUS, - EVENT_ACTION, - EVENT_KIND, -} from '@kbn/rule-data-utils'; -import { flattenWithPrefix } from '@kbn/securitysolution-rules'; - -import { orderBy, get } from 'lodash'; - -import { RuleExecutionStatus } from '@kbn/security-solution-plugin/common/detection_engine/rule_monitoring'; -import { - EqlRuleCreateProps, - QueryRuleCreateProps, - SavedQueryRuleCreateProps, - ThresholdRuleCreateProps, -} from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; -import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; -import { - ALERT_ANCESTORS, - ALERT_DEPTH, - ALERT_ORIGINAL_TIME, - ALERT_ORIGINAL_EVENT, - ALERT_ORIGINAL_EVENT_CATEGORY, - ALERT_GROUP_ID, - ALERT_THRESHOLD_RESULT, -} from '@kbn/security-solution-plugin/common/field_maps/field_names'; -import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; -import { - createRule, - createSignalsIndex, - deleteAllAlerts, - deleteSignalsIndex, - getEqlRuleForSignalTesting, - getOpenSignals, - getRuleForSignalTesting, - getSignalsByIds, - getSignalsByRuleIds, - getSimpleRule, - getThresholdRuleForSignalTesting, - waitForRuleSuccessOrStatus, - waitForSignalsToBePresent, -} from '../../utils'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; - -/** - * Specific _id to use for some of the tests. If the archiver changes and you see errors - * here, update this to a new value of a chosen auditbeat record and update the tests values. - */ -export const ID = 'BhbXBmkBR346wHgn4PeZ'; - -// eslint-disable-next-line import/no-default-export -export default ({ getService }: FtrProviderContext) => { - const supertest = getService('supertest'); - const esArchiver = getService('esArchiver'); - const es = getService('es'); - const log = getService('log'); - - describe('Generating signals from source indexes', () => { - beforeEach(async () => { - await deleteSignalsIndex(supertest, log); - await createSignalsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest, log); - await deleteAllAlerts(supertest, log); - }); - - describe('Signals from audit beat are of the expected structure', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - it('should have the specific audit record for _id or none of these tests below will pass', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).greaterThan(0); - }); - - it('should abide by max_signals > 100', async () => { - const maxSignals = 500; - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - max_signals: maxSignals, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, maxSignals, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id], maxSignals); - expect(signalsOpen.hits.hits.length).equal(maxSignals); - }); - - it('should have recorded the rule_id within the signal', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits[0]._source![ALERT_RULE_RULE_ID]).eql(getSimpleRule().rule_id); - }); - - it('should query and get back expected signal structure using a basic KQL query', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - const signal = signalsOpen.hits.hits[0]._source!; - - expect(signal).eql({ - ...signal, - [ALERT_ANCESTORS]: [ - { - id: 'BhbXBmkBR346wHgn4PeZ', - type: 'event', - index: 'auditbeat-8.0.0-2019.02.19-000001', - depth: 0, - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 1, - [ALERT_ORIGINAL_TIME]: '2019-02-19T17:40:03.790Z', - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'socket_closed', - dataset: 'socket', - kind: 'event', - module: 'system', - }), - }); - }); - - it('should query and get back expected signal structure using a saved query rule', async () => { - const rule: SavedQueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - type: 'saved_query', - query: `_id:${ID}`, - saved_id: 'doesnt-exist', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - const signal = signalsOpen.hits.hits[0]._source!; - expect(signal).eql({ - ...signal, - [ALERT_ANCESTORS]: [ - { - id: 'BhbXBmkBR346wHgn4PeZ', - type: 'event', - index: 'auditbeat-8.0.0-2019.02.19-000001', - depth: 0, - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 1, - [ALERT_ORIGINAL_TIME]: '2019-02-19T17:40:03.790Z', - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'socket_closed', - dataset: 'socket', - kind: 'event', - module: 'system', - }), - }); - }); - - it('should query and get back expected signal structure when it is a signal on a signal', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - const { id: createdId } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, createdId); - await waitForSignalsToBePresent(supertest, log, 1, [createdId]); - - // Run signals on top of that 1 signal which should create a single signal (on top of) a signal - const ruleForSignals: QueryRuleCreateProps = { - ...getRuleForSignalTesting([`.alerts-security.alerts-default*`]), - rule_id: 'signal-on-signal', - }; - - const { id } = await createRule(supertest, log, ruleForSignals); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - - // Get our single signal on top of a signal - const signalsOpen = await getSignalsByRuleIds(supertest, log, ['signal-on-signal']); - - const signal = signalsOpen.hits.hits[0]._source!; - expect(signal).eql({ - ...signal, - [ALERT_ANCESTORS]: [ - { - id: 'BhbXBmkBR346wHgn4PeZ', - type: 'event', - index: 'auditbeat-8.0.0-2019.02.19-000001', - depth: 0, - }, - { - ...(signal[ALERT_ANCESTORS] as Ancestor[])[1], - type: 'signal', - index: '.internal.alerts-security.alerts-default-000001', - depth: 1, - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 2, - [ALERT_ORIGINAL_TIME]: signal[ALERT_ORIGINAL_TIME], // original_time will always be changing sine it's based on a signal created here, so skip testing it - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'socket_closed', - dataset: 'socket', - kind: 'signal', - module: 'system', - }), - }); - }); - - describe('EQL Rules', () => { - before(async () => { - await esArchiver.load( - 'x-pack/test/functional/es_archives/security_solution/timestamp_override_6' - ); - }); - - after(async () => { - await esArchiver.unload( - 'x-pack/test/functional/es_archives/security_solution/timestamp_override_6' - ); - }); - - it('generates a correctly formatted signal from EQL non-sequence queries', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(1); - const fullSignal = signals.hits.hits[0]._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - - expect(fullSignal).eql({ - ...fullSignal, - agent: { - ephemeral_id: '0010d67a-14f7-41da-be30-489fea735967', - hostname: 'suricata-zeek-sensor-toronto', - id: 'a1d7b39c-f898-4dbe-a761-efb61939302d', - type: 'auditbeat', - version: '8.0.0', - }, - auditd: { - data: { - audit_enabled: '1', - old: '1', - }, - message_type: 'config_change', - result: 'success', - sequence: 1496, - session: 'unset', - summary: { - actor: { - primary: 'unset', - }, - object: { - primary: '1', - type: 'audit-config', - }, - }, - }, - cloud: { - instance: { - id: '133555295', - }, - provider: 'digitalocean', - region: 'tor1', - }, - ecs: { - version: '1.0.0-beta2', - }, - ...flattenWithPrefix('event', { - action: 'changed-audit-configuration', - category: 'configuration', - module: 'auditd', - kind: 'signal', - }), - host: { - architecture: 'x86_64', - containerized: false, - hostname: 'suricata-zeek-sensor-toronto', - id: '8cc95778cce5407c809480e8e32ad76b', - name: 'suricata-zeek-sensor-toronto', - os: { - codename: 'bionic', - family: 'debian', - kernel: '4.15.0-45-generic', - name: 'Ubuntu', - platform: 'ubuntu', - version: '18.04.2 LTS (Bionic Beaver)', - }, - }, - service: { - type: 'auditd', - }, - user: { - audit: { - id: 'unset', - }, - }, - [ALERT_REASON]: - 'configuration event on suricata-zeek-sensor-toronto created high alert Signal Testing Query.', - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 1, - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: '9xbRBmkBR346wHgngz2D', - index: 'auditbeat-8.0.0-2019.02.19-000001', - type: 'event', - }, - ], - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'changed-audit-configuration', - category: 'configuration', - module: 'auditd', - }), - }); - }); - - it('generates up to max_signals for non-sequence EQL queries', async () => { - const rule: EqlRuleCreateProps = getEqlRuleForSignalTesting(['auditbeat-*']); - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 100, [id]); - const signals = await getSignalsByIds(supertest, log, [id], 1000); - const filteredSignals = signals.hits.hits.filter( - (signal) => signal._source?.[ALERT_DEPTH] === 1 - ); - expect(filteredSignals.length).eql(100); - }); - - it('uses the provided event_category_override', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'config_change where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', - event_category_override: 'auditd.message_type', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(1); - const fullSignal = signals.hits.hits[0]._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - - expect(fullSignal).eql({ - ...fullSignal, - auditd: { - data: { - audit_enabled: '1', - old: '1', - }, - message_type: 'config_change', - result: 'success', - sequence: 1496, - session: 'unset', - summary: { - actor: { - primary: 'unset', - }, - object: { - primary: '1', - type: 'audit-config', - }, - }, - }, - ...flattenWithPrefix('event', { - action: 'changed-audit-configuration', - category: 'configuration', - module: 'auditd', - kind: 'signal', - }), - service: { - type: 'auditd', - }, - user: { - audit: { - id: 'unset', - }, - }, - [ALERT_REASON]: - 'configuration event on suricata-zeek-sensor-toronto created high alert Signal Testing Query.', - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 1, - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: '9xbRBmkBR346wHgngz2D', - index: 'auditbeat-8.0.0-2019.02.19-000001', - type: 'event', - }, - ], - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'changed-audit-configuration', - category: 'configuration', - module: 'auditd', - }), - }); - }); - - it('uses the provided timestamp_field', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['fake.index.1']), - query: 'any where true', - timestamp_field: 'created_at', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(3); - - const createdAtHits = signals.hits.hits.map((hit) => hit._source?.created_at); - expect(createdAtHits).to.eql([1622676785, 1622676790, 1622676795]); - }); - - it('uses the provided tiebreaker_field', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['fake.index.1']), - query: 'any where true', - tiebreaker_field: 'locale', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(3); - - const createdAtHits = signals.hits.hits.map((hit) => hit._source?.locale); - expect(createdAtHits).to.eql(['es', 'pt', 'ua']); - }); - - it('generates building block signals from EQL sequences in the expected form', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'sequence by host.name [anomoly where true] [any where true]', // TODO: spelling - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 3, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - const buildingBlock = signals.hits.hits.find( - (signal) => - signal._source?.[ALERT_DEPTH] === 1 && - get(signal._source, ALERT_ORIGINAL_EVENT_CATEGORY) === 'anomoly' - ); - expect(buildingBlock).not.eql(undefined); - const fullSignal = buildingBlock?._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - - expect(fullSignal).eql({ - ...fullSignal, - agent: { - ephemeral_id: '1b4978a0-48be-49b1-ac96-323425b389ab', - hostname: 'zeek-sensor-amsterdam', - id: 'e52588e6-7aa3-4c89-a2c4-d6bc5c286db1', - type: 'auditbeat', - version: '8.0.0', - }, - auditd: { - data: { - a0: '3', - a1: '107', - a2: '1', - a3: '7ffc186b58e0', - arch: 'x86_64', - auid: 'unset', - dev: 'eth0', - exit: '0', - gid: '0', - old_prom: '0', - prom: '256', - ses: 'unset', - syscall: 'setsockopt', - tty: '(none)', - uid: '0', - }, - message_type: 'anom_promiscuous', - result: 'success', - sequence: 1392, - session: 'unset', - summary: { - actor: { - primary: 'unset', - secondary: 'root', - }, - how: '/usr/bin/bro', - object: { - primary: 'eth0', - type: 'network-device', - }, - }, - }, - cloud: { instance: { id: '133551048' }, provider: 'digitalocean', region: 'ams3' }, - ecs: { version: '1.0.0-beta2' }, - ...flattenWithPrefix('event', { - action: 'changed-promiscuous-mode-on-device', - category: 'anomoly', - module: 'auditd', - kind: 'signal', - }), - host: { - architecture: 'x86_64', - containerized: false, - hostname: 'zeek-sensor-amsterdam', - id: '2ce8b1e7d69e4a1d9c6bcddc473da9d9', - name: 'zeek-sensor-amsterdam', - os: { - codename: 'bionic', - family: 'debian', - kernel: '4.15.0-45-generic', - name: 'Ubuntu', - platform: 'ubuntu', - version: '18.04.2 LTS (Bionic Beaver)', - }, - }, - process: { - executable: '/usr/bin/bro', - name: 'bro', - pid: 30157, - ppid: 30151, - title: - '/usr/bin/bro -i eth0 -U .status -p broctl -p broctl-live -p standalone -p local -p bro local.bro broctl broctl/standalone broctl', - }, - service: { type: 'auditd' }, - user: { - audit: { id: 'unset' }, - effective: { - group: { - id: '0', - name: 'root', - }, - id: '0', - name: 'root', - }, - filesystem: { - group: { - id: '0', - name: 'root', - }, - id: '0', - name: 'root', - }, - group: { id: '0', name: 'root' }, - id: '0', - name: 'root', - saved: { - group: { - id: '0', - name: 'root', - }, - id: '0', - name: 'root', - }, - }, - [ALERT_REASON]: - 'anomoly event with process bro, by root on zeek-sensor-amsterdam created high alert Signal Testing Query.', - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_GROUP_ID]: fullSignal[ALERT_GROUP_ID], - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 1, - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: 'VhXOBmkBR346wHgnLP8T', - index: 'auditbeat-8.0.0-2019.02.19-000001', - type: 'event', - }, - ], - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'changed-promiscuous-mode-on-device', - category: 'anomoly', - module: 'auditd', - }), - }); - }); - - it('generates shell signals from EQL sequences in the expected form', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'sequence by host.name [anomoly where true] [any where true]', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 3, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - const sequenceSignal = signalsOpen.hits.hits.find( - (signal) => signal._source?.[ALERT_DEPTH] === 2 - ); - const source = sequenceSignal?._source; - if (!source) { - return expect(source).to.be.ok(); - } - const eventIds = (source?.[ALERT_ANCESTORS] as Ancestor[]) - .filter((event) => event.depth === 1) - .map((event) => event.id); - expect(source).eql({ - ...source, - agent: { - ephemeral_id: '1b4978a0-48be-49b1-ac96-323425b389ab', - hostname: 'zeek-sensor-amsterdam', - id: 'e52588e6-7aa3-4c89-a2c4-d6bc5c286db1', - type: 'auditbeat', - version: '8.0.0', - }, - auditd: { session: 'unset', summary: { actor: { primary: 'unset' } } }, - cloud: { instance: { id: '133551048' }, provider: 'digitalocean', region: 'ams3' }, - ecs: { version: '1.0.0-beta2' }, - [EVENT_KIND]: 'signal', - host: { - architecture: 'x86_64', - containerized: false, - hostname: 'zeek-sensor-amsterdam', - id: '2ce8b1e7d69e4a1d9c6bcddc473da9d9', - name: 'zeek-sensor-amsterdam', - os: { - codename: 'bionic', - family: 'debian', - kernel: '4.15.0-45-generic', - name: 'Ubuntu', - platform: 'ubuntu', - version: '18.04.2 LTS (Bionic Beaver)', - }, - }, - service: { type: 'auditd' }, - user: { audit: { id: 'unset' }, id: '0', name: 'root' }, - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_DEPTH]: 2, - [ALERT_GROUP_ID]: source[ALERT_GROUP_ID], - [ALERT_REASON]: - 'event by root on zeek-sensor-amsterdam created high alert Signal Testing Query.', - [ALERT_RULE_UUID]: source[ALERT_RULE_UUID], - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: 'VhXOBmkBR346wHgnLP8T', - index: 'auditbeat-8.0.0-2019.02.19-000001', - type: 'event', - }, - { - depth: 1, - id: eventIds[0], - index: '', - rule: source[ALERT_RULE_UUID], - type: 'signal', - }, - { - depth: 0, - id: '4hbXBmkBR346wHgn6fdp', - index: 'auditbeat-8.0.0-2019.02.19-000001', - type: 'event', - }, - { - depth: 1, - id: eventIds[1], - index: '', - rule: source[ALERT_RULE_UUID], - type: 'signal', - }, - ], - }); - }); - - it('generates up to max_signals with an EQL rule', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'sequence by host.name [any where true] [any where true]', - max_signals: 200, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - // For EQL rules, max_signals is the maximum number of detected sequences: each sequence has a building block - // alert for each event in the sequence, so max_signals=200 results in 400 building blocks in addition to - // 200 regular alerts - await waitForSignalsToBePresent(supertest, log, 600, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id], 1000); - expect(signalsOpen.hits.hits.length).eql(600); - const shellSignals = signalsOpen.hits.hits.filter( - (signal) => signal._source?.[ALERT_DEPTH] === 2 - ); - const buildingBlocks = signalsOpen.hits.hits.filter( - (signal) => signal._source?.[ALERT_DEPTH] === 1 - ); - expect(shellSignals.length).eql(200); - expect(buildingBlocks.length).eql(400); - }); - - it('generates signals when an index name contains special characters to encode', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*', '']), - query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(1); - }); - - it('uses the provided filters', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'any where true', - filters: [ - { - meta: { - alias: null, - negate: false, - disabled: false, - type: 'phrase', - key: 'source.ip', - params: { - query: '46.148.18.163', - }, - }, - query: { - match_phrase: { - 'source.ip': '46.148.18.163', - }, - }, - }, - { - meta: { - alias: null, - negate: false, - disabled: false, - type: 'phrase', - key: 'event.action', - params: { - query: 'error', - }, - }, - query: { - match_phrase: { - 'event.action': 'error', - }, - }, - }, - ], - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(2); - }); - - describe('EQL alerts should be be enriched', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - it('should be enriched with host risk score', async () => { - const rule: EqlRuleCreateProps = { - ...getEqlRuleForSignalTesting(['auditbeat-*']), - query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signals = await getSignalsByIds(supertest, log, [id]); - expect(signals.hits.hits.length).eql(1); - const fullSignal = signals.hits.hits[0]._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - expect(fullSignal?.host?.risk?.calculated_level).to.eql('Critical'); - expect(fullSignal?.host?.risk?.calculated_score_norm).to.eql(96); - }); - }); - }); - - describe('Threshold Rules', () => { - it('generates 1 signal from Threshold rules when threshold is met', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: ['host.id'], - value: 700, - }, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).eql(1); - const fullSignal = signalsOpen.hits.hits[0]._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - const eventIds = (fullSignal?.[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); - expect(fullSignal).eql({ - ...fullSignal, - 'host.id': '8cc95778cce5407c809480e8e32ad76b', - [EVENT_KIND]: 'signal', - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: eventIds[0], - index: 'auditbeat-*', - type: 'event', - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_REASON]: 'event created high alert Signal Testing Query.', - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_DEPTH]: 1, - [ALERT_THRESHOLD_RESULT]: { - terms: [ - { - field: 'host.id', - value: '8cc95778cce5407c809480e8e32ad76b', - }, - ], - count: 788, - from: '2019-02-19T07:12:05.332Z', - }, - }); - }); - - it('generates 2 signals from Threshold rules when threshold is met', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: 'host.id', - value: 100, - }, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).eql(2); - }); - - it('applies the provided query before bucketing ', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - query: 'host.id:"2ab45fc1c41e4c84bbd02202a7e5761f"', - threshold: { - field: 'process.name', - value: 21, - }, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).eql(1); - }); - - it('generates no signals from Threshold rules when threshold is met and cardinality is not met', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: 'host.id', - value: 100, - cardinality: [ - { - field: 'destination.ip', - value: 100, - }, - ], - }, - }; - const createdRule = await createRule(supertest, log, rule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(0); - }); - - it('generates no signals from Threshold rules when cardinality is met and threshold is not met', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: 'host.id', - value: 1000, - cardinality: [ - { - field: 'destination.ip', - value: 5, - }, - ], - }, - }; - const createdRule = await createRule(supertest, log, rule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(0); - }); - - it('generates signals from Threshold rules when threshold and cardinality are both met', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: 'host.id', - value: 100, - cardinality: [ - { - field: 'destination.ip', - value: 5, - }, - ], - }, - }; - const createdRule = await createRule(supertest, log, rule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(1); - const fullSignal = signalsOpen.hits.hits[0]._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - const eventIds = (fullSignal?.[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); - expect(fullSignal).eql({ - ...fullSignal, - 'host.id': '8cc95778cce5407c809480e8e32ad76b', - [EVENT_KIND]: 'signal', - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: eventIds[0], - index: 'auditbeat-*', - type: 'event', - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_REASON]: `event created high alert Signal Testing Query.`, - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_DEPTH]: 1, - [ALERT_THRESHOLD_RESULT]: { - terms: [ - { - field: 'host.id', - value: '8cc95778cce5407c809480e8e32ad76b', - }, - ], - cardinality: [ - { - field: 'destination.ip', - value: 7, - }, - ], - count: 788, - from: '2019-02-19T07:12:05.332Z', - }, - }); - }); - - it('should not generate signals if only one field meets the threshold requirement', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: ['host.id', 'process.name'], - value: 22, - }, - }; - const createdRule = await createRule(supertest, log, rule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(0); - }); - - it('generates signals from Threshold rules when bucketing by multiple fields', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: ['host.id', 'process.name', 'event.module'], - value: 21, - }, - }; - const createdRule = await createRule(supertest, log, rule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).eql(1); - const fullSignal = signalsOpen.hits.hits[0]._source; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - const eventIds = (fullSignal[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); - expect(fullSignal).eql({ - ...fullSignal, - 'event.module': 'system', - 'host.id': '2ab45fc1c41e4c84bbd02202a7e5761f', - 'process.name': 'sshd', - [EVENT_KIND]: 'signal', - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: eventIds[0], - index: 'auditbeat-*', - type: 'event', - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_REASON]: `event with process sshd, created high alert Signal Testing Query.`, - [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], - [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], - [ALERT_DEPTH]: 1, - [ALERT_THRESHOLD_RESULT]: { - terms: [ - { - field: 'host.id', - value: '2ab45fc1c41e4c84bbd02202a7e5761f', - }, - { - field: 'process.name', - value: 'sshd', - }, - { - field: 'event.module', - value: 'system', - }, - ], - count: 21, - from: '2019-02-19T20:22:03.561Z', - }, - }); - }); - - describe('Timestamp override and fallback', async () => { - before(async () => { - await esArchiver.load( - 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' - ); - }); - - after(async () => { - await esArchiver.unload( - 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' - ); - }); - - it('applies timestamp override when using single field', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['timestamp-fallback-test']), - threshold: { - field: 'host.name', - value: 1, - }, - timestamp_override: 'event.ingested', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).eql(4); - - for (const hit of signalsOpen.hits.hits) { - const originalTime = hit._source?.[ALERT_ORIGINAL_TIME]; - const hostName = hit._source?.['host.name']; - if (hostName === 'host-1') { - expect(originalTime).eql('2020-12-16T15:15:18.570Z'); - } else if (hostName === 'host-2') { - expect(originalTime).eql('2020-12-16T15:16:18.570Z'); - } else if (hostName === 'host-3') { - expect(originalTime).eql('2020-12-16T16:15:18.570Z'); - } else { - expect(originalTime).eql('2020-12-16T16:16:18.570Z'); - } - } - }); - - it('applies timestamp override when using multiple fields', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['timestamp-fallback-test']), - threshold: { - field: ['host.name', 'source.ip'], - value: 1, - }, - timestamp_override: 'event.ingested', - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits.length).eql(4); - - for (const hit of signalsOpen.hits.hits) { - const originalTime = hit._source?.[ALERT_ORIGINAL_TIME]; - const hostName = hit._source?.['host.name']; - if (hostName === 'host-1') { - expect(originalTime).eql('2020-12-16T15:15:18.570Z'); - } else if (hostName === 'host-2') { - expect(originalTime).eql('2020-12-16T15:16:18.570Z'); - } else if (hostName === 'host-3') { - expect(originalTime).eql('2020-12-16T16:15:18.570Z'); - } else { - expect(originalTime).eql('2020-12-16T16:16:18.570Z'); - } - } - }); - }); - - describe('Threshold alerts should be be enriched', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - it('should be enriched with host risk score', async () => { - const rule: ThresholdRuleCreateProps = { - ...getThresholdRuleForSignalTesting(['auditbeat-*']), - threshold: { - field: 'host.name', - value: 100, - }, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 2, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - - expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); - expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_score_norm).to.eql(20); - expect(signalsOpen.hits.hits[1]?._source?.host?.risk?.calculated_level).to.eql( - 'Critical' - ); - expect(signalsOpen.hits.hits[1]?._source?.host?.risk?.calculated_score_norm).to.eql(96); - }); - }); - }); - - describe('Enrich alerts: query rule', () => { - describe('without index avalable', () => { - it('should do not have risk score fields', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits[0]?._source?.host?.risk).to.eql(undefined); - expect(signalsOpen.hits.hits[0]?._source?.user?.risk).to.eql(undefined); - }); - }); - - describe('with host risk score', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); - }); - - it('should host have risk score field and do not have user risk score', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID} or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi`, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - - const alerts = signalsOpen.hits.hits ?? []; - const firstAlert = alerts.find( - (alert) => alert?._source?.host?.name === 'suricata-zeek-sensor-toronto' - ); - const secondAlert = alerts.find( - (alert) => alert?._source?.host?.name === 'suricata-sensor-london' - ); - const thirdAlert = alerts.find((alert) => alert?._source?.host?.name === 'IE11WIN8_1'); - - expect(firstAlert?._source?.host?.risk?.calculated_level).to.eql('Critical'); - expect(firstAlert?._source?.host?.risk?.calculated_score_norm).to.eql(96); - expect(firstAlert?._source?.user?.risk).to.eql(undefined); - expect(secondAlert?._source?.host?.risk?.calculated_level).to.eql('Low'); - expect(secondAlert?._source?.host?.risk?.calculated_score_norm).to.eql(20); - expect(thirdAlert?._source?.host?.risk).to.eql(undefined); - }); - }); - - describe('with host and risk score and user risk score', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); - await esArchiver.load('x-pack/test/functional/es_archives/entity/user_risk'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); - await esArchiver.unload('x-pack/test/functional/es_archives/entity/user_risk'); - }); - - it('should have host and user risk score fields', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsOpen = await getSignalsByIds(supertest, log, [id]); - expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_level).to.eql( - 'Critical' - ); - expect(signalsOpen.hits.hits[0]?._source?.host?.risk?.calculated_score_norm).to.eql(96); - expect(signalsOpen.hits.hits[0]?._source?.user?.risk?.calculated_level).to.eql('Low'); - expect(signalsOpen.hits.hits[0]?._source?.user?.risk?.calculated_score_norm).to.eql(11); - }); - }); - }); - }); - - /** - * Here we test the functionality of Severity and Risk Score overrides (also called "mappings" - * in the code). If the rule specifies a mapping, then the final Severity or Risk Score - * value of the signal will be taken from the mapped field of the source event. - */ - describe('Signals generated from events with custom severity and risk score fields', () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/signals/severity_risk_overrides'); - }); - - after(async () => { - await esArchiver.unload( - 'x-pack/test/functional/es_archives/signals/severity_risk_overrides' - ); - }); - - const executeRuleAndGetSignals = async (rule: QueryRuleCreateProps) => { - const { id } = await createRule(supertest, log, rule); - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 4, [id]); - const signalsResponse = await getSignalsByIds(supertest, log, [id]); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - return signalsOrderedByEventId; - }; - - it('should get default severity and risk score if there is no mapping', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['signal_overrides']), - severity: 'medium', - risk_score: 75, - }; - - const signals = await executeRuleAndGetSignals(rule); - - expect(signals.length).equal(4); - signals.forEach((s) => { - expect(s?.[ALERT_SEVERITY]).equal('medium'); - expect(s?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([]); - - expect(s?.[ALERT_RISK_SCORE]).equal(75); - expect(s?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([]); - }); - }); - - it('should get overridden severity if the rule has a mapping for it', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['signal_overrides']), - severity: 'medium', - severity_mapping: [ - { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, - { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, - ], - risk_score: 75, - }; - - const signals = await executeRuleAndGetSignals(rule); - const severities = signals.map((s) => ({ - id: (s?.[ALERT_ANCESTORS] as Ancestor[])[0].id, - value: s?.[ALERT_SEVERITY], - })); - - expect(signals.length).equal(4); - expect(severities).eql([ - { id: '1', value: 'high' }, - { id: '2', value: 'critical' }, - { id: '3', value: 'critical' }, - { id: '4', value: 'critical' }, - ]); - - signals.forEach((s) => { - expect(s?.[ALERT_RISK_SCORE]).equal(75); - expect(s?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([]); - expect(s?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([ - { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, - { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, - ]); - }); - }); - - it('should get overridden risk score if the rule has a mapping for it', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['signal_overrides']), - severity: 'medium', - risk_score: 75, - risk_score_mapping: [ - { field: 'my_risk', operator: 'equals', value: '', risk_score: undefined }, - ], - }; - - const signals = await executeRuleAndGetSignals(rule); - const riskScores = signals.map((s) => ({ - id: (s?.[ALERT_ANCESTORS] as Ancestor[])[0].id, - value: s?.[ALERT_RISK_SCORE], - })); - - expect(signals.length).equal(4); - expect(riskScores).eql([ - { id: '1', value: 31.14 }, - { id: '2', value: 32.14 }, - { id: '3', value: 33.14 }, - { id: '4', value: 34.14 }, - ]); - - signals.forEach((s) => { - expect(s?.[ALERT_SEVERITY]).equal('medium'); - expect(s?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([]); - expect(s?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([ - { field: 'my_risk', operator: 'equals', value: '' }, - ]); - }); - }); - - it('should get overridden severity and risk score if the rule has both mappings', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['signal_overrides']), - severity: 'medium', - severity_mapping: [ - { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, - { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, - ], - risk_score: 75, - risk_score_mapping: [ - { field: 'my_risk', operator: 'equals', value: '', risk_score: undefined }, - ], - }; - - const signals = await executeRuleAndGetSignals(rule); - const values = signals.map((s) => ({ - id: (s?.[ALERT_ANCESTORS] as Ancestor[])[0].id, - severity: s?.[ALERT_SEVERITY], - risk: s?.[ALERT_RISK_SCORE], - })); - - expect(signals.length).equal(4); - expect(values).eql([ - { id: '1', severity: 'high', risk: 31.14 }, - { id: '2', severity: 'critical', risk: 32.14 }, - { id: '3', severity: 'critical', risk: 33.14 }, - { id: '4', severity: 'critical', risk: 34.14 }, - ]); - - signals.forEach((s) => { - expect(s?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([ - { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, - { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, - ]); - expect(s?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([ - { field: 'my_risk', operator: 'equals', value: '' }, - ]); - }); - }); - }); - - describe('Signals generated from events with name override field', async () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - beforeEach(async () => { - await deleteSignalsIndex(supertest, log); - await createSignalsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest, log); - await deleteAllAlerts(supertest, log); - }); - - it('should generate signals with name_override field', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - rule_name_override: 'event.action', - }; - - const { id } = await createRule(supertest, log, rule); - - await waitForRuleSuccessOrStatus(supertest, log, id); - await waitForSignalsToBePresent(supertest, log, 1, [id]); - const signalsResponse = await getSignalsByIds(supertest, log, [id], 1); - const signals = signalsResponse.hits.hits.map((hit) => hit._source); - const signalsOrderedByEventId = orderBy(signals, 'signal.parent.id', 'asc'); - const fullSignal = signalsOrderedByEventId[0]; - if (!fullSignal) { - return expect(fullSignal).to.be.ok(); - } - - expect(fullSignal).eql({ - ...fullSignal, - [EVENT_ACTION]: 'boot', - [ALERT_ANCESTORS]: [ - { - depth: 0, - id: 'UBXOBmkBR346wHgnLP8T', - index: 'auditbeat-8.0.0-2019.02.19-000001', - type: 'event', - }, - ], - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_REASON]: `event on zeek-sensor-amsterdam created high alert boot.`, - [ALERT_RULE_NAME]: 'boot', - [ALERT_RULE_RULE_NAME_OVERRIDE]: 'event.action', - [ALERT_DEPTH]: 1, - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'boot', - dataset: 'login', - kind: 'event', - module: 'system', - origin: '/var/log/wtmp', - }), - }); - }); - }); - - describe('Signal deduplication', async () => { - before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - after(async () => { - await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); - }); - - beforeEach(async () => { - await deleteSignalsIndex(supertest, log); - }); - - afterEach(async () => { - await deleteSignalsIndex(supertest, log); - await deleteAllAlerts(supertest, log); - }); - - it('should not generate duplicate signals', async () => { - const rule: QueryRuleCreateProps = { - ...getRuleForSignalTesting(['auditbeat-*']), - query: `_id:${ID}`, - }; - - const ruleResponse = await createRule(supertest, log, rule); - - const signals = await getOpenSignals(supertest, log, es, ruleResponse); - expect(signals.hits.hits.length).to.eql(1); - - const statusResponse = await supertest - .get(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .query({ id: ruleResponse.id }); - - // TODO: https://github.com/elastic/kibana/pull/121644 clean up, make type-safe - const ruleStatusDate = statusResponse.body?.execution_summary?.last_execution.date; - const initialStatusDate = new Date(ruleStatusDate); - - const initialSignal = signals.hits.hits[0]; - - // Disable the rule then re-enable to trigger another run - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ rule_id: ruleResponse.rule_id, enabled: false }) - .expect(200); - - await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .send({ rule_id: ruleResponse.rule_id, enabled: true }) - .expect(200); - - await waitForRuleSuccessOrStatus( - supertest, - log, - ruleResponse.id, - RuleExecutionStatus.succeeded, - initialStatusDate - ); - - const newSignals = await getOpenSignals(supertest, log, es, ruleResponse); - expect(newSignals.hits.hits.length).to.eql(1); - expect(newSignals.hits.hits[0]).to.eql(initialSignal); - }); - }); - }); -}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts index 3064d412da1bd..690498a287530 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts @@ -23,16 +23,13 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_rules')); loadTestFile(require.resolve('./preview_rules')); loadTestFile(require.resolve('./create_rules_bulk')); - loadTestFile(require.resolve('./create_ml')); loadTestFile(require.resolve('./create_new_terms')); - loadTestFile(require.resolve('./create_threat_matching')); loadTestFile(require.resolve('./create_rule_exceptions')); loadTestFile(require.resolve('./delete_rules')); loadTestFile(require.resolve('./delete_rules_bulk')); loadTestFile(require.resolve('./export_rules')); loadTestFile(require.resolve('./find_rules')); loadTestFile(require.resolve('./find_rule_exception_references')); - loadTestFile(require.resolve('./generating_signals')); loadTestFile(require.resolve('./get_prepackaged_rules_status')); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/preview_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/preview_rules.ts index 3e0dd166a0cd2..b38545e9c03c9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/preview_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/preview_rules.ts @@ -20,8 +20,8 @@ export default ({ getService }: FtrProviderContext) => { const supertestWithoutAuth = getService('supertestWithoutAuth'); const log = getService('log'); - describe('create_rules', () => { - describe('creating rules', () => { + describe('preview_rules', () => { + describe('previewing rules', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/README.md b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/README.md new file mode 100644 index 0000000000000..3a72c90e3ec54 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/README.md @@ -0,0 +1,11 @@ +### Security Rule Execution Logic Tests + +These tests use the rule preview API as a fast way to verify that rules generate alerts as expected with various parameter settings. This avoids the costly overhead of creating a real rule and waiting for it to be scheduled. The preview route also returns rule statuses directly in the API response instead of writing the statuses to saved objects, which saves significant time as well. + +For assurance that the real rule execution works, one test for each rule type still creates a real rule and waits for the execution through the alerting framework and resulting alerts. + +As a result, the tests here typically run ~10x faster than the tests they replaced that were creating actual rules and running them. We can therefore add more tests here and get better coverage of the rule execution logic (which is currently, as of 8.5, somewhat lacking). + +Since the rule execution logic is primarily focused around generating and executing Elasticsearch queries, we need significant testing around whether or not the queries are returning the expected results. This is not achievable with unit tests at the moment, since we need to mock Elasticsearch results. The tests here are the preferred way to ensure that rules are executing the correct logic to generate alerts from source data. + +Testing rules with exceptions is still slow, even with the preview API, because the exception list has to be created for real and then cleaned up after the test - exceptions live in saved objects, so creating exceptions for individual tests slows them down significantly (>1s per test vs ~200ms for a test without exceptions). This is an area for future improvement. diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts new file mode 100644 index 0000000000000..2430b8f2148d9 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +// eslint-disable-next-line import/no-default-export +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../config.base.ts')); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('.')], + }; +} diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/eql.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/eql.ts new file mode 100644 index 0000000000000..cffc0a311ef3f --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/eql.ts @@ -0,0 +1,606 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + ALERT_REASON, + ALERT_RULE_UUID, + ALERT_WORKFLOW_STATUS, + EVENT_KIND, +} from '@kbn/rule-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; + +import { get } from 'lodash'; + +import { EqlRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_TIME, + ALERT_ORIGINAL_EVENT, + ALERT_ORIGINAL_EVENT_CATEGORY, + ALERT_GROUP_ID, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + createRule, + deleteAllAlerts, + deleteSignalsIndex, + getEqlRuleForSignalTesting, + getOpenSignals, + getPreviewAlerts, + previewRule, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + describe('EQL type rules', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/timestamp_override_6' + ); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/timestamp_override_6' + ); + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + }); + + // First test creates a real rule - remaining tests use preview API + it('generates a correctly formatted signal from EQL non-sequence queries', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).eql(1); + const fullSignal = alerts.hits.hits[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + + expect(fullSignal).eql({ + ...fullSignal, + agent: { + ephemeral_id: '0010d67a-14f7-41da-be30-489fea735967', + hostname: 'suricata-zeek-sensor-toronto', + id: 'a1d7b39c-f898-4dbe-a761-efb61939302d', + type: 'auditbeat', + version: '8.0.0', + }, + auditd: { + data: { + audit_enabled: '1', + old: '1', + }, + message_type: 'config_change', + result: 'success', + sequence: 1496, + session: 'unset', + summary: { + actor: { + primary: 'unset', + }, + object: { + primary: '1', + type: 'audit-config', + }, + }, + }, + cloud: { + instance: { + id: '133555295', + }, + provider: 'digitalocean', + region: 'tor1', + }, + ecs: { + version: '1.0.0-beta2', + }, + ...flattenWithPrefix('event', { + action: 'changed-audit-configuration', + category: 'configuration', + module: 'auditd', + kind: 'signal', + }), + host: { + architecture: 'x86_64', + containerized: false, + hostname: 'suricata-zeek-sensor-toronto', + id: '8cc95778cce5407c809480e8e32ad76b', + name: 'suricata-zeek-sensor-toronto', + os: { + codename: 'bionic', + family: 'debian', + kernel: '4.15.0-45-generic', + name: 'Ubuntu', + platform: 'ubuntu', + version: '18.04.2 LTS (Bionic Beaver)', + }, + }, + service: { + type: 'auditd', + }, + user: { + audit: { + id: 'unset', + }, + }, + [ALERT_REASON]: + 'configuration event on suricata-zeek-sensor-toronto created high alert Signal Testing Query.', + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 1, + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: '9xbRBmkBR346wHgngz2D', + index: 'auditbeat-8.0.0-2019.02.19-000001', + type: 'event', + }, + ], + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'changed-audit-configuration', + category: 'configuration', + module: 'auditd', + }), + }); + }); + + it('generates up to max_signals for non-sequence EQL queries', async () => { + const maxSignals = 200; + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + max_signals: maxSignals, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxSignals * 2 }); + expect(previewAlerts.length).eql(maxSignals); + }); + + it('uses the provided event_category_override', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'config_change where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', + event_category_override: 'auditd.message_type', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); + const fullSignal = previewAlerts[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + + expect(fullSignal).eql({ + ...fullSignal, + auditd: { + data: { + audit_enabled: '1', + old: '1', + }, + message_type: 'config_change', + result: 'success', + sequence: 1496, + session: 'unset', + summary: { + actor: { + primary: 'unset', + }, + object: { + primary: '1', + type: 'audit-config', + }, + }, + }, + ...flattenWithPrefix('event', { + action: 'changed-audit-configuration', + category: 'configuration', + module: 'auditd', + kind: 'signal', + }), + service: { + type: 'auditd', + }, + user: { + audit: { + id: 'unset', + }, + }, + [ALERT_REASON]: + 'configuration event on suricata-zeek-sensor-toronto created high alert Signal Testing Query.', + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 1, + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: '9xbRBmkBR346wHgngz2D', + index: 'auditbeat-8.0.0-2019.02.19-000001', + type: 'event', + }, + ], + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'changed-audit-configuration', + category: 'configuration', + module: 'auditd', + }), + }); + }); + + it('uses the provided timestamp_field', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['fake.index.1']), + query: 'any where true', + timestamp_field: 'created_at', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(3); + + const createdAtHits = previewAlerts.map((hit) => hit._source?.created_at).sort(); + expect(createdAtHits).to.eql([1622676785, 1622676790, 1622676795]); + }); + + it('uses the provided tiebreaker_field', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['fake.index.1']), + query: 'any where true', + tiebreaker_field: 'locale', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(3); + + const createdAtHits = previewAlerts.map((hit) => hit._source?.locale); + expect(createdAtHits).to.eql(['es', 'pt', 'ua']); + }); + + it('generates building block signals from EQL sequences in the expected form', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'sequence by host.name [anomoly where true] [any where true]', // TODO: spelling + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const buildingBlock = previewAlerts.find( + (alert) => + alert._source?.[ALERT_DEPTH] === 1 && + get(alert._source, ALERT_ORIGINAL_EVENT_CATEGORY) === 'anomoly' + ); + expect(buildingBlock).not.eql(undefined); + const fullSignal = buildingBlock?._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + + expect(fullSignal).eql({ + ...fullSignal, + agent: { + ephemeral_id: '1b4978a0-48be-49b1-ac96-323425b389ab', + hostname: 'zeek-sensor-amsterdam', + id: 'e52588e6-7aa3-4c89-a2c4-d6bc5c286db1', + type: 'auditbeat', + version: '8.0.0', + }, + auditd: { + data: { + a0: '3', + a1: '107', + a2: '1', + a3: '7ffc186b58e0', + arch: 'x86_64', + auid: 'unset', + dev: 'eth0', + exit: '0', + gid: '0', + old_prom: '0', + prom: '256', + ses: 'unset', + syscall: 'setsockopt', + tty: '(none)', + uid: '0', + }, + message_type: 'anom_promiscuous', + result: 'success', + sequence: 1392, + session: 'unset', + summary: { + actor: { + primary: 'unset', + secondary: 'root', + }, + how: '/usr/bin/bro', + object: { + primary: 'eth0', + type: 'network-device', + }, + }, + }, + cloud: { instance: { id: '133551048' }, provider: 'digitalocean', region: 'ams3' }, + ecs: { version: '1.0.0-beta2' }, + ...flattenWithPrefix('event', { + action: 'changed-promiscuous-mode-on-device', + category: 'anomoly', + module: 'auditd', + kind: 'signal', + }), + host: { + architecture: 'x86_64', + containerized: false, + hostname: 'zeek-sensor-amsterdam', + id: '2ce8b1e7d69e4a1d9c6bcddc473da9d9', + name: 'zeek-sensor-amsterdam', + os: { + codename: 'bionic', + family: 'debian', + kernel: '4.15.0-45-generic', + name: 'Ubuntu', + platform: 'ubuntu', + version: '18.04.2 LTS (Bionic Beaver)', + }, + }, + process: { + executable: '/usr/bin/bro', + name: 'bro', + pid: 30157, + ppid: 30151, + title: + '/usr/bin/bro -i eth0 -U .status -p broctl -p broctl-live -p standalone -p local -p bro local.bro broctl broctl/standalone broctl', + }, + service: { type: 'auditd' }, + user: { + audit: { id: 'unset' }, + effective: { + group: { + id: '0', + name: 'root', + }, + id: '0', + name: 'root', + }, + filesystem: { + group: { + id: '0', + name: 'root', + }, + id: '0', + name: 'root', + }, + group: { id: '0', name: 'root' }, + id: '0', + name: 'root', + saved: { + group: { + id: '0', + name: 'root', + }, + id: '0', + name: 'root', + }, + }, + [ALERT_REASON]: + 'anomoly event with process bro, by root on zeek-sensor-amsterdam created high alert Signal Testing Query.', + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_GROUP_ID]: fullSignal[ALERT_GROUP_ID], + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 1, + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: 'VhXOBmkBR346wHgnLP8T', + index: 'auditbeat-8.0.0-2019.02.19-000001', + type: 'event', + }, + ], + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'changed-promiscuous-mode-on-device', + category: 'anomoly', + module: 'auditd', + }), + }); + }); + + it('generates shell signals from EQL sequences in the expected form', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'sequence by host.name [anomoly where true] [any where true]', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const sequenceAlert = previewAlerts.find((alert) => alert._source?.[ALERT_DEPTH] === 2); + const source = sequenceAlert?._source; + if (!source) { + return expect(source).to.be.ok(); + } + const eventIds = (source?.[ALERT_ANCESTORS] as Ancestor[]) + .filter((event) => event.depth === 1) + .map((event) => event.id); + expect(source).eql({ + ...source, + agent: { + ephemeral_id: '1b4978a0-48be-49b1-ac96-323425b389ab', + hostname: 'zeek-sensor-amsterdam', + id: 'e52588e6-7aa3-4c89-a2c4-d6bc5c286db1', + type: 'auditbeat', + version: '8.0.0', + }, + auditd: { session: 'unset', summary: { actor: { primary: 'unset' } } }, + cloud: { instance: { id: '133551048' }, provider: 'digitalocean', region: 'ams3' }, + ecs: { version: '1.0.0-beta2' }, + [EVENT_KIND]: 'signal', + host: { + architecture: 'x86_64', + containerized: false, + hostname: 'zeek-sensor-amsterdam', + id: '2ce8b1e7d69e4a1d9c6bcddc473da9d9', + name: 'zeek-sensor-amsterdam', + os: { + codename: 'bionic', + family: 'debian', + kernel: '4.15.0-45-generic', + name: 'Ubuntu', + platform: 'ubuntu', + version: '18.04.2 LTS (Bionic Beaver)', + }, + }, + service: { type: 'auditd' }, + user: { audit: { id: 'unset' }, id: '0', name: 'root' }, + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 2, + [ALERT_GROUP_ID]: source[ALERT_GROUP_ID], + [ALERT_REASON]: + 'event by root on zeek-sensor-amsterdam created high alert Signal Testing Query.', + [ALERT_RULE_UUID]: source[ALERT_RULE_UUID], + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: 'VhXOBmkBR346wHgnLP8T', + index: 'auditbeat-8.0.0-2019.02.19-000001', + type: 'event', + }, + { + depth: 1, + id: eventIds[0], + index: '', + rule: source[ALERT_RULE_UUID], + type: 'signal', + }, + { + depth: 0, + id: '4hbXBmkBR346wHgn6fdp', + index: 'auditbeat-8.0.0-2019.02.19-000001', + type: 'event', + }, + { + depth: 1, + id: eventIds[1], + index: '', + rule: source[ALERT_RULE_UUID], + type: 'signal', + }, + ], + }); + }); + + it('generates up to max_signals with an EQL rule', async () => { + const maxSignals = 200; + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'sequence by host.name [any where true] [any where true]', + max_signals: maxSignals, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxSignals * 5 }); + // For EQL rules, max_signals is the maximum number of detected sequences: each sequence has a building block + // alert for each event in the sequence, so max_signals=200 results in 400 building blocks in addition to + // 200 regular alerts + expect(previewAlerts.length).eql(maxSignals * 3); + const shellSignals = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 2); + const buildingBlocks = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 1); + expect(shellSignals.length).eql(maxSignals); + expect(buildingBlocks.length).eql(maxSignals * 2); + }); + + it('generates signals when an index name contains special characters to encode', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*', '']), + query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); + }); + + it('uses the provided filters', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'any where true', + filters: [ + { + meta: { + alias: null, + negate: false, + disabled: false, + type: 'phrase', + key: 'source.ip', + params: { + query: '46.148.18.163', + }, + }, + query: { + match_phrase: { + 'source.ip': '46.148.18.163', + }, + }, + }, + { + meta: { + alias: null, + negate: false, + disabled: false, + type: 'phrase', + key: 'event.action', + params: { + query: 'error', + }, + }, + query: { + match_phrase: { + 'event.action': 'error', + }, + }, + }, + ], + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(2); + }); + + describe('with host risk index', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: EqlRuleCreateProps = { + ...getEqlRuleForSignalTesting(['auditbeat-*']), + query: 'configuration where agent.id=="a1d7b39c-f898-4dbe-a761-efb61939302d"', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); + const fullSignal = previewAlerts[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + expect(fullSignal?.host?.risk?.calculated_level).to.eql('Critical'); + expect(fullSignal?.host?.risk?.calculated_score_norm).to.eql(96); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/index.ts new file mode 100644 index 0000000000000..547e3a4706e3c --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default ({ loadTestFile }: FtrProviderContext): void => { + describe('detection engine api security and spaces enabled - rule execution logic', function () { + loadTestFile(require.resolve('./eql')); + loadTestFile(require.resolve('./machine_learning')); + loadTestFile(require.resolve('./new_terms')); + loadTestFile(require.resolve('./query')); + loadTestFile(require.resolve('./saved_query')); + loadTestFile(require.resolve('./threat_match')); + loadTestFile(require.resolve('./threshold')); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/machine_learning.ts similarity index 69% rename from x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts rename to x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/machine_learning.ts index e1a3d4f0796c4..0949d8255bed2 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_ml.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/machine_learning.ts @@ -33,9 +33,14 @@ import { import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createRule, - createRuleWithExceptionEntries, deleteAllAlerts, + deleteSignalsIndex, + executeSetupModuleRequest, + forceStartDatafeeds, getOpenSignals, + getPreviewAlerts, + previewRule, + previewRuleWithExceptionEntries, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -47,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => { const siemModule = 'security_linux_v3'; const mlJobId = 'v3_linux_anomalous_network_activity'; - const testRule: MachineLearningRuleCreateProps = { + const rule: MachineLearningRuleCreateProps = { name: 'Test ML rule', description: 'Test ML rule description', risk_score: 50, @@ -56,63 +61,32 @@ export default ({ getService }: FtrProviderContext) => { anomaly_threshold: 30, machine_learning_job_id: mlJobId, from: '1900-01-01T00:00:00.000Z', + rule_id: 'ml-rule-id', }; - async function executeSetupModuleRequest(module: string, rspCode: number) { - const { body } = await supertest - .post(`/api/ml/modules/setup/${module}`) - .set('kbn-xsrf', 'true') - .send({ - prefix: '', - groups: ['auditbeat'], - indexPatternName: 'auditbeat-*', - startDatafeed: false, - useDedicatedIndex: true, - applyToAllSpaces: true, - }) - .expect(rspCode); - - return body; - } - - async function forceStartDatafeeds(jobId: string, rspCode: number) { - const { body } = await supertest - .post(`/api/ml/jobs/force_start_datafeeds`) - .set('kbn-xsrf', 'true') - .send({ - datafeedIds: [`datafeed-${jobId}`], - start: new Date().getUTCMilliseconds(), - }) - .expect(rspCode); - - return body; - } - - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125033 // FLAKY: https://github.com/elastic/kibana/issues/142993 - describe.skip('Generating signals from ml anomalies', () => { + describe.skip('Machine learning type rules', () => { before(async () => { // Order is critical here: auditbeat data must be loaded before attempting to start the ML job, // as the job looks for certain indices on start await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); - await executeSetupModuleRequest(siemModule, 200); - await forceStartDatafeeds(mlJobId, 200); + await executeSetupModuleRequest({ module: siemModule, rspCode: 200, supertest }); + await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest }); await esArchiver.load('x-pack/test/functional/es_archives/security_solution/anomalies'); }); after(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/anomalies'); - }); - - afterEach(async () => { + await deleteSignalsIndex(supertest, log); await deleteAllAlerts(supertest, log); }); + // First test creates a real rule - remaining tests use preview API it('should create 1 alert from ML rule when record meets anomaly_threshold', async () => { - const createdRule = await createRule(supertest, log, testRule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).toBe(1); - const signal = signalsOpen.hits.hits[0]; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).toBe(1); + const signal = alerts.hits.hits[0]; expect(signal._source).toEqual( expect.objectContaining({ @@ -162,7 +136,7 @@ export default ({ getService }: FtrProviderContext) => { required_fields: [], risk_score: 50, risk_score_mapping: [], - rule_id: createdRule.rule_id, + rule_id: 'ml-rule-id', setup: '', severity: 'critical', severity_mapping: [], @@ -185,13 +159,12 @@ export default ({ getService }: FtrProviderContext) => { }); it('should create 7 alerts from ML rule when records meet anomaly_threshold', async () => { - const rule: MachineLearningRuleCreateProps = { - ...testRule, - anomaly_threshold: 20, - }; - const createdRule = await createRule(supertest, log, rule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).toBe(7); + const { previewId } = await previewRule({ + supertest, + rule: { ...rule, anomaly_threshold: 20 }, + }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).toBe(7); }); describe('with non-value list exception', () => { @@ -199,18 +172,23 @@ export default ({ getService }: FtrProviderContext) => { await deleteAllExceptions(supertest, log); }); it('generates no signals when an exception is added for an ML rule', async () => { - const createdRule = await createRuleWithExceptionEntries(supertest, log, testRule, [ - [ - { - field: 'host.name', - operator: 'included', - type: 'match', - value: 'mothra', - }, + const { previewId } = await previewRuleWithExceptionEntries({ + supertest, + log, + rule, + entries: [ + [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'mothra', + }, + ], ], - ]); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).toBe(0); + }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).toBe(0); }); }); @@ -227,21 +205,26 @@ export default ({ getService }: FtrProviderContext) => { it('generates no signals when a value list exception is added for an ML rule', async () => { const valueListId = 'value-list-id'; await importFile(supertest, log, 'keyword', ['mothra'], valueListId); - const createdRule = await createRuleWithExceptionEntries(supertest, log, testRule, [ - [ - { - field: 'host.name', - operator: 'included', - type: 'list', - list: { - id: valueListId, - type: 'keyword', + const { previewId } = await previewRuleWithExceptionEntries({ + supertest, + log, + rule, + entries: [ + [ + { + field: 'host.name', + operator: 'included', + type: 'list', + list: { + id: valueListId, + type: 'keyword', + }, }, - }, + ], ], - ]); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).toBe(0); + }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).toBe(0); }); }); @@ -255,10 +238,10 @@ export default ({ getService }: FtrProviderContext) => { }); it('should be enriched with host risk score', async () => { - const createdRule = await createRule(supertest, log, testRule); - const signalsOpen = await getOpenSignals(supertest, log, es, createdRule); - expect(signalsOpen.hits.hits.length).toBe(1); - const fullSignal = signalsOpen.hits.hits[0]._source; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).toBe(1); + const fullSignal = previewAlerts[0]._source; expect(fullSignal?.host?.risk?.calculated_level).toBe('Low'); expect(fullSignal?.host?.risk?.calculated_score_norm).toBe(1); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts new file mode 100644 index 0000000000000..4bfbe92118599 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -0,0 +1,385 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { orderBy } from 'lodash'; +import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks'; +import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; +import { + createRule, + deleteAllAlerts, + deleteSignalsIndex, + getOpenSignals, + getPreviewAlerts, + previewRule, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; +import { deleteAllExceptions } from '../../../lists_api_integration/utils'; + +const removeRandomValuedProperties = (alert: DetectionAlert | undefined) => { + if (!alert) { + return undefined; + } + const { + 'kibana.version': version, + 'kibana.alert.rule.execution.uuid': execUuid, + 'kibana.alert.rule.uuid': uuid, + '@timestamp': timestamp, + 'kibana.alert.rule.created_at': createdAt, + 'kibana.alert.rule.updated_at': updatedAt, + 'kibana.alert.uuid': alertUuid, + ...restOfAlert + } = alert; + return restOfAlert; +}; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + describe('New terms type rules', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + }); + + // First test creates a real rule - remaining tests use preview API + + // This test also tests that alerts are NOT created for terms that are not new: the host name + // suricata-sensor-san-francisco appears in a document at 2019-02-19T20:42:08.230Z, but also appears + // in earlier documents so is not new. An alert should not be generated for that term. + it('should generate 1 alert with 1 selected field', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + + expect(alerts.hits.hits.length).eql(1); + expect(removeRandomValuedProperties(alerts.hits.hits[0]._source)).eql({ + 'kibana.alert.new_terms': ['zeek-newyork-sha-aa8df15'], + 'kibana.alert.rule.category': 'New Terms Rule', + 'kibana.alert.rule.consumer': 'siem', + 'kibana.alert.rule.name': 'Query with a rule id', + 'kibana.alert.rule.producer': 'siem', + 'kibana.alert.rule.rule_type_id': 'siem.newTermsRule', + 'kibana.space_ids': ['default'], + 'kibana.alert.rule.tags': [], + agent: { + ephemeral_id: '7cc2091a-72f1-4c63-843b-fdeb622f9c69', + hostname: 'zeek-newyork-sha-aa8df15', + id: '4b4462ef-93d2-409c-87a6-299d942e5047', + type: 'auditbeat', + version: '8.0.0', + }, + cloud: { instance: { id: '139865230' }, provider: 'digitalocean', region: 'nyc1' }, + ecs: { version: '1.0.0-beta2' }, + host: { + architecture: 'x86_64', + hostname: 'zeek-newyork-sha-aa8df15', + id: '3729d06ce9964aa98549f41cbd99334d', + ip: ['157.230.208.30', '10.10.0.6', 'fe80::24ce:f7ff:fede:a571'], + mac: ['26:ce:f7:de:a5:71'], + name: 'zeek-newyork-sha-aa8df15', + os: { + codename: 'cosmic', + family: 'debian', + kernel: '4.18.0-10-generic', + name: 'Ubuntu', + platform: 'ubuntu', + version: '18.10 (Cosmic Cuttlefish)', + }, + }, + message: + 'Login by user root (UID: 0) on pts/0 (PID: 20638) from 8.42.77.171 (IP: 8.42.77.171)', + process: { pid: 20638 }, + service: { type: 'system' }, + source: { ip: '8.42.77.171' }, + user: { id: 0, name: 'root', terminal: 'pts/0' }, + 'event.action': 'user_login', + 'event.category': 'authentication', + 'event.dataset': 'login', + 'event.kind': 'signal', + 'event.module': 'system', + 'event.origin': '/var/log/wtmp', + 'event.outcome': 'success', + 'event.type': 'authentication_success', + 'kibana.alert.original_time': '2019-02-19T20:42:08.230Z', + 'kibana.alert.ancestors': [ + { + id: 'x07wJ2oB9v5HJNSHhyxi', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + 'kibana.alert.status': 'active', + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.depth': 1, + 'kibana.alert.reason': + 'authentication event with source 8.42.77.171 by root on zeek-newyork-sha-aa8df15 created high alert Query with a rule id.', + 'kibana.alert.severity': 'high', + 'kibana.alert.risk_score': 55, + 'kibana.alert.rule.parameters': { + description: 'Detecting root and admin users', + risk_score: 55, + severity: 'high', + author: [], + false_positives: [], + from: '2019-02-19T20:42:00.000Z', + rule_id: 'rule-1', + max_signals: 100, + risk_score_mapping: [], + severity_mapping: [], + threat: [], + to: 'now', + references: [], + version: 1, + exceptions_list: [], + immutable: false, + related_integrations: [], + required_fields: [], + setup: '', + type: 'new_terms', + query: '*', + new_terms_fields: ['host.name'], + history_window_start: '2019-01-19T20:42:00.000Z', + index: ['auditbeat-*'], + language: 'kuery', + }, + 'kibana.alert.rule.actions': [], + 'kibana.alert.rule.author': [], + 'kibana.alert.rule.created_by': 'elastic', + 'kibana.alert.rule.description': 'Detecting root and admin users', + 'kibana.alert.rule.enabled': true, + 'kibana.alert.rule.exceptions_list': [], + 'kibana.alert.rule.false_positives': [], + 'kibana.alert.rule.from': '2019-02-19T20:42:00.000Z', + 'kibana.alert.rule.immutable': false, + 'kibana.alert.rule.indices': ['auditbeat-*'], + 'kibana.alert.rule.interval': '5m', + 'kibana.alert.rule.max_signals': 100, + 'kibana.alert.rule.references': [], + 'kibana.alert.rule.risk_score_mapping': [], + 'kibana.alert.rule.rule_id': 'rule-1', + 'kibana.alert.rule.severity_mapping': [], + 'kibana.alert.rule.threat': [], + 'kibana.alert.rule.to': 'now', + 'kibana.alert.rule.type': 'new_terms', + 'kibana.alert.rule.updated_by': 'elastic', + 'kibana.alert.rule.version': 1, + 'kibana.alert.rule.risk_score': 55, + 'kibana.alert.rule.severity': 'high', + 'kibana.alert.original_event.action': 'user_login', + 'kibana.alert.original_event.category': 'authentication', + 'kibana.alert.original_event.dataset': 'login', + 'kibana.alert.original_event.kind': 'event', + 'kibana.alert.original_event.module': 'system', + 'kibana.alert.original_event.origin': '/var/log/wtmp', + 'kibana.alert.original_event.outcome': 'success', + 'kibana.alert.original_event.type': 'authentication_success', + }); + }); + + it('should generate 3 alerts when 1 document has 3 new values', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.ip'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(3); + const previewAlertsOrderedByHostIp = orderBy( + previewAlerts, + '_source.kibana.alert.new_terms', + 'asc' + ); + expect(previewAlertsOrderedByHostIp[0]._source?.['kibana.alert.new_terms']).eql([ + '10.10.0.6', + ]); + expect(previewAlertsOrderedByHostIp[1]._source?.['kibana.alert.new_terms']).eql([ + '157.230.208.30', + ]); + expect(previewAlertsOrderedByHostIp[2]._source?.['kibana.alert.new_terms']).eql([ + 'fe80::24ce:f7ff:fede:a571', + ]); + }); + + it('should generate alerts for every term when history window is small', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name'], + from: '2019-02-19T20:42:00.000Z', + // Set the history_window_start close to 'from' so we should alert on all terms in the time range + history_window_start: '2019-02-19T20:41:59.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(5); + const hostNames = previewAlerts + .map((signal) => signal._source?.['kibana.alert.new_terms']) + .sort(); + expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); + expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); + expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); + expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); + expect(hostNames[4]).eql(['zeek-sensor-san-francisco']); + }); + + describe('timestamp override and fallback', () => { + before(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' + ); + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/timestamp_override_3' + ); + }); + after(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' + ); + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/timestamp_override_3' + ); + }); + + it('should generate the correct alerts', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + // myfakeindex-3 does not have event.ingested mapped so we can test if the runtime field + // 'kibana.combined_timestamp' handles unmapped fields properly + index: ['timestamp-fallback-test', 'myfakeindex-3'], + new_terms_fields: ['host.name'], + from: '2020-12-16T16:00:00.000Z', + // Set the history_window_start close to 'from' so we should alert on all terms in the time range + history_window_start: '2020-12-16T15:59:00.000Z', + timestamp_override: 'event.ingested', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(2); + const hostNames = previewAlerts + .map((signal) => signal._source?.['kibana.alert.new_terms']) + .sort(); + expect(hostNames[0]).eql(['host-3']); + expect(hostNames[1]).eql(['host-4']); + }); + }); + + describe('with exceptions', async () => { + afterEach(async () => { + await deleteAllExceptions(supertest, log); + }); + + it('should apply exceptions', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name'], + from: '2019-02-19T20:42:00.000Z', + // Set the history_window_start close to 'from' so we should alert on all terms in the time range + history_window_start: '2019-02-19T20:41:59.000Z', + }; + + const { previewId } = await previewRuleWithExceptionEntries({ + supertest, + log, + rule, + entries: [ + [ + { + field: 'host.name', + operator: 'included', + type: 'match', + value: 'zeek-sensor-san-francisco', + }, + ], + ], + }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).eql(4); + const hostNames = previewAlerts + .map((signal) => signal._source?.['kibana.alert.new_terms']) + .sort(); + expect(hostNames[0]).eql(['suricata-sensor-amsterdam']); + expect(hostNames[1]).eql(['suricata-sensor-san-francisco']); + expect(hostNames[2]).eql(['zeek-newyork-sha-aa8df15']); + expect(hostNames[3]).eql(['zeek-sensor-amsterdam']); + }); + }); + + it('should work for max signals > 100', async () => { + const maxSignals = 200; + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['process.pid'], + from: '2018-02-19T20:42:00.000Z', + // Set the history_window_start close to 'from' so we should alert on all terms in the time range + history_window_start: '2018-02-19T20:41:59.000Z', + max_signals: maxSignals, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxSignals * 2 }); + + expect(previewAlerts.length).eql(maxSignals); + const processPids = previewAlerts + .map((signal) => signal._source?.['kibana.alert.new_terms']) + .sort(); + expect(processPids[0]).eql([1]); + }); + + describe('alerts should be be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + new_terms_fields: ['host.name'], + from: '2019-02-19T20:42:00.000Z', + history_window_start: '2019-01-19T20:42:00.000Z', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts new file mode 100644 index 0000000000000..8090e4d2ce709 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -0,0 +1,426 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + ALERT_RISK_SCORE, + ALERT_RULE_PARAMETERS, + ALERT_RULE_RULE_ID, + ALERT_SEVERITY, + ALERT_WORKFLOW_STATUS, +} from '@kbn/rule-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; + +import { orderBy } from 'lodash'; + +import { QueryRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_TIME, + ALERT_ORIGINAL_EVENT, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + createRule, + deleteAllAlerts, + deleteSignalsIndex, + getOpenSignals, + getPreviewAlerts, + getRuleForSignalTesting, + getSimpleRule, + previewRule, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +/** + * Specific _id to use for some of the tests. If the archiver changes and you see errors + * here, update this to a new value of a chosen auditbeat record and update the tests values. + */ +const ID = 'BhbXBmkBR346wHgn4PeZ'; + +/** + * Test coverage: + * [x] - Happy path generating 1 alert + * [x] - Rule type respects max signals + * [x] - Alerts on alerts + */ + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + describe('Query type rules', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/alerts/8.1.0'); + await esArchiver.load('x-pack/test/functional/es_archives/signals/severity_risk_overrides'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/alerts/8.1.0'); + await esArchiver.unload('x-pack/test/functional/es_archives/signals/severity_risk_overrides'); + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + }); + + // First test creates a real rule - remaining tests use preview API + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).greaterThan(0); + expect(alerts.hits.hits[0]._source?.['kibana.alert.ancestors'][0].id).eql(ID); + }); + + it('should abide by max_signals > 100', async () => { + const maxSignals = 200; + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + max_signals: maxSignals, + }; + const { previewId } = await previewRule({ supertest, rule }); + // Search for 2x max_signals to make sure we aren't making more than max_signals + const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxSignals * 2 }); + expect(previewAlerts.length).equal(maxSignals); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts[0]._source?.[ALERT_RULE_RULE_ID]).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const signal = previewAlerts[0]._source; + + expect(signal).eql({ + ...signal, + [ALERT_ANCESTORS]: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 1, + [ALERT_ORIGINAL_TIME]: '2019-02-19T17:40:03.790Z', + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'socket_closed', + dataset: 'socket', + kind: 'event', + module: 'system', + }), + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + const alertId = '30a75fe46d3dbdfab55982036f77a8d60e2d1112e96f277c3b8c22f9bb57817a'; + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting([`.alerts-security.alerts-default*`]), + rule_id: 'signal-on-signal', + query: `_id:${alertId}`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).to.eql(1); + + const signal = previewAlerts[0]._source; + + if (!signal) { + return expect(signal).to.be.ok(); + } + + expect(signal).eql({ + ...signal, + [ALERT_ANCESTORS]: [ + { + id: 'ahEToH8BK09aFtXZFVMq', + type: 'event', + index: 'events-index-000001', + depth: 0, + }, + { + rule: '031d5c00-a72f-11ec-a8a3-7b1c8077fc3e', + id: '30a75fe46d3dbdfab55982036f77a8d60e2d1112e96f277c3b8c22f9bb57817a', + type: 'signal', + index: '.internal.alerts-security.alerts-default-000001', + depth: 1, + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 2, + [ALERT_ORIGINAL_TIME]: '2022-03-19T02:48:12.634Z', + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + agent_id_status: 'verified', + ingested: '2022-03-19T02:47:57.376Z', + dataset: 'elastic_agent.filebeat', + }), + }); + }); + + it('should not have risk score fields without risk indices', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts[0]?._source?.host?.risk).to.eql(undefined); + expect(previewAlerts[0]?._source?.user?.risk).to.eql(undefined); + }); + + describe('with host risk index', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should host have risk score field and do not have user risk score', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID} or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi`, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + const firstAlert = previewAlerts.find( + (alert) => alert?._source?.host?.name === 'suricata-zeek-sensor-toronto' + ); + const secondAlert = previewAlerts.find( + (alert) => alert?._source?.host?.name === 'suricata-sensor-london' + ); + const thirdAlert = previewAlerts.find( + (alert) => alert?._source?.host?.name === 'IE11WIN8_1' + ); + + expect(firstAlert?._source?.host?.risk?.calculated_level).to.eql('Critical'); + expect(firstAlert?._source?.host?.risk?.calculated_score_norm).to.eql(96); + expect(firstAlert?._source?.user?.risk).to.eql(undefined); + expect(secondAlert?._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(secondAlert?._source?.host?.risk?.calculated_score_norm).to.eql(20); + expect(thirdAlert?._source?.host?.risk).to.eql(undefined); + }); + }); + + describe('with host and user risk indices', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + await esArchiver.load('x-pack/test/functional/es_archives/entity/user_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + await esArchiver.unload('x-pack/test/functional/es_archives/entity/user_risk'); + }); + + it('should have host and user risk score fields', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts[0]?._source?.host?.risk?.calculated_level).to.eql('Critical'); + expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(96); + expect(previewAlerts[0]?._source?.user?.risk?.calculated_level).to.eql('Low'); + expect(previewAlerts[0]?._source?.user?.risk?.calculated_score_norm).to.eql(11); + }); + }); + + /** + * Here we test the functionality of Severity and Risk Score overrides (also called "mappings" + * in the code). If the rule specifies a mapping, then the final Severity or Risk Score + * value of the signal will be taken from the mapped field of the source event. + */ + it('should get default severity and risk score if there is no mapping', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + risk_score: 75, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).equal(4); + previewAlerts.forEach((alert) => { + expect(alert._source?.[ALERT_SEVERITY]).equal('medium'); + expect(alert._source?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([]); + + expect(alert._source?.[ALERT_RISK_SCORE]).equal(75); + expect(alert._source?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([]); + }); + }); + + it('should get overridden severity if the rule has a mapping for it', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + severity_mapping: [ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ], + risk_score: 75, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const alertsOrderedByParentId = orderBy(previewAlerts, 'signal.parent.id', 'asc'); + const severities = alertsOrderedByParentId.map((alert) => ({ + id: (alert._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id, + value: alert._source?.[ALERT_SEVERITY], + })); + + expect(alertsOrderedByParentId.length).equal(4); + expect(severities).eql([ + { id: '1', value: 'high' }, + { id: '2', value: 'critical' }, + { id: '3', value: 'critical' }, + { id: '4', value: 'critical' }, + ]); + + alertsOrderedByParentId.forEach((alert) => { + expect(alert._source?.[ALERT_RISK_SCORE]).equal(75); + expect(alert._source?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([]); + expect(alert._source?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ]); + }); + }); + + it('should get overridden risk score if the rule has a mapping for it', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + risk_score: 75, + risk_score_mapping: [ + { field: 'my_risk', operator: 'equals', value: '', risk_score: undefined }, + ], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const alertsOrderedByParentId = orderBy(previewAlerts, 'signal.parent.id', 'asc'); + const riskScores = alertsOrderedByParentId.map((alert) => ({ + id: (alert._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id, + value: alert._source?.[ALERT_RISK_SCORE], + })); + + expect(alertsOrderedByParentId.length).equal(4); + expect(riskScores).eql([ + { id: '1', value: 31.14 }, + { id: '2', value: 32.14 }, + { id: '3', value: 33.14 }, + { id: '4', value: 34.14 }, + ]); + + alertsOrderedByParentId.forEach((alert) => { + expect(alert._source?.[ALERT_SEVERITY]).equal('medium'); + expect(alert._source?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([]); + expect(alert._source?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([ + { field: 'my_risk', operator: 'equals', value: '' }, + ]); + }); + }); + + it('should get overridden severity and risk score if the rule has both mappings', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['signal_overrides']), + severity: 'medium', + severity_mapping: [ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ], + risk_score: 75, + risk_score_mapping: [ + { field: 'my_risk', operator: 'equals', value: '', risk_score: undefined }, + ], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const alertsOrderedByParentId = orderBy(previewAlerts, 'signal.parent.id', 'asc'); + const values = alertsOrderedByParentId.map((alert) => ({ + id: (alert._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id, + severity: alert._source?.[ALERT_SEVERITY], + risk: alert._source?.[ALERT_RISK_SCORE], + })); + + expect(alertsOrderedByParentId.length).equal(4); + expect(values).eql([ + { id: '1', severity: 'high', risk: 31.14 }, + { id: '2', severity: 'critical', risk: 32.14 }, + { id: '3', severity: 'critical', risk: 33.14 }, + { id: '4', severity: 'critical', risk: 34.14 }, + ]); + + alertsOrderedByParentId.forEach((alert) => { + expect(alert._source?.[ALERT_RULE_PARAMETERS].severity_mapping).eql([ + { field: 'my_severity', operator: 'equals', value: 'sev_900', severity: 'high' }, + { field: 'my_severity', operator: 'equals', value: 'sev_max', severity: 'critical' }, + ]); + expect(alert._source?.[ALERT_RULE_PARAMETERS].risk_score_mapping).eql([ + { field: 'my_risk', operator: 'equals', value: '' }, + ]); + }); + }); + + it('should generate signals with name_override field', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `event.action:boot`, + rule_name_override: 'event.action', + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const fullSignal = previewAlerts[0]; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + + expect(previewAlerts[0]._source?.['kibana.alert.rule.name']).to.eql('boot'); + }); + + it('should not generate duplicate signals', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + + const { previewId } = await previewRule({ supertest, rule, invocationCount: 2 }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).to.eql(1); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/saved_query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/saved_query.ts new file mode 100644 index 0000000000000..c6d26e994a99d --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/saved_query.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { ALERT_WORKFLOW_STATUS } from '@kbn/rule-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; + +import { SavedQueryRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_TIME, + ALERT_ORIGINAL_EVENT, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + createRule, + deleteAllAlerts, + deleteSignalsIndex, + getOpenSignals, + getRuleForSignalTesting, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +/** + * Specific _id to use for some of the tests. If the archiver changes and you see errors + * here, update this to a new value of a chosen auditbeat record and update the tests values. + */ +const ID = 'BhbXBmkBR346wHgn4PeZ'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + describe('Saved query type rules', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + }); + + // First test creates a real rule - remaining tests use preview API + it('should query and get back expected signal structure using a saved query rule', async () => { + const rule: SavedQueryRuleCreateProps = { + ...getRuleForSignalTesting(['auditbeat-*']), + type: 'saved_query', + query: `_id:${ID}`, + saved_id: 'doesnt-exist', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + const signal = alerts.hits.hits[0]._source; + expect(signal).eql({ + ...signal, + [ALERT_ANCESTORS]: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_DEPTH]: 1, + [ALERT_ORIGINAL_TIME]: '2019-02-19T17:40:03.790Z', + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'socket_closed', + dataset: 'socket', + kind: 'event', + module: 'system', + }), + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threat_match.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threat_match.ts new file mode 100644 index 0000000000000..dfa1f81f6c5d2 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threat_match.ts @@ -0,0 +1,1306 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, isEqual } from 'lodash'; +import expect from '@kbn/expect'; +import { + ALERT_REASON, + ALERT_RULE_UUID, + ALERT_STATUS, + ALERT_RULE_NAMESPACE, + ALERT_RULE_UPDATED_AT, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + SPACE_IDS, + VERSION, +} from '@kbn/rule-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; + +import { ThreatMatchRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; + +import { ENRICHMENT_TYPES } from '@kbn/security-solution-plugin/common/cti/constants'; +import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_EVENT_ACTION, + ALERT_ORIGINAL_EVENT_CATEGORY, + ALERT_ORIGINAL_EVENT_MODULE, + ALERT_ORIGINAL_TIME, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + previewRule, + getOpenSignals, + getPreviewAlerts, + deleteSignalsIndex, + deleteAllAlerts, + createRule, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +const format = (value: unknown): string => JSON.stringify(value, null, 2); + +// Asserts that each expected value is included in the subject, independent of +// ordering. Uses _.isEqual for value comparison. +const assertContains = (subject: unknown[], expected: unknown[]) => + expected.forEach((expectedValue) => + expect(subject.some((value) => isEqual(value, expectedValue))).to.eql( + true, + `expected ${format(subject)} to contain ${format(expectedValue)}` + ) + ); + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + const es = getService('es'); + const log = getService('log'); + + /** + * Specific api integration tests for threat matching rule type + */ + describe('Threat match type rules', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + await deleteSignalsIndex(supertest, log); + await deleteAllAlerts(supertest, log); + }); + + // First test creates a real rule - remaining tests use preview API + it('should be able to execute and get 10 signals when doing a specific query', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + // We match host.name against host.name + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).equal(10); + const fullSource = alerts.hits.hits.find( + (signal) => + (signal._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id === '7yJ-B2kBR346wHgnhlMn' + ); + const fullSignal = fullSource?._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + expect(fullSignal).eql({ + ...fullSignal, + '@timestamp': fullSignal['@timestamp'], + agent: { + ephemeral_id: '1b4978a0-48be-49b1-ac96-323425b389ab', + hostname: 'zeek-sensor-amsterdam', + id: 'e52588e6-7aa3-4c89-a2c4-d6bc5c286db1', + type: 'auditbeat', + version: '8.0.0', + }, + auditd: { + data: { + hostname: '46.101.47.213', + op: 'PAM:bad_ident', + terminal: 'ssh', + }, + message_type: 'user_err', + result: 'fail', + sequence: 2267, + session: 'unset', + summary: { + actor: { + primary: 'unset', + secondary: 'root', + }, + how: '/usr/sbin/sshd', + object: { + primary: 'ssh', + secondary: '46.101.47.213', + type: 'user-session', + }, + }, + }, + cloud: { + instance: { + id: '133551048', + }, + provider: 'digitalocean', + region: 'ams3', + }, + ecs: { + version: '1.0.0-beta2', + }, + ...flattenWithPrefix('event', { + action: 'error', + category: 'user-login', + module: 'auditd', + kind: 'signal', + }), + host: { + architecture: 'x86_64', + containerized: false, + hostname: 'zeek-sensor-amsterdam', + id: '2ce8b1e7d69e4a1d9c6bcddc473da9d9', + name: 'zeek-sensor-amsterdam', + os: { + codename: 'bionic', + family: 'debian', + kernel: '4.15.0-45-generic', + name: 'Ubuntu', + platform: 'ubuntu', + version: '18.04.2 LTS (Bionic Beaver)', + }, + }, + network: { + direction: 'incoming', + }, + process: { + executable: '/usr/sbin/sshd', + pid: 32739, + }, + service: { + type: 'auditd', + }, + source: { + ip: '46.101.47.213', + }, + user: { + audit: { + id: 'unset', + }, + id: '0', + name: 'root', + }, + [ALERT_ANCESTORS]: [ + { + id: '7yJ-B2kBR346wHgnhlMn', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + [ALERT_DEPTH]: 1, + [ALERT_ORIGINAL_EVENT_ACTION]: 'error', + [ALERT_ORIGINAL_EVENT_CATEGORY]: 'user-login', + [ALERT_ORIGINAL_EVENT_MODULE]: 'auditd', + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_REASON]: + 'user-login event with source 46.101.47.213 by root on zeek-sensor-amsterdam created high alert Query with a rule id.', + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_STATUS]: 'active', + [ALERT_UUID]: fullSignal[ALERT_UUID], + [ALERT_WORKFLOW_STATUS]: 'open', + [SPACE_IDS]: ['default'], + [VERSION]: fullSignal[VERSION], + threat: { + enrichments: get(fullSignal, 'threat.enrichments'), + }, + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { + actions: [], + author: [], + category: 'Indicator Match Rule', + consumer: 'siem', + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + exceptions_list: [], + false_positives: [], + from: '1900-01-01T00:00:00.000Z', + immutable: false, + interval: '5m', + max_signals: 100, + name: 'Query with a rule id', + producer: 'siem', + references: [], + risk_score: 55, + risk_score_mapping: [], + rule_type_id: 'siem.indicatorRule', + severity: 'high', + severity_mapping: [], + tags: [], + threat: [], + to: 'now', + type: 'threat_match', + updated_at: fullSignal[ALERT_RULE_UPDATED_AT], + updated_by: 'elastic', + uuid: fullSignal[ALERT_RULE_UUID], + version: 1, + }), + }); + }); + + it('should return 0 matches if the mapping does not match against anything in the mapping', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + // We match host.name against host.name + { + entries: [ + { + field: 'host.name', + value: 'invalid.mapping.value', // invalid mapping value + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(0); + }); + + it('should return 0 signals when using an AND and one of the clauses does not have data', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + { + entries: [ + { + field: 'source.ip', + value: 'source.ip', + type: 'mapping', + }, + { + field: 'source.ip', + value: 'destination.ip', // All records from the threat query do NOT have destination.ip, so those records that do not should drop this entire AND clause. + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(0); + }); + + it('should return 0 signals when using an AND and one of the clauses has a made up value that does not exist', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + type: 'threat_match', + index: ['auditbeat-*'], + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + { + entries: [ + { + field: 'source.ip', + value: 'source.ip', + type: 'mapping', + }, + { + field: 'source.ip', + value: 'made.up.non.existent.field', // made up field should not match + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(0); + }); + + describe('timeout behavior', () => { + // TODO: unskip this and see if we can make it not flaky + it.skip('will return an error if a rule execution exceeds the rule interval', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a short interval', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: '*:*', // broad query to take more time + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + concurrent_searches: 1, + interval: '1s', // short interval + items_per_search: 1, // iterate only 1 threat item per loop to ensure we're slow + }; + + const { logs } = await previewRule({ supertest, rule }); + expect(logs[0].errors[0]).to.contain('execution has exceeded its allotted interval'); + }); + }); + + describe('indicator enrichment: threat-first search', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/filebeat/threat_intel'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/filebeat/threat_intel'); + }); + + it('enriches signals with the single indicator that matched', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', // narrow events down to 2 with a destination.ip + threat_indicator_path: 'threat.indicator', + threat_query: 'threat.indicator.domain: 159.89.119.67', // narrow things down to indicators with a domain + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(2); + + const threats = previewAlerts.map((hit) => hit._source?.threat); + expect(threats).to.eql([ + { + enrichments: [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + type: 'url', + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ], + }, + { + enrichments: [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + type: 'url', + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ], + }, + ]); + }); + + it('enriches signals with multiple indicators if several matched', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'NOT source.port:35326', // specify query to have signals more than treat indicators, but only 1 will match + threat_indicator_path: 'threat.indicator', + threat_query: 'threat.indicator.ip: *', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(1); + + const [threat] = previewAlerts.map((hit) => hit._source?.threat) as Array<{ + enrichments: unknown[]; + }>; + + assertContains(threat.enrichments, [ + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + provider: 'other_provider', + type: 'ip', + }, + + matched: { + atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + }); + + it('adds a single indicator that matched multiple fields', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'NOT source.port:35326', // specify query to have signals more than treat indicators, but only 1 will match + threat_indicator_path: 'threat.indicator', + threat_query: 'threat.indicator.port: 57324 or threat.indicator.ip:45.115.45.3', // narrow our query to a single indicator + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(1); + + const [threat] = previewAlerts.map((hit) => hit._source?.threat) as Array<{ + enrichments: unknown[]; + }>; + + assertContains(threat.enrichments, [ + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + // We do not merge matched indicators during enrichment, so in + // certain circumstances a given indicator document could appear + // multiple times in an enriched alert (albeit with different + // threat.indicator.matched data). That's the case with the + // first and third indicators matched, here. + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + + matched: { + atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.port', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + provider: 'other_provider', + type: 'ip', + }, + matched: { + atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + }); + + it('generates multiple signals with multiple matches', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + threat_language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', // narrow our query to a single record that matches two indicators + threat_indicator_path: 'threat.indicator', + threat_query: '*:*', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(2); + + const threats = previewAlerts.map((hit) => hit._source?.threat) as Array<{ + enrichments: unknown[]; + }>; + + assertContains(threats[0].enrichments, [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.port', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + + assertContains(threats[1].enrichments, [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + }); + }); + + describe('indicator enrichment: event-first search', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/filebeat/threat_intel'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/filebeat/threat_intel'); + }); + + it('enriches signals with the single indicator that matched', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'destination.ip:159.89.119.67', + threat_indicator_path: 'threat.indicator', + threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(2); + + const threats = previewAlerts.map((hit) => hit._source?.threat); + expect(threats).to.eql([ + { + enrichments: [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + type: 'url', + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ], + }, + { + enrichments: [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + type: 'url', + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ], + }, + ]); + }); + + it('enriches signals with multiple indicators if several matched', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'source.port: 57324', // narrow our query to a single record that matches two indicators + threat_indicator_path: 'threat.indicator', + threat_query: 'threat.indicator.ip: *', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(1); + + const [threat] = previewAlerts.map((hit) => hit._source?.threat) as Array<{ + enrichments: unknown[]; + }>; + + assertContains(threat.enrichments, [ + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + provider: 'other_provider', + type: 'ip', + }, + + matched: { + atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + }); + + it('adds a single indicator that matched multiple fields', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'source.port: 57324', // narrow our query to a single record that matches two indicators + threat_indicator_path: 'threat.indicator', + threat_query: 'threat.indicator.ip: *', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(1); + + const [threat] = previewAlerts.map((hit) => hit._source?.threat) as Array<{ + enrichments: unknown[]; + }>; + + assertContains(threat.enrichments, [ + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + // We do not merge matched indicators during enrichment, so in + // certain circumstances a given indicator document could appear + // multiple times in an enriched alert (albeit with different + // threat.indicator.matched data). That's the case with the + // first and third indicators matched, here. + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + + matched: { + atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.port', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + provider: 'other_provider', + type: 'ip', + }, + matched: { + atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + }); + + it('generates multiple signals with multiple matches', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + threat_language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '(source.port:57324 and source.ip:45.115.45.3) or destination.ip:159.89.119.67', // narrow our query to a single record that matches two indicators + threat_indicator_path: 'threat.indicator', + threat_query: '*:*', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).equal(2); + + const threats = previewAlerts.map((hit) => hit._source?.threat) as Array<{ + enrichments: unknown[]; + }>; + + assertContains(threats[0].enrichments, [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + { + feed: {}, + indicator: { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + port: 57324, + provider: 'geenensp', + type: 'url', + }, + matched: { + atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.port', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + + assertContains(threats[1].enrichments, [ + { + feed: {}, + indicator: { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + matched: { + atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'destination.ip', + type: ENRICHMENT_TYPES.IndicatorMatchRule, + }, + }, + ]); + }); + }); + + describe('alerts should be enriched', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: ThreatMatchRuleCreateProps = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'source.ip: "188.166.120.93"', // narrow things down with a query to a specific source ip + threat_index: ['auditbeat-*'], // We use auditbeat as both the matching index and the threat list for simplicity + threat_mapping: [ + // We match host.name against host.name + { + entries: [ + { + field: 'host.name', + value: 'host.name', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 100 }); + expect(previewAlerts.length).equal(88); + const fullSource = previewAlerts.find( + (signal) => + (signal._source?.[ALERT_ANCESTORS] as Ancestor[])[0].id === '7yJ-B2kBR346wHgnhlMn' + ); + const fullSignal = fullSource?._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + + expect(fullSignal?.host?.risk?.calculated_level).to.eql('Critical'); + expect(fullSignal?.host?.risk?.calculated_score_norm).to.eql(70); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threshold.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threshold.ts new file mode 100644 index 0000000000000..e3294ae9a8156 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/threshold.ts @@ -0,0 +1,385 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + ALERT_REASON, + ALERT_RULE_UUID, + ALERT_WORKFLOW_STATUS, + EVENT_KIND, +} from '@kbn/rule-data-utils'; + +import { ThresholdRuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; +import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/types'; +import { + ALERT_ANCESTORS, + ALERT_DEPTH, + ALERT_ORIGINAL_TIME, + ALERT_THRESHOLD_RESULT, +} from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { + createRule, + getOpenSignals, + getPreviewAlerts, + getThresholdRuleForSignalTesting, + previewRule, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + + describe('Threshold type rules', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts'); + }); + + // First test creates a real rule - remaining tests use preview API + it('generates 1 signal from Threshold rules when threshold is met', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: ['host.id'], + value: 700, + }, + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenSignals(supertest, log, es, createdRule); + expect(alerts.hits.hits.length).eql(1); + const fullSignal = alerts.hits.hits[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + const eventIds = (fullSignal?.[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); + expect(fullSignal).eql({ + ...fullSignal, + 'host.id': '8cc95778cce5407c809480e8e32ad76b', + [EVENT_KIND]: 'signal', + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: eventIds[0], + index: 'auditbeat-*', + type: 'event', + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_REASON]: 'event created high alert Signal Testing Query.', + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_DEPTH]: 1, + [ALERT_THRESHOLD_RESULT]: { + terms: [ + { + field: 'host.id', + value: '8cc95778cce5407c809480e8e32ad76b', + }, + ], + count: 788, + from: '2019-02-19T07:12:05.332Z', + }, + }); + }); + + it('generates 2 signals from Threshold rules when threshold is met', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: 'host.id', + value: 100, + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(2); + }); + + it('applies the provided query before bucketing ', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + query: 'host.id:"2ab45fc1c41e4c84bbd02202a7e5761f"', + threshold: { + field: 'process.name', + value: 21, + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); + }); + + it('generates no signals from Threshold rules when threshold is met and cardinality is not met', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: 'host.id', + value: 100, + cardinality: [ + { + field: 'destination.ip', + value: 100, + }, + ], + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(0); + }); + + it('generates no signals from Threshold rules when cardinality is met and threshold is not met', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: 'host.id', + value: 1000, + cardinality: [ + { + field: 'destination.ip', + value: 5, + }, + ], + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(0); + }); + + it('generates signals from Threshold rules when threshold and cardinality are both met', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: 'host.id', + value: 100, + cardinality: [ + { + field: 'destination.ip', + value: 5, + }, + ], + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); + const fullSignal = previewAlerts[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + const eventIds = (fullSignal?.[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); + expect(fullSignal).eql({ + ...fullSignal, + 'host.id': '8cc95778cce5407c809480e8e32ad76b', + [EVENT_KIND]: 'signal', + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: eventIds[0], + index: 'auditbeat-*', + type: 'event', + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_REASON]: `event created high alert Signal Testing Query.`, + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_DEPTH]: 1, + [ALERT_THRESHOLD_RESULT]: { + terms: [ + { + field: 'host.id', + value: '8cc95778cce5407c809480e8e32ad76b', + }, + ], + cardinality: [ + { + field: 'destination.ip', + value: 7, + }, + ], + count: 788, + from: '2019-02-19T07:12:05.332Z', + }, + }); + }); + + it('should not generate signals if only one field meets the threshold requirement', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: ['host.id', 'process.name'], + value: 22, + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(0); + }); + + it('generates signals from Threshold rules when bucketing by multiple fields', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: ['host.id', 'process.name', 'event.module'], + value: 21, + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(1); + const fullSignal = previewAlerts[0]._source; + if (!fullSignal) { + return expect(fullSignal).to.be.ok(); + } + const eventIds = (fullSignal[ALERT_ANCESTORS] as Ancestor[]).map((event) => event.id); + expect(fullSignal).eql({ + ...fullSignal, + 'event.module': 'system', + 'host.id': '2ab45fc1c41e4c84bbd02202a7e5761f', + 'process.name': 'sshd', + [EVENT_KIND]: 'signal', + [ALERT_ANCESTORS]: [ + { + depth: 0, + id: eventIds[0], + index: 'auditbeat-*', + type: 'event', + }, + ], + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_REASON]: `event with process sshd, created high alert Signal Testing Query.`, + [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID], + [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME], + [ALERT_DEPTH]: 1, + [ALERT_THRESHOLD_RESULT]: { + terms: [ + { + field: 'host.id', + value: '2ab45fc1c41e4c84bbd02202a7e5761f', + }, + { + field: 'process.name', + value: 'sshd', + }, + { + field: 'event.module', + value: 'system', + }, + ], + count: 21, + from: '2019-02-19T20:22:03.561Z', + }, + }); + }); + + describe('Timestamp override and fallback', async () => { + before(async () => { + await esArchiver.load( + 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' + ); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/functional/es_archives/security_solution/timestamp_fallback' + ); + }); + + it('applies timestamp override when using single field', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['timestamp-fallback-test']), + threshold: { + field: 'host.name', + value: 1, + }, + timestamp_override: 'event.ingested', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(4); + + for (const hit of previewAlerts) { + const originalTime = hit._source?.[ALERT_ORIGINAL_TIME]; + const hostName = hit._source?.['host.name']; + if (hostName === 'host-1') { + expect(originalTime).eql('2020-12-16T15:15:18.570Z'); + } else if (hostName === 'host-2') { + expect(originalTime).eql('2020-12-16T15:16:18.570Z'); + } else if (hostName === 'host-3') { + expect(originalTime).eql('2020-12-16T16:15:18.570Z'); + } else { + expect(originalTime).eql('2020-12-16T16:16:18.570Z'); + } + } + }); + + it('applies timestamp override when using multiple fields', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['timestamp-fallback-test']), + threshold: { + field: ['host.name', 'source.ip'], + value: 1, + }, + timestamp_override: 'event.ingested', + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + expect(previewAlerts.length).eql(4); + + for (const hit of previewAlerts) { + const originalTime = hit._source?.[ALERT_ORIGINAL_TIME]; + const hostName = hit._source?.['host.name']; + if (hostName === 'host-1') { + expect(originalTime).eql('2020-12-16T15:15:18.570Z'); + } else if (hostName === 'host-2') { + expect(originalTime).eql('2020-12-16T15:16:18.570Z'); + } else if (hostName === 'host-3') { + expect(originalTime).eql('2020-12-16T16:15:18.570Z'); + } else { + expect(originalTime).eql('2020-12-16T16:16:18.570Z'); + } + } + }); + }); + + describe('with host risk index', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/host_risk'); + }); + + it('should be enriched with host risk score', async () => { + const rule: ThresholdRuleCreateProps = { + ...getThresholdRuleForSignalTesting(['auditbeat-*']), + threshold: { + field: 'host.name', + value: 100, + }, + }; + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts[0]?._source?.host?.risk?.calculated_level).to.eql('Low'); + expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(20); + expect(previewAlerts[1]?._source?.host?.risk?.calculated_level).to.eql('Critical'); + expect(previewAlerts[1]?._source?.host?.risk?.calculated_score_norm).to.eql(96); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/get_preview_alerts.ts b/x-pack/test/detection_engine_api_integration/utils/get_preview_alerts.ts new file mode 100644 index 0000000000000..48682e6b1e8b0 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/get_preview_alerts.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; +import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts'; +import { RiskEnrichmentFields } from '@kbn/security-solution-plugin/server/lib/detection_engine/signals/enrichments/types'; +import { refreshIndex } from './refresh_index'; + +/** + * Refresh an index, making changes available to search. + * Useful for tests where we want to ensure that a rule does NOT create alerts, e.g. testing exceptions. + * @param es The ElasticSearch handle + */ +export const getPreviewAlerts = async ({ + es, + previewId, + size, +}: { + es: Client; + previewId: string; + size?: number; +}) => { + const index = '.preview.alerts-security.alerts-*'; + await refreshIndex(es, index); + const query = { + bool: { + filter: { + term: { + [ALERT_RULE_UUID]: previewId, + }, + }, + }, + }; + const result = await es.search({ + index, + size, + query, + }); + return result.hits.hits; +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index 866136f172f12..b686589addc09 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -41,6 +41,7 @@ export * from './get_legacy_action_so'; export * from './get_legacy_actions_so_by_id'; export * from './get_open_signals'; export * from './get_prepackaged_rule_status'; +export * from './get_preview_alerts'; export * from './get_query_all_signals'; export * from './get_query_signal_ids'; export * from './get_query_signals_ids'; @@ -79,6 +80,9 @@ export * from './get_slack_action'; export * from './get_web_hook_action'; export * from './index_event_log_execution_events'; export * from './install_prepackaged_rules'; +export * from './machine_learning_setup'; +export * from './preview_rule_with_exception_entries'; +export * from './preview_rule'; export * from './refresh_index'; export * from './remove_time_fields_from_telemetry_stats'; export * from './remove_server_generated_properties'; diff --git a/x-pack/test/detection_engine_api_integration/utils/machine_learning_setup.ts b/x-pack/test/detection_engine_api_integration/utils/machine_learning_setup.ts new file mode 100644 index 0000000000000..b376df9407c4b --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/machine_learning_setup.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; + +export const executeSetupModuleRequest = async ({ + module, + rspCode, + supertest, +}: { + module: string; + rspCode: number; + supertest: SuperTest.SuperTest; +}) => { + const { body } = await supertest + .post(`/api/ml/modules/setup/${module}`) + .set('kbn-xsrf', 'true') + .send({ + prefix: '', + groups: ['auditbeat'], + indexPatternName: 'auditbeat-*', + startDatafeed: false, + useDedicatedIndex: true, + applyToAllSpaces: true, + }) + .expect(rspCode); + + return body; +}; + +export const forceStartDatafeeds = async ({ + jobId, + rspCode, + supertest, +}: { + jobId: string; + rspCode: number; + supertest: SuperTest.SuperTest; +}) => { + const { body } = await supertest + .post(`/api/ml/jobs/force_start_datafeeds`) + .set('kbn-xsrf', 'true') + .send({ + datafeedIds: [`datafeed-${jobId}`], + start: new Date().getUTCMilliseconds(), + }) + .expect(rspCode); + + return body; +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/preview_rule.ts b/x-pack/test/detection_engine_api_integration/utils/preview_rule.ts new file mode 100644 index 0000000000000..1360209d9a175 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/preview_rule.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; +import type { + RuleCreateProps, + PreviewRulesSchema, + RulePreviewLogs, +} from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; + +import { DETECTION_ENGINE_RULES_PREVIEW } from '@kbn/security-solution-plugin/common/constants'; + +/** + * Runs the preview for a rule. Any generated alerts will be written to .preview.alerts. + * This is much faster than actually running the rule, and can also quickly simulate multiple + * consecutive rule runs, e.g. for ensuring that rule state is properly handled across runs. + * @param supertest The supertest deps + * @param rule The rule to create + */ +export const previewRule = async ({ + supertest, + rule, + invocationCount = 1, + timeframeEnd = new Date(), +}: { + supertest: SuperTest.SuperTest; + rule: RuleCreateProps; + invocationCount?: number; + timeframeEnd?: Date; +}): Promise<{ + previewId: string; + logs: RulePreviewLogs[]; + isAborted: boolean; +}> => { + const previewRequest: PreviewRulesSchema = { + ...rule, + invocationCount, + timeframeEnd: timeframeEnd.toISOString(), + }; + const response = await supertest + .post(DETECTION_ENGINE_RULES_PREVIEW) + .set('kbn-xsrf', 'true') + .send(previewRequest) + .expect(200); + return response.body; +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/preview_rule_with_exception_entries.ts b/x-pack/test/detection_engine_api_integration/utils/preview_rule_with_exception_entries.ts new file mode 100644 index 0000000000000..efd5c71ac7047 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/preview_rule_with_exception_entries.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import type SuperTest from 'supertest'; +import type { NonEmptyEntriesArray, OsTypeArray } from '@kbn/securitysolution-io-ts-list-types'; +import type { RuleCreateProps } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema'; + +import { createContainerWithEntries } from './create_container_with_entries'; +import { createContainerWithEndpointEntries } from './create_container_with_endpoint_entries'; +import { previewRule } from './preview_rule'; + +/** + * Convenience testing function where you can pass in just the entries and you will + * get a rule created with the entries added to an exception list and exception list item + * all auto-created at once. + * @param supertest super test agent + * @param rule The rule to create and attach an exception list to + * @param entries The entries to create the rule and exception list from + * @param endpointEntries The endpoint entries to create the rule and exception list from + * @param osTypes The os types to optionally add or not to add to the container + */ +export const previewRuleWithExceptionEntries = async ({ + supertest, + log, + rule, + entries, + endpointEntries, + invocationCount, + timeframeEnd, +}: { + supertest: SuperTest.SuperTest; + log: ToolingLog; + rule: RuleCreateProps; + entries: NonEmptyEntriesArray[]; + endpointEntries?: Array<{ + entries: NonEmptyEntriesArray; + osTypes: OsTypeArray | undefined; + }>; + invocationCount?: number; + timeframeEnd?: Date; +}) => { + const maybeExceptionList = await createContainerWithEntries(supertest, log, entries); + const maybeEndpointList = await createContainerWithEndpointEntries( + supertest, + log, + endpointEntries ?? [] + ); + + return previewRule({ + supertest, + rule: { + ...rule, + exceptions_list: [...maybeExceptionList, ...maybeEndpointList], + }, + invocationCount, + timeframeEnd, + }); +}; diff --git a/x-pack/test/functional/es_archives/security_solution/alerts/8.0.0/data.json.gz b/x-pack/test/functional/es_archives/security_solution/alerts/8.0.0/data.json.gz index 26952621f10e4bce828d8cb5e86504a8715be539..6a7559434f703e7ed3f292918c5c2212de9d5aed 100644 GIT binary patch literal 9203 zcmbW5cT|(zmd2@4HAoj}(u4FWp(sl49RZQvMT8KFK#;C4GzAom8X?}GuZ7p&JNrJ zd|Ns3)$;LO3jcXXQ%(!ueudKZxQ}&Qye#Gm31O}lDLX~B`|AmfjZQZRzUEOF7i%yM z4=3XtDsyEAa}yX#Y3=2)wNR5OwBx$*{p*$I2=j|Hi=U2MW#>bk6RAfKnd8s;nO65Q z1$zCRxgLp<9ImpN5j4JwOU)YIms6;H#}{RK5G*H1!#PQ>&a05!$4f(4O{`1bRJ(Ve{)9q&SOM2BC+ z_fSWx@OSCCZ&UZzyZLI<&#%p$2i-;?9iXcxt@6J{d02+VTNVz_ow_9WUls$)rdyJ@ z<<0ta?G?ms4RqU?nxzO=HH+#gC@Pj^_E)%{y3&<|ce+jJIZNG^FkU)FL7gh2dKP3Z zs~ML=`zKJSHS3!SD&dad)62wXZ)&oN8F`V=MGv|*VoN0{qTs- z`lS+!(-CoK8kDwi7FTh6sNIxeqA;(yz55w&^yGN%1D=6tz16KZUsc~bIhHw9ylGNB z^AJ3-^)jaeRTX-1nzxD%qkWkK9U6yI?`tEhrMv_!RR4#m@Fl{X3H4=JauW4VB6X`o*KDQOHX%aUki~`SjyRujxg#FXgYNqptAY! zc4{&&5Lu{LR89+17a18LaHyP_ofK=(pLO1<6fR3@X8{(7uRDpo){Ax=N#aq&7$SLj7$|somo7K6j4@qR8u%{#NB$=~e_ch{ z`Uz3`>%Uz2yE|tOQCbKQjccs0cm8-^Y`jE~;Xb4@pwk7@6|n7TedS)q(K0XLVIvnq`*r)< z;3~S07}5YL30YXEdzfyWT(O)Ct%}_?c1SOrny0W%i}1ISsC7f5oIgZp3sC1Z(Q6Jw z&BzW&N|qH=hv6=w9uJGRraZ6Ma>jQkoGlx9j<%vneDt{1Xtu1SC2q?2C%`PmBzYq) zx6oh}Fuj#nIFA^QjlB&?ZBE5>jC$`oHTTQj7>TKZyR#!g(sc?jLl|#cV2OM|u2P|3 z{r>(_&jL?RmHn}22cEGyuD4Ob4;K#7HQ#$S%#fG?T3o&BjG!rX;2Sd z-rPM#1{4;d(Id;%+K^S}T{H6@uYDUPDe05?%IW#qQAov{Ip(SC{xi7_<{~7^s-x@u zx2>(`xL!~T!U&Vg0CA=js&X=a73cQvrumoJSj*1O+?Vd=mD%wMS4f zhPZ@xOvXZ4Q&BIJb%}M=oI+R;el#?-rcM!Qj|K~?QuKmCs8z=FNbtc#YfF}U2W4r} zyAFFQJhAQFuzY!Ya-w%mn~JpTC&BkB+P-$7f|~Kc`*wTOX@0Yxm=Gw%dKUDii{)NO zH>ZgA%VQhSeOyEU%5u+pu5ABRvrUEeDC+W{xf@kAZ4uPGZ;gFVTFBdS@6S0zYc~)~ zX_#93J;+;3IU*oXH#4RAK7%D)eO6NQ-EgV6?w13>jgiF^DjeZ=IYO!4`Pr3!CRTL0 zZk1roCN-N7pPez=;c55^n^KjicTh)U^IPazWxmbhmsxjK6$kLFU_Tx@D7ZT$19XIPi+VeIPha;&CN;wVqlY4Yw9IZip4iBNAAjs2kqC zo}$WxXFp?=^G1V=UJ9|2fjxo=0e|r9)>VFJ;1YxUYkqyz7R$98)pi{>uhtt5dyH9k z8W0vx7a_Duz-)!;tL8^n`$s%2_ZEV?P-QHC3%X?E4u>_*YVOsFpf41CzkCvjBmAVe z%Wb!(XA+blG^NU$!OARL%C5%d&RIca4ps5Q7&Hxm+na8ev{fcypv|ZrJSA#i{*t^o^IJ;7(JbFuRt9xv^HsiVc7* z9SpTw3~OIX35U55IO3DYGV#?KA>u7Qbx!^RwJi3fx$|X#P9qoYm3FV1B`1TL-N^93 zRnV(4`BQOzas3dZwI#9U(Log8=4#1k^L@%cMQ0K+v;9}myHNXYUp1%pVx@3;pMj+asdt@ay?+Xr5bv`<-#%V3S$VbRN?zQ`-TRh0G7(T|~ z^B|F+^5*&kAsykGnsZpiYIw7XX7&KVW_7p^wvfDqy?=ASo?qnSLU5-+gL5+&hSlLz z=}HZ$q~>8Oy=@1}J#0!f(igN>4@(%X|7j3a4u2_xUNpo7cY6K}%?XCd!}Zd|sKQ!x zGhR4{TA>PPN}UX}D2K&wrm#m7ufHzTP_?B^p6L$DhK)VEL-k$&m;0)IuCEDPmFmB& zBvJ3axlI@*>%L?frfgjBs<1Vt9dUk>@{=X|tFHGeNz*W^qUe^70Y+;bojlsZD3d*Xo50h$Evd=Y{Ts092fq8LO2*6_Y&;^GTz&S+ANe z`73(*F>`(QZ@CIO2fmzw29~w!N)PbFd|1u%k}zmuSaot)KGuz`I_d~h9L((!>4%Lr z>45Z=K9d*tmTPdR+}5p?1E2>2iydC3#QHQ&w!^nlcigK|F^7=Gn0nPo^$e2R0RtJ{ zFhwr=CC6H7W9dM}rLZVqTVXvN(1@DvZoZYEUFo~Yp7KuBK zr%N6AVn!w-)`j`&A5#_Xh$_wVjTqZZJX_P9$qL1{p!8S0+E4KJhXaHAGFv%cw(tT} zYdQs0wRhF!=7pu(1wV#(KfDtpl>4;wyo8gT*j=KfIHQ2ew=cmonB9H;apIYpu0pjk zvX)hgK9J1+qr+$Q1K9x8mb_IzOrq9C-5hJ?onX5G!xh+pibF8$o3T2Yt~INu9jkj24zj~G-oeROglzUnpi4`46L+afgAq~^&nySXhw z>g6evgL$jU-;OOb@s$oGT9raAm+wXnN8~ghhhq;cld3~W(T&4K|81w-+p=Wu*4s)6dHkUd&H=oPT{Z`i_K6=kYGUNHibK&Eio)dG(C@qK4$MtQ$o3PX)CG3 z_Q+zYB?Ir5QzU{atv*w8$H8I`;IBZq5WFpdDUxEagV)tpbS(xM2@+i8H&^}w8QIc~ z1Dvr>CZb zqiJ};^+5ksA#+sWD=Azs-Wma7Qw*`Gg4l4(ceuc}c8znh-8Ie`UHK8ig6RXzpSGJ) zF| z7D=*T(2{eqp)l*01BhBVYqwBnSakv$Q#@OS4fE?SAhnX4c_{tdCzj(Bw8OWqw?UMb z_TMz}5-6(&ZQ)HXUiIqFj!vAL9vgy;{9$SKr_Nq8Jz9KXix zpKvD|8Kn%jpc6_u$#?rdbl z0sfZ%r#)%*Eg6NRB1QOLyhp!NUnKk!_SenHf`n{i)X8QUn8N6ArM9GEWnM58Qx1Wh%e^%U+G%|5 zexLDMI(smsch8j6zb?YeQ%&{Q@r_IhGnj+3p4VC%&i<5ifRSuUBvzAI zTx_xP54o}~K9VO?>h;kIMS)?xX-ZY>4hi{z)RYJ_md7i>2Rs?X@*+hrCA(G}$O&+L zp8&E|MMfWq(JeC8C02c`WX^FbZ^DkYsH*246F+I_mr^uwaV;ax6^T}5avg^PAW<&z z_g@S+5IJTDsT%SX=)?6XUs+QxZOaF6ry+bStvPBkEG?{&udFJU#G$Pm6yMA@c z+>)WJaxn_@nDPxT+K8JHcmNL=I`xx>(`V+oy}3?7$@&dvNceFgaP%91#m zxZtfS+%zg7Zc8mJ>FTW9PJ}~KoENleCUn?8^4GIYUUVcxtlZ}&`}@=FTUg$H;U~p9 zCyH~TMSgU$4SFOIo_#lpE^lfEf6w5tCU%&(T@f$I%Zs?K)?3y@J5E_5^flOWrJ~EM zu`RiGTra!j0p$0U|DLI-Y765TnX>v{$h!8cUM=4TLjlU_xQF;_M2JA$G?v&D<`aRl zuisnjj}0T8NZ1iRebtnpTQ&(}s4M^nJs*8P!YWLBwvSL72qC`BPwqf$rV;PE!lZ+> z+Ec6kq#D!ksav16EP3QJZ#qwiKhpDibhc8*e;(+Ep7|n?*pGWIo5l9cT|Q@TeaeRG zy(p|vh=p}ZJk;(}YM9V6v<&XGkm73cd^nNeQk8g5rL;0iP2j>mcC}PLcekvKE495; zijz%;yZA5+WPuLLaZUrl5)+cZ-1l0M)OIcL@O|n9YAHI2{(Tn0Eq`Vl>`t&-N)Sjp zW^sDTmlfCMjA6=G6w&~h#+y1Fb?DTFXUSFH3&0BjEdT*q1dx+5LhKq0l80v_)iEI; z!72=G0rH1ovmiw48|31PLlKg&W`oP{4Ni|-7w_ClMs(|arwVm~f;QgUEf&t=nZ7~OSPCYhjk zpW=};aHwP91o3QXev2(QX4jur!j8r>y~8%2b+bI?*vaNZ%Qt=DNK&!TtJ1K0mjnDJ z_fyB)9<$LMjL=f4T*LP5hw3>kI*Y;)EHSJP)zK0a%7HgcBggbYQ`qXLp_jPO%Yh5# zoB~Uz(EYSPLqy!zMA<`i?t>QS529tlcZq`u$yOG@PUs)I+}pM%GsVS(H2bAw)$_U5 z25KtYB8qg8z-hJ4d53X7ei8M8ToL41?ngzJ*BN7S$sCX3jX*OyuQg~TNT?of={>vt zWWkPbie$?_N9&MxYpQn&$Y=%&*-@d}nx}T#cSj!H3=Bjw?v_&8X}_eMl? z)=R2e-+|EgYwr6g;zP7l;JY*LR?nwSKdduDpsW681lJ%9`pnpKcZ|uf>mN?3Cq$5^ zPtp7;@L7XZ~T~U9F*!qdm*-q2f1OM!4d`%l(|hpe2Nl(Fc|SjW#vmH5R2^@ zNzI15el~E78sXGLkVb_dSqtSmE2-U>h4spsoG-~!8Nn1gQAI<&>_RkmE3FAKE!yAk zodwa)tLmOCHaOTH8hLp z&>XVY9hHtvylJbnIPb7Ne1`PWw%dla5@?rMLWm}7k#&Dl_6Sjs77 zu7KI4gZ6uv94R$+)OMHhNRe?reoJWt(?Ha432@qN!lZms^(%+os+Tr&97++HUDY{1 zWN0?Ps+h45iI|qsE1WDTMLMdPC+#ZJC06ND7p(qTWG@Cw<78{;tZ&}LhwO>3s%@e6 zBZX`6r+lR$rcQL($j+Bjr;C>||9>Bq#483GxUzCk9LT`0Q`{lqR-^$?#Sm%KIncn$u=U_Vn=_q+gWoUvWz}x-Pnya3Mv%trI!@YxC@(W zJARYloIXCJ8;fvd(D&P~8?T8z*bJCdR+TU_VKO#0M>gmIO%#j`ulvx-wjKAIO%u8) zIsy&Q#+UMtI3AW`yiQ_%?8ArZH%&=9FL62pze2GLu?}KEbn>n#J8l~06j1q^ zcm53M=SvD5PjKc$8b)o@6F*E8?ynU|@VMFHEI3%G7G5V=Qe*eogSsH2RY4$Au}YuV zkmzo~Gee`GP*Ewv;ibYIO&2d>KK(KAbw-W&3dJ1pvPfrm*#Z^Gl$57ClzPj5v36rx z2qMDC4qVrI;ROc{dESv;1~vda1W@*@#Z*{7Fyq1`0FsG+B^+>ls|^NP06F7s zosZ`svzDPmkjy--s}A>n>T~2HTlOuhH@hZ^>znY)zAzbC6QP67bWTGx%xNi@PvZ)B3&=@3K8jK znMUBX-#v<`L_g%u0NOmkxNBI1fZX!5j;FSvSS;)Dpg;^VFc>tdTXYROcQDg-((j&U7p7z^1pV2HUXgAb8RYai$Ys&n$zE-X2#E;%*mWMCk z;$Q{Wj-xLIR{@Rn)}hAHFs5weLdn}fq&D0_4qbd$bcwemqYS6~7lTI){cOd3r=P`u0187vp&{;s$2Z_-)w9D8BZCut?lWre!iI&NnSy`rqL6|I32L#qai2k6A!X0i6!43!#&dH} zPn9K1{7^aN7ba`8Ef6pka27r|QO2h3yi0^P|JbD!05}#cc95Q1|rL>s{aTz_Z(Kbf5Ul6C4 z6l)IQ*6D2PJ%v6#tTGhL_cnq^nSGphDo?sX92>D66VNt?zc6hJ1)N`iNAUyCs=jxD znTuC~zCZZApBNBIB$rFXTsc8|+=aas))Fm6x};Ay96u;fr!L)a&dMb28!`pD6!I(# z8@a6Mf#YZDHq~1$#F4~S>bXYdOy|+3pT2iN{8*#N0OB7Wd94I>o&lJBqIrA=AW}UL{QB9MZg(BI8i2a@CG(vV!*k0?)R=*zaIS0LEpO?_ku1A z=YyGjG!6H*gCxJs7ap0&#wU{2%oJx#OL4Ax3>wu|t2r#sTd67>$V;&#o=Et@o3W9AQ_6&o~w84S^j~+Uj`uaq=0`b2x~qIK-;ZZiGtCsAXigqH&gMpn(=(T z3Yr2#Wmsvb_Yw*F>jD+m2@roipgXka$lm2`tW%UO_O$G}KR@XMBQ%<4kZ#GKa^ z&vU>L3yomB;JNZXEI6yfLoNWX7e7Ie94pfElOU`FSHdeFcKzaYl?SvH@}iHs(%oJ4 z&&rt}$HWPc>X~xr89<8fkR&@(@b-B{2nK^&TDJHSri_|1662JTW;z&u_*?m`L=lM` zgk2qly|J^C9)x+P8|`2<&RoXd?dH}5`>t5X0?zjWGyiSrZL2afee~uP>P-x-6pvwQ z?-R?A6E1XhVgrN;kB*-v7p;_+hMgcyjM=eu^m8Xj?G9FYVgMufr*JN-g-@eXKl5w< z&!@8S{DLb%_g^yqJY3lS{@@!~?;qY*b|D7Hw5pj+;eRk*?O^s1t#nn(n#L!rw|MSJ z9`;Z;mnSl|?&e415!3B=8fEz!dbTEPQC>9h<1V!G^BSQ)vHYTb^Tq~+{Fk6GSsmHq zGrVP%pUV3f{HuxB^AE*;t9|9Y?>oG%Qh>J7f6Co<+7o1M&_9@8n#5TCql5ac=Jy?& zy7^KzDZK@R_>6Ulfy>)6&S{-(32HoS;s$?MD>MBU)=&V4DXYa#_1{r9JG=g;sJ|;* XdYx?f%IOv|-!J2LtHp!q2yp%bp5ZPj literal 9231 zcmbt(c{r5O-@c-xL|H-zHOQ7V+k|AvI}V%U1R=Oq3;ijErqi zNW;uf%4B~Z)pvP+*Z2J`*YBS*&o$S1&N-jY{kiXRpY!mF&y@&2Bm-B}WXRTWtxtZK40{O3Vk z)KuEhqRBK5_ba&^VyKt`sA2|rq?r$Yl8MBDHGOIELWpcPgN7dO=qIF`Ua}fozoPvcTvv+mq7nd^}ys_T?Dsk-Aq2827JZ6Ss zJo{4L$P?Ck+2pcR^vu-M&bm4^zuPd#A$#CrX(=e=#quQNC{&DgM- z!p}8hB(+7$t2|{EuiXi- z3^lzrNz}H_9s6P!$9dJDCRJQxX0{2=H!Qc0iQoE(HwBN?+gau#MwC%e`A+wPVHCor zoHsUT!aGp#W@!3?S7DI%d~sRgOGQTE+X1ika+tpV$O$+fTpw+9eL(9iUG=YSkG({f zn3?bDZ{#kAW^Bp~)uo<(fU<~#w_t+eNzrIq{L=JxpI;wg;IIu{Exqhx`IM0QB(7Mv5J}gDj#B9aeSju`j}nMDaUGwY!8S&Dj^ivRW|cgIM}u4K{h!I>7Ll* z&vVeK@Sc+`G;g+sySatE#`$`?4-R*4ag#^rhA(5N&c{NDwcU`#PuC*Ry)*j^wj!M@ zrKImc2bTNWp3|szj`_qp_ACBa&xc_`1}VNuV2IV7Ak}`P^+z_-WzRw;(q{&LNioPG zPua52_#9?(OC>eGWu|`pVB~Q3OW4rm(K$CmS=?Gq9XiVjm-zTD?daU}dk&r9gBfG` zgP@>%100!e9zQShcJeNKIr-Z{|5KRC_K1u~EwQGoBEWY1BXIWVGc+qov-3xe2fqxwkn!EYy4qTFu{k}Z?6&Lf55iU`Qz%A= zwuwi_se0g5P4pLo_fy~A9HQDK%UY8rlxy?ngy7d!%ZOnu=&VSm%2#Czf`nF#ixXwk zShb3@>1%#SdPfTja>MNFA4qNY9uCchHXKgBS}&}S^?Eq|#{A)y)4M4){#S!f1{5ta z{ECVjH`mu?_xG@Y$nR@a1*@NhtDitE2CrpL3H5W!2*o6?e1+&cg zt7VE7mBr062(u}|+jmPik0I2EKjqER8hRvw_+}U3*-UJYK7k*T*Y(^n+5AXQK%daknGFY+@%S}O^t4Yxw_LKVIT&s8gu_Q>E|#Yf#0Mr+W-ee`+lk}>eG}J} z;d-%BmzMnCz)aj{vb$o|b*h!~N|jdvCtTw*BARlWY82&e2TkOOn68oIs^zA40lyIG|@N3bOA z83G%UjnBm6%|~V(d&=f-z0~yy3{c9dK|X-OHh(Py;~$fi@3K*d#>Bjrt!AC5nabX6 zEIFgsL@}~@VijwR4}RXTzM)d>I_MaGU>ao-zGp1mVq`p2J9?JGAGMw7(3TN^+0T-7 z^D!`@bT(raZ%BGkStp_O>#oZuw94S)*H( ziz|$s8cu62?_Scj&ynxehB>mrQ1UZMQ)fLlOdi-w`1d`-RJYmn9Kh~*OwxCTx!j=L zWw=UOf!YrCZ+|+F{7Jlf#XzXPm1Fu0Lt?(Ym+<&gzBW15xN~rb#=3sn{lopWef?K; zn~P4py;JTu{gD`bw(%X#AT$iOFL$F% zmeja4JIyjRH1hgvkmsWskJfuZd9WPvl*dT>7RH~FTB+T)H^5O%npm0(-n&HF!%%Ix z?NgpM+|FMhJhs(ow4scyR{9KRUl7=_@2pYFSNa?pM%hLC*E?-|o5BaAY8hFA2H5FJ zu3VDW7ILc&G1BXDt~)<>ku&;K2~CDiT8-4jAC`J!CE-1DqC^1{lQu6?l@Hc<^$n9lvx5f%i}B z_nF%74!4d9wTH`v>tv-S-YWijR$A!wV6KdMML|`3Pk{s_~d^7!D8Ytxx`XpjJ|`MDvN$|7`)9r8N!K{{=L|L zTl?;s1Eo=b;!VVun>-&Ni(eor{$Q>T+UnC?%*0uO-9TrW)Fl=8+LsWp+7oOtnuMoK zLWZBHP16a`GpRREXaehLehq$iSsKX0?zO1eG~NqYF76j4(GQZdR8AeW`%~6q$1Y@T zAM`SyZKp|&FlAi53tcw+;M|@s*^xV93c;WI5yc{-?Y1%DBxM6-9f(l@FZ~#&j7dO zJ_YwWIC}Fput5fKZgy}lITWn}1z!ZUf``4f!@bB4Za|+llJBQ8?xC%D$U7x9BEQ^! zvLd0*DEejvy{u!A{gxCV^X{FuOrlcMN5yh7ZtyWMAeswf7Jm0(>1&rR;&Tnh1Ky){ z%fTOZf0@b+9fq@*+20RR160}0|LDbaPg-;>0}Wd4@aW~=W!7HSef=uhKWCP@Zdch* zVau7q_9s6`oe~s_0PS1-XsP6{S;l!OqM5v+#(cS{Q*KwZL^>&N(wk!g_80hE9=h}BfPx}qrJ;agCzpI)kU zcEB*|CO^(WCCeTmMaS{b@=x7QAs-}j0F@b%@eZeRE^bgpHX#%L`u!kJ1Zk@79eGL3fKiQ7vCSnxN5$E=(|G`B{-UVC-an2LU*iS~^UVdL|5>j!5 zSSCKz-^eK|t#1tWnO)q_K)oI$mTVNoQdc{d=0i%CN1&Z3I1G~S&7#!SQW|L@9Y zfV@*^vPS(X$jMRl=Ts=oQIR_q9s8{a&HjIjheA?)g6|zB;{_;>i5P=_=7zZ*EuErG zMleeug2unQhdbI>mbQTq5<+z1;54l`_bPopei4M}vljL0zoIZ@{eMVKvAh!ckfa-h zpSL}rf&*X1d16@%W^th6mfe(1Tk^Ig;0qe%$=NotH==pHZVK{G$!`U3f6kja zhu6-szw_+sExN?%K>-Pc?z>B&nF_U55pF&&z}v#D zsnA|BXg=jT5}m3#^}UU+?sEL9PBut{Ii?!nUE-D{bT+$TmDN{p6shjdgxCJJ2LlQT z)OfFVlyMTdbcGDTv^dGC`%5Ji2NM?RSQ26tVfXbWMEU#mhTonjX8Zr&G{$Ea<8nAAh8G^h7!9dNAuA%A=l_5XJX@TK`&x zPEC@mau@A~Q_9Whb91E)s#4QngXV9&Z7fuE-i^I_Z(^^|kWm>n&|#qy$ms!>ZRe6L ztJu41XJ@v5BA;**XV&b^QO_{FlgfJ9R*1G=-%1205BAC%6oxDpN`RI^OG z+}HLtw7huvr_fagX_$IzF+CoR8#0g#Gf}?oZ%~EII;T;M8$Yvt*5Jmekrt-QF)2ZF z=lM-#({-=-zLEs%Jlf8MmbyTniA6-fx5h`jG{E1;iq3)!u(BhaT(5C+zxW#d8Shty zgS1I`{Fp`&(p)}3fDX*lbfieQC!QPlovp^pmBXJew_DR5M|*Av;)cs&sLN~uzDNiG zqfQ)+njhf6&IQf8?)_eFC26DE)MzKU zq$B!Xge#uoG9{ErQWo<}YgPn0oVf3;kXh23Lc8p(y0Dq4TBH25YfC=9!a{ z74Je`&vI5r^nJdnV|RdU5?-8QeU`UN4(t^lw^1yo6kOZ&OqMPKnR~VM#k`V08)un| zDyWu2m0|YGl6pGA%+h0y)nezZG>7%HE(6{P5JxD!ZT;S5b2W($mRDyc@yi>HOkGsi zp3-kr5+^h59}E3%P91VZ9`@7mQ0wvSCxvPqmKOt$lpxx!`v7{u%u;j-0s(0-x>*9~ z5p?dfc*+a@t9h*XhHpl#u0ANyPQR_GtNwQk*nwzf3-fq#gQ3KnSBbQfKe5pu# zud9(6Je4^^Y!VIiR^I`Ho#ZV4iF2@LnDfde33IFwmS4_g4rZi*;};BSkP&)iDtGz| zknKM=?~Ov95PrCgK2v+kH?j~{ncLyJ7AE)=S(0GUustUwM`O)irk!!cH1RwrubmEt0#c?M&S zxSg{TYUu^?!?0&V0rD?Y+p`BBhF4}nB3xIN7yMU`JM)#Iu8CBMFl?&KqY}Z_<$w!DrD{275%c*W?%8$ z>BQV6Crorc^kLzbI3%#@m92AY2i?gCX?~w&j6voFdYJIaVs8!{W;$qV^wGogPIDu# zgA`NGraea-mmPaa%#tc0A(b!nQ{Jmy&GW9ts`Z##c7Q&u*C_AvknwchVW_16mT zo;lx z;=rdd9Kbtnv0ThMZmc;gT^o53;M5CE_Qh%lKe)WP&quS+ZZ>U%OGF5=%qYfqGL7|$ zgRLXVY_yI`?ZZVt%dA{haAKGGcN(!@dCQ-_zAW%k2P7cOmR6ma+?kUZuhkU5(g3re zI%3G;wFtPUFuJ|5{^>FF()EVT?rDo`GXSCt#-zXkIa zK*SLeBPkejB$?z(X^XBazg51k5tc;wL94^jP~RLHDl+XNfeaO!_STYL;t2LhW6p%k z>(iBD)yfKtKM1`t4m_=GMD#+wk~izD>99 zPF`{Yt>2V?T&@BiDt+>tUZpk}VblFVL#zdd?g?I|v&*gnU5Ej_ z6QmfcKkApmXAU=fDNiqPf_NPM`>Z3`(eT>k237pAQg*x8qvrHm71~d4pl^GrjI~sX zdB#+P3B6R<_LG^vtFL+D#Mvk6UYQcI3Whu1mgOp(8;m4IVjyM?%faa}0VC`$V%14$ zx|bcaNzwBVFNK(AQeU#}j0x1FW!!SpXgMo%XQjNj-@B;cLh}2Yk@!bhcLfC$NMWp-$hxuPjHbg0C_jBhmn@y05tZq70h|lcIELerrsT z=qLO*+OuqrN2PQdB0BcU%q5W+ClE>1dJBKq38GbF`K`RrPmu|tA^aL*g~)W_e`-2= zalIPEf4|So^%kmyDb>O2Q_4@nI0tWD+|VvY2-*nXF?oVWhW@govw!UJ0Q-oF{V%Hk zYDI=IybW89OgvT!Y*Z@6VJ2h1W6)U%rE`;h5ni`S6S_EZDwvJ5vl>?VrE(nl(%AjB z8}{^*g$d3D%G*q`XB19!GYdI|ePL09l(Drf<@u%rxL!8w@*awQc-i%VGlM|Mxr*yO z@|E_{Sr*mD2ly%O@es!&J5hoB@&6%w{AmEI61hojb@_6(9Uq3FeQE$YcxTlkjE>yM zCTbtxRrBu2kHbf&Xwr?NfK*U$1}Wsh&3~E@j#6x`1}6hu)Qt%78t8E>0sWsrdIEst z`wx(Y2|ptKo3Ri9xClqK{yV@WvGF4Ns(OX?EBUmN+lW30u_UxmIjUSDUJ?x z{lc1AemcpI(k$SQz)J7^G-`+(&~7T@4zlsVF*x9{sP#B=dZ;`8k_liG?;h;jfrwx* zy~I;M`}p>YQbOq^@NSO@M>+xciBtx4?^@V|fFq=Bx$FGPr&<#dpZxS9ojhaG`P^Qh zSot_zqGR)l8>(ZufD*ivJ20!Q;}RnS%f5PEExo7zT=i8t|KSe{kDf~^esb1M6`>md z>6SnNzLSU*`SY>xQ2Y$R;tl*FH_b;$vo3Yhm@f6@!a7!IsYlGILUMe+ogeYz@R!w3 zABHuJw#J7qoFt~;J`A(DIl1VZEkLF|hlgVJ7^YGBU=E=3RKkV7X7Ls{e}-pzI~eVC zhcv4s-1~8u`iZsNuT-}cUO>kXzu81js!|D88?Haeu>FOL3Dcr;x)j@N98UQ0LFkA~ z#BrYEfH|arzxCZwGbrO(c}R{(`0U1iKNm}&-L*%_S<%GEongFZJbDRK;@icm1VF&WnGCKHdXRBJ?_*Z<#%w5wmHLVrXg)zYFd zY)5;F)I*KNtJXV%2uXp9o`{9%sPDD*V@y|Ao3l9ITGLCVaX#t-EJV`yZq;Q*Yoc1XA6;E15pdsa@L04#^(?kp&1TY_w}1#6?eI{P*VQ)+5x z^p{_k7QKX)8osvEM-BY80#3DXsPjr`-ON3(sWgv&pPTc_Ktt*f6wgqAw4I*bCZ%nI z8(=v=C&7k#PO*QwdF5u6hRtpGy!SK;xT>o}{=?1RFU9f*ID8nM+u5#m76i7GM(1~Y zpOTya9}lrp43Xo{gfIdwPqnNAvd{MzaCvrufHVBzyW<1-qC9yOW5(2@fDC;lo*q=> zCt<;*K{pr{$ciNdX^veQcbFH0enG6eF%5q&)*oI@smn9s31-MP>}DFZf(=A`mQ3zC zwOnUqdP`E=BzEd_^?ND#q_r<0^DL-P{BHJh2KUTVOh02az0^ru~=G=k8 zslzm8<&&AW01_n~7;3AW%p_AVrSBdCMJMp^^m3h1+n$q2p;RfTQ6Swht!+ZpalJ)W z)aS+Vm?t_OI#Kzn?g`gzVh@bzci@4aPbAI>_G?c_!Sd*FAazbI!RP7p!eQ6>bfS~m zbz%m}^r#>)VgI%vb#~`ob|{?fw&Wb~D^>u%EfO&l zKR<|>Z6B6*qq)K#Z>Lt+P#o!8Yi)JXKNz}yXJ^}A&WXpVdalUX4WFlsX`U#Xvaq@$ z_iS}EPDGk*8X7PDbnUI*A7gBUI~ck|i?KCz+Gd1;l`oaQ}fqA72vz}XIwNwW#8!~_oT9u_YTF}Wq} ziQ^N*Z*Qx#cT6+NFqv3Oh}nSqq(ffNsW9|4*TyK_303PpmcD*}yIolL)t<9O=cj zRYD00Oz%dMfW?A9MPn^m=m|4gvlW1y_TCME<#+VlM8q!r6}OjKfH#$oDM1F%te; z2sG4Yo24x?2+pq2=x5nE^dIx(*FK z7C|HT21fgP70PK>wY=BzcvX{lwKf`4?V8hMdFjZ^X^SZfaembd`(2=NYM6}E2x3W+ zrnE3Fg_DmXv8hd@2o&{S=TIX0nTcX*og;a<899%$T(QMeqKr+DG81mIj`L<~p zd#0|gPuSFg&+b%AxxQK(tKaW$o}dT*ybsLyYVOY}!fEC&(KX$3iAY%hPCD1<{-C56 zJ}R%9I&QO>WP*^<%HZPH^nOtBT>UiDl?Osb7o~poR90QT{)|@Sgdec{g>V`IF5&mM z3s7b)Zy$L9203}J8=&ZzcV&XAB2k(0vlipPg?YV$hHnzD07C?@%%94?13w4)wWak6 zufvI;Ir%e+mom1x)%7ayKTv2e= zy|f(e^eizE1OvO3rG9@2W4ifPUFqSE{K<(mCF5|E?|#33pV4WmxlyKD?c| zS(0dwYjmTss(1pSTWWu5;!KLp>7rgvU}6y_nM`?lpt1nF*ji-wdBYVOtqsg$U=UvS z(*awdwb9E!X9Ijt5x4ulj9(^w$mU5(f3@(H0reF3A_&ANHJ~_Mq_^T+fAZJ5M$sWn zX9q}&sm@E{QP?Nwq@KmRJAwi@2ZB!vFw_FK@-&h@zkYz6i#Zu#bR1x3r=j{U DK?YAz diff --git a/x-pack/test/functional/es_archives/security_solution/alerts/8.0.0/mappings.json.gz b/x-pack/test/functional/es_archives/security_solution/alerts/8.0.0/mappings.json.gz index 3a26e140e7eaa87becd588b84b6cfae9955a2905..5a7347bcb63de38d68f5ca61618dc75ddd283901 100644 GIT binary patch literal 9609 zcmV;4C3e~$iwFowOgm!$18re&aA|I5b1rIgZ*BnXUD=Z3wvv6Xujr^}yQ14O?HS*v z`4a|2LK1FqGZ%H0Z}{H_fR++PESW%pQnlF;4yzaj zzi-c{_bb-6qTWp!UA)(<61k$P89!fS&niB!sjTu*-nT5*eS2=rFCV|ZiVdr$@2H;V zT`!tiz4N~>UthjHUw-_1)?sbew6Xnnf%(R9W0?3urxTz_9RN|WR5o>#nMWhRW` zI@#S4BOsxsM>LVs!*N7DrW}&alj-yIWXel{oK37KI^LUR46aSgNWwQI>#^GK&Nm^; zrrsH5<1eNuzGJociw)-Pg~)W2%REf$w5JhPzaeEG$Fpz6r zu-V`4g8FNpb}|a?6yCu6$f@Z}5=>}W_BB*j%k%ClA0oE|i?9d^YU1|t)X$5IXn*O) z4H2cGX|s|acu6JN*c;C*-q~ek?-#)!hJJ}liV+8T1I6uFmh=i}*EOScZ9z_C&c1`m zWsyz4Y_uH!%3rhd~&+dEs!qY!PI>)g9muP_$@!vdOmzJ$_4x;dTCIgIn zQj*dTIf%G5`mnA}`vm=oH+09u!<~n$lMQKwTPK*`x4&Gk4XoSopqa8BFzh0%2aNTA zu^uqi1IABtz<4r!o;YusdSG(Z-42V>fkTnqP_f+|_E=M(i0u~Iv*pIZOK04o*=YUY zq(YL^(a95l2%+PhPzCP?QJcc0Yf=VArlVXBqe#}IjN2#Hqzn}CH7R3F%2<;!PG7G{ z8D=7DQpTE;ae8S@${;4aCS^Fu{bWfQMR~g%o#vN1L1eltW*EC5ySxxPY58TW+%=sT z_7$yqd6Qb*+nJk2b~&qoZXVfKQSZ!^8sU=4v)(X947-WssJ#LSPGg)^)a|#G1pBn@1!&f-shcYAx;3?(NmN> zvsbKz6BboVehfO7IA&UEdhbQo6r(+)mHr%4-_BIG8@cX($bk#;2$sy~e+IVGG(XvJ zTYDQvkp2WPc#Sr=(|f`6$~|WV@3L*lcJUztf@qes>@7Yx9GyQ$6+JO0u`EA-=}jVl zcP5;=%Tp6h72vgrAims^>3H9i2ify+PEkpwZR(!W%y&!sV`Gp!M{OGEiqm=5HK@os zNv{$pG-<^3mH_>kPQ`|NpEY&)o(=rX6SS5lVg8y89Wy7^ooW6MWs$Rc&6)!iNFcZA zYFx-#!C^={JhS=^aP)F%S?`+8bk;@^qL_?%$C=rBceny(v0{0W>donnC}E1Rv&9S% zK?Yt&Hpk`>SbCLG)Qj zzbsW_I7*^_&HB7CyNl1Hwq?USd&38ztZq7!;}}D&=kgHkX`g?I_i{5(_)U>i2IdZE z`WKfYV!p&t>o(%p1*|OZ#=B*+tS!$WsVux*YjobuH2E9PN0Lo^kcwyLUgq^t*WQ`| zUc;{A+AJjZ)F-q({6j8g%HCHpEKj7dI~5z`Fl|Op>KgcLvPnHbiYDix8=l;!HkGTp^OTV-~m7nTV26QO=3R1_zTn;{3xVd z*`gapDZafxjGzT)CIo#HtU!9ctK?!yzSKP`Wy0%@iaxv__e=$tOR?!#XYYwn3$WbA zrSPCEA^s-!im=$^4W(_Qq?;de002CdHEnfGTm6Tmt=`7@%*RGVr0@vml#whUnn)kPF8}uZ>mNa@xzv~G z49eN5Z@8hI&3!+{xx=JVsNB`jFzGB3o_Of8`E(93pW?LRX0P%w&sAtgg@NPtw?V8)|@#KNV6R` zrN}ec35g=AR}Av8I}mSIAH-D*pq?ip&gjGQ@~?_zU}mLNRJ!L6<{um^a?PiUs?g?R)9HQ9kT3t5)V8@G0y{l-4XOe%?J5V z&@Rk(jtqgtb~0NYh7}+WLp*qsmnWsQb4=fO?r{o&lGw6Jl(a!31&2pU@RfHprzJ_h zvz2VvnoGjMiIOJ+M~B={Gm<8{)A}OMhLF{h>GQ-G9R`$jekyDVSz()|u&%E=>oRqY zw)<}<#LeaI7Y4z~-qpGQKQD2`OT~wMLrl_<*AY%t*UEP3{9V7?E{7wu#=0IJd_AO@ zZO1}5kf}uh*qjdZ! z0}^XS9&d^`pt2~aTX3Fcr18qf6Dyohf4qu!GZYt+cKo5x;zypy06&L2HAbKl&$ots zE6YgKzOleg!sD=?0mI_jY7tGp3^zpMc`a}nuKTe{xJ{i-#F_D`wiye&o;lOsBuwP> zmXyWw;5FNnu!CS2{kNR!7W7#uj8UN(AJcWYJ~mrc_B?BwUa0z_bXMZFYe4&X*r~j# z#z757!gK$?yF>}qv{Wt&FV#r*fzKYqtrBE^^|wM0&)Lb~;?js$jWKqC5T&SkQbyG|r@ z5ouP6Ldy0GDUo}Iz1nkKEN{oGb96-z=IUS5aHMti3Da08N$l39${Ws?*^L$?mX_K!5jc1y0pgo-XaRVJW{XAmy^z)z*)K8+CrT%%E zrilVaGo1jKkqwl>^Mw2NKt*8r#r(a?S5=BS?T&orF^C~$kp2vKoc6VkHlyO5TPM{f3#1J(>pi=%09+^<&e{p8JNcl(+vf!J`JjPIS}?`FKUiI_y=rSA}p9xK@|o?C`MWRBt6 zkRG%2H00f50|YGKelLG~url0^wJW`meEGlk^EIM>@fRoN!f`YLf*--Gwof;TZ6_MY zxgGr{d?oEgLhE(3*EGz< z1^}@_Sq>qfpJAn#-EKO0;88RG?|c$lm9RyG=LPYNR_rqZ(ku4a<+T<2?8}OM zHb1AYJM?GVht?h1BEph8^iP8Nx8kfVW`db^(G_RSaz*|X zCe_hHuI_y9jPNXaV zjHGI;!a5+1bEU|hKHfT(!1KphMcw{OnTC%lJRyDPMhBEs*ytZv>2J85BI@L%axYk| zu9i2tcP{MAFt)eUc)1N(p2@B{R>>)ue46+wzq}pkkuWu+hH|3o$wxLp)!mDoUp;+P z#Dl2rIdgO>$V0tx*|jtIf?TF~$*k8y#8X$mfXRq+6$3j)p#UuKNpiVvK^qCy! zWaJ%-FrN7RE3!nDV(s?F{5UV!`UsL}>K#~y=U)7lX7Z}V(X@%s*<^1fL~%%pu+r|| zdy%s;Yqr~zkfj$sj=Og!T=1iic4doh7^V2EC>$3EMYiC~grJXt6-du_m0T>zm%2x# zOnBW<(TDfro^@Ouk~bae>^%``>6BXslrFWe)C>oV@FmF!MU<<}-*|4t{KAqF)*LE? ztHCl3UJIrwzSEl3d-jc*;BVi5rzZLRn{9^=B-t07N8EaWS_nkG%+8D;4={SI$BeMf zbGq9mWZ{)QUe_C>srnh93l7$Ni@5q_r#o&!-s(bek@vdh;#-gy>mT54& zRW3I=#QL*U&n=UW+j_KB#&MPS4{la z0BOc7Tsstdn6@2T$d!HYnIJj8su<4^OU!-|9Cbv}xNF=md)B=}gwgcx3So2sJ3$y> zxAqUC3D)?5hYzQgC!e%tjgCmVFl%uH0h{JVB(+$zH31eGT=Ap?BD-y{MGNE7AFP@e zF?tYf>yhfo^!d63O5xM=!;G}E2Fen6@lFUEBBs9`hThRHNy%jgd1>MTwQ&UusJXTm zhxn-`ABv_yQh8_twvV5qC7b6QIH#yBT@BS91$q=;>=y^xVgZ?t?$Y_4#_qC5+a1QZ zzHrN5v~9fZ{HiikUA_Kn?hcu}MIB#n`;Owqk64`DrjV9|7GjC}H0y zxKnro^CKrS0Q-cNWnV*eH4XN>2QA)D;b?qw71Af#U;1%FL}_T+OkZO}C7R8W@j{f? z*!x8=h@oHNk|Hh0nnIGKS3tY28LevzR9oWeJD6M++4RdlFQ_ZIXZxWpW-#rcO&Q~x zb8Fux)8|_YOw0_{MNBYWA4J#Gm8Dn>N2}fQig)t-L}2gU4J+@Kh^2NkEo>rcI}g9% z&Yi^>xVt!KR0Li3>Vm%|6ho6ITV&yaXbO?*$To94;+aJ=%#&Hf!s3G< zNdLihq!#7Aljhs+CQjvgIWYlsxg)$!Dbj@vOejFOAyy#8 z0w##Wz0uo7ObJ>eRsYE(+fr&JBpkP_7OlLO06MpJpQyf5|K)jQYvS=1} zIzZfr0#*30BKZhu79e4&rkJN#QHyh80urbhhrIFIXps_~xqw8H``bYPAgCi{y08HW zB~@piT=SfmzyuQgtGYA8U&9b9)CVJ^U|9{gPmD~uBR7~8&i@=OnQ)Y{UDyDHl4h(J zQlFA8Y+yn`YP@14(mY^x7%=rBQlbl+yBUYK0ufk<7m~gT1dOFPCFaR`xt=n_dj!;Lj`w66lr(Ot7vYKdKz1 zQH1YgF=+vGmOV^>s)E6m1j+G19VU#r*84E?y%WiDCJqwCUd9nG$(c6HdSq`$aN#n| z%DJWw5(pCB3z9?=hKbPl18QL+G*IM%OJz-(QMn?2XUh<63QF+-0qU?`Injm-qUKcA zNb9E)^se6znWk*+YWm@9y}=G}7}qVDFiZqhRc<{}LgM?|PIkYlkhVSoYK^Z&TF#db zW=0aWB-TL%PfTg~n3n42oSC+Wrox(gl-uZdJ^!;bS<}bhS`vzD-bx?>yDBDAOZ~yy zF1qCx9CqNq^L;J;G*)PTNd1`e@htl1nUWWYKijOl2o9N;4(a=G1DSD0sbI5CKkJ=& z`YCvq90h=P%XPb)(^36%bdcr?n3hX6$30ge4T8?uC0hlU$hv8cU+upkb+h4HC^lHd z)twZnPTAM=aRPx*%i?4@vH**vuRlp^02tW7#50wD6NuB?FM@Ft0PIj6+ajey`HDPB zoJ0=gEAk+5fH}EAvYctp1$lD)ZOYy^UcwX<-cd%?wrqZ6C$dh~O3(NErYth~|FH~8 zno~GXDv;%{!a?Sz;3cT~F~sz|ZRg9C&@V^MsH49Pm?^!mN6y51A?PYM>=KzbLvsNM zlp>}ulLsDO7f6|Cs;+6q?mWd$qJ&#hoZj=ygkx`Gw0U`3=9SFj>t(kobzlibe+R@AcFS?aXE zLV$d{LKt#5>?9{Inldk(^7FbA`QCQd--WB?jz9l~To*zfH|=@Ypo^iLSK&dU+D=<_ z*5HUse*HEqR`#R($Z(=lA6**a_?0X!W>KutGTVy66HkoO-AmlY<=Vpa1K`WNahHyI z$CqZ5)MYIst;=)SIBWL$<+*e7>=(B?q&;#PhE8lo`*jpEUjD9huv13lG2hm6ah-k| zv*HV}lWI3(<*w<(u(x@k!?W~8q$vCo~h)CQfxZb!FrwAi&t(PvEm5EZs3~s%16A3a8}@OC3qWTjg9J9Er5dMb)M`M|${PKuC4#?-B`@i=r5Fh?n8sn;5|4 zB}ipE-olkFrnVN_E60US>2Ke^{z0mV%Yk<_Q;3*)g)DDoioPGygwv#=uiUjWvscoD z(~gB#_YJ6teh~F;6#bqT_wN!_GUlkMUq%Q-UhQ&%$n9#Z#eMzK`Zl_(=`wv60t}uf zP@1GJw7MBGw)H2o=!fZ26BOGG-`asT6Y*`wO)2tBcK#hRJggeVZ}z@j1F@78Cvw?LkpQn8d-@0KoLpBa=})D zP)E`*U96QJl#&@U2U-b)-D2(Xqngn2gQ1-y;kj6!deBhRK^SecaG(t zFlL;F5O{6@Dh^PNbsH|*>_Osc(@w566yG1BEEEQ3WU^S2@}scxuq*XQcs)`yJBWOF zncaV>NSY5l6Um80`}R4jYu;%vU?-wSxR&h2Za;XIR0Mg?vWZiVV@HA_;RRZPOX`qT z6InQqvtv44dKqS8=^Bj*Bfm;^W*?o>9+u(OvcA{7@@U#FvRUqP*Mw7eI>etNU#PSj z0>?+swa@o_A(PF84oJ@pfFMZERG|#gn!qHxuqP!2j^~3+MWN|bQ2Gh#S`iQtAnG(jRMaOOb9khO+UUebckXQnLQ_L1($1mjAOySuQb34KX1KHu za;Csk4}vorF1>?*Att4R=q!a!;~+c30a7>C&3N66t;E*N_!Dl%sI-b?=8fT77@&5= z25IM7L9ju9#6wfLuYQj51cGm93ZkA6DqYkCe@jN4#`_AJH&3p=Zv=GSR_>PSmXfWi z+l#Y(yadZKrM(?vS|$$<6Yq&v0;r+}*@a~8AD*Zi+q{Z4BBTM_f#E3Ycl3A#ZwX!` z@~|oDFCh)1>%24sw10V#dq|b3p$QIDaaardXyOrR93TYoxXlcV8V(Qq;n2dVLVy5- z@Cz$1qOs4+1)+iyS+@`*a8g5h!Gt*w;x}D#9t7o^z?}v|_NJMQ5HxQDC)`G7(M(R|B4!9gZ2|=c zL(#r40wBUVv+p3t**N$D6k;~9dl!h3P3+u-A!E}rMWJC6R-6Xjfk z5RI4K3_RvlOoc88vG@`4&}>z}2~U}8>DTnxHQRj6Hopt=HQO94_geZj=em}DVVSO_ zU&MUZ(yw(hUN_@^+-CgAOTWCRxt@*PMAMFx3rb{%AmqkongdaBsn{fL4H*5FhEdYi z08nl(3_J+8cX0s2&}`#)z!7Xo_$2Kw08(x0${Q;x?S-)kk=8`XheR8oOx@_SAcg8i zolV`leuUY8UU8tCd8`&AH8+y%-KsH-RGo;ip!wj*zy&)!AqcTll}8X%*bA>XJ5M5JiHVArafHv|Wwg!-NOzAK+WJhL&X&@iwU}RlT z`rIl%@@galH`*#t)mgI?{0OU&8~~`Q%V?HB@spv}`w&y(PwY^X)HqIFWYkFV05sG{ z!vl_h8X3R#qn@VTsT^T$7`6Z+ydBqB-jFu@q9w5RoGK3q;xQ-6L&AE=Y1jfi<5cNE zU~f1*LzoAg%DBOKy_YsVAmQeibWBKyIf6bitQ>hNfrpaG7y-h^5$6?n$T)%tG)x?M zra*;;qt7p%aPYd3ts7Z-hU-T5!44d%B|VNmx&FQp^|^1h!w+WM3Rl3x*-LH726=F( zXj`4eaUbV`LrtQA<%1E6)jNeAeZZGSxEy%Cuf?B6FZ0@&X7Ba5arK&h2I6u^nDn1C zo=tdZB*p&nWcqw1`}0uV4uWfVo^#RejeDoNO}7gxFNN7+@vf>|{kT=Qv$XT}if)$g z?f$?p>W-o+&}wkk8AqBb?%LvL;_iCln$k#@ zCuoEN!_^D%!ESY@2QgdS>2M~iJAK{A z){Si4$gEsRHnLCPPPc6uIA?_d>z7))H(!x;p=E@$zok|4w?Pf)(BzFN{SGHt$NRA& zkm_jKt?n2=i}f?BI|lOIa_$&f^Zv8=V-omHR?o(IAY2cG2!`u{@Ppmz*$85`dN$xp zR?o(|k*yoqx{+DAl5Aw3z_Vcog~fRg&P@$>Nj#Z8Pclq8HXC7Qj)^h#hd_$DXSrc* zEicQwde$j}wLT5$#&T4};wU+)Uy+2&RjwBz=Hks;DvoMaBqc}HDz0Ic>Qt;a<|b9k z!(2?xtGa_IDlI~vmgKsCHfe%lO4ybI(oWE~ zsXWNcS~oqoTo&2%%j>51IhEPco8FV@^Q~o`vEAG?=JxOLgE0EQy*g#Vy?{gqLp*gb zLP6e)EvrOng-#7tn4|<>c~^5Tv5_`rROBGM00kCp5k5?TZ&Raf;lmXh-RX9O~xU}{4Y=7HBm z6OpZnT0wnoPVMnZxnaFmb8O3o8LkO1o~rAE=$cvuxU2}NA#?{`$sr{rxvYxox#VX~ zh6vJePuKrHF_TP&Tkzv&Q5XDe!u7cnTb{q?`TO&A zbJRhxY*an-88+wgM$hw6HS_#;!MAKIuSVorIZMlCn8Wiwv(kExg=|e;j>cOXo-;Yg zRaV567SU+!nYv1Z-`;cK-#RMRwg%~jX3*EJ^0F{&8>Ogp9?0{z|M~j$*~?=uib7v_ z@H})Qzv|@YWBNqaY_opPI?d|&oB08$?c@-#(<9S~@udFy`|1AygvM}0YheNaO?%YN literal 9711 zcmVhJd10o>_Q{cmVsH;5T|GoeysYDTv1dyOqZFYpiDv8WYyb{RQhkrf&@WZBm zYv|@5KWtc4(AUk6@~b1^gZ%Da;t%mB;f!>&yL=(OdM2EdYY(lo5vPa0jkRitFOq^cNiF0yAO?a5SD@hI&YlB>Qwx8}$9-(T34 zRMU54&(pSJbtT{V_4(8Dr_JTZKW81*v~?p|z$POjzW=wL3D!L+n_KGVp)N>IuP9$h zMfSAnZ(lPoQon5n#*5k4_LtP}b<6fl42$;HD_LLY+qb$cHnV5`e6sz0)hI=dHV0bL zmhg;e#kI1#C5AvkPLF6Jr-$Q+yiYk#I#0UK*OMvd%yTx8qG)NSn=yWEB3crzDOnBW zmbR`5$?IybnT<`lGyB8wUD3^Y&%p@}|6(ykN+xd5Xi)#DUYvs1|)h&sl zE(r^md^cqFvH(_U$j%YdvZAG-#%*2Mn_a z>j7guV5|p>^?>o+959}ApC`_ny6TBob+^M}b>LuRHiPWS19gFD$=| z@O|B~{!rqomsg?Hot?R@MVFHb=;jfPWz}9^sSz%zIP3L8#IWl~4$3Q#;50@_iQRrv zSa4vK_>h?iAB&^pAwQ8*ZlPbu=<)KIe)mi!ZqI=s-SW6a4whS8S4BKL3nM zvMVTcPDdn*1I0PlkX}1s{D=f;anj}o)(dT;-v!)aXhcyj<#pMR>J61tZm6w9=nrs@s`rcOzHbH!*NQ9zjmD{%2r2 zb@P+-x3#x%1o2M*z0+ueJG~c7uRKsv&^Fs~vX2iLAVjl-WpD7o`O*1(kkJ!;63g=Q z7vCfTcxS??+dMVlQ~_R_2;$2vnU;1nKZ>4@bBYQwOqXTBTS9~*Np<>kM=xQD zp*6(}J%S9hifoR_Be3);XO;BX#SYF3v-bx~;RrK>$Imtk;EN}rp%*4Af-R4i-Ap)gOwT_BIw4;6QCEkh6K;kz= zQt633py^&*mWcThN3Gk4V;iutoEz_s%(Au|ha_G&yH@MGooVt{nhz+OcrO&s-oDJM zQP5j&l0}GQN^vHhMy?Ran<0za z>U2cq43!y@Y7(I>T>2_A;GD{E$HhiQD+&~Lbe5ZlLXw`ymO0{K6ZwMDlWgwmcV%`mCf3I5aOE)#1L9=W`fX1!3w0K`%)~H1RhLj_UEb02&900(d%9^&irmg-((pGQdeEMUfN2G9sbIMSb5Kg2IVV8gX^69sr z)y&mpI=ysusvB-_XLHvLaqcjo6iRz_G)y{!geM-VY(AYsj7gY|{lL0|I4mb^iY(OU z`dB&+0|43vE)$`)hKi-L*=0;j`oc4nJz@q7^WYS0PE~cEk)1x^FyRk(b4!w2y3=Ql z1k!AVEoXTqIw4U+^@>40v`6Ia>YccZ0aWuu#2I~fUj7wvi+>}4a7SAvmI4es$IrAl z2i6k}<|#tFZUXBjux4Ata!UyIsZS?2$v=`K7Ge z4Q0<9sAdQ`-VcV#f{6f~P!~ccjR~JxU$=Zh<|FNhOs){`eE#(5Q=}m$6yi%rlUhoQ z$uW@4!~~~yc_3xrLEU!xF{Jk!%oSiRjPwdHr(RkC=2n0?R~@qGdK3>lMlsI=fZZYV zL`8e?P|z++_m&KS#db1N9)=Mh&xd&MCNEA(Yv<^`aopn+1SPQ}CF8h3BMFCxO7MlY z6~!fqzq4`D@AM^M;Y87sfuloiUNaIWy3@EK&zg|clkW4x80~v6>-<#M6tcp$bzxjz zch+U{9BuaBPKfKv-7gG+mA$QX0e)WOikFi1hZ>orC9fl#s;ZT3)A_r8yj>0>w8pv~ z9(+BdnQdbs9LUtd0BlYNdh)`yq`IxPuzMvr)fe-+@3K-=I962Opuv#0G=GCiYngL4e%RmrOk!KGNkWP#K{KFKO14 zppJ6mDB!$41EP&eQ_%}DzzA4jZkLjCb7-t|yip0wi+g$6(JL{?k`qUMbSb`l=?!weYKudN2r)XQ)~G@jQ2r{Ss_%7ok0=|r4quWFmI@YgeE z`rCwwyv~xccpkJOTMj!2hS7h`scJ!&ai)z5&G_i9%XP8Y5#G_PsXHd?i_%$%yS4`H z=V7PvvK)Fj90||;J#CK(Pq&=qR3GfXp4G^ z_xgJ>W-Q(jYxLygWZ75p2urrLs*CyCr{BK%S|Y`eC$&UQrb4>)E{_CeY(OOXa?WL< zliQXhbP*|5l0wS%^eK^h`h(naZR7!`H1*t0E)_#Un!mgjnMWehMXF203z1ztYaKy< z;T;W7#2kt-4O56SAds`=SMzVyQKtE!V!|e02il)$!hv>h?uIRNDEl%v+ zq2@&<{y*@z(tOeu(mCx2N`OrOM!9Qi#yS2U33nd~-5v|Y24&k(wRLISCs6-bKwO8m zrPZ|j-~;dGQH(w*_7k6=D&MZeGAovl?YWr}b z*tD#Moa=o+vh0?^U#Gk6^V_+Z=&z)`NNBu{_L_#ixZGZw(lneIfgt?UuBi^mGJv-v zBo{B0 zRt0WE1#$PB%z@eIuPlcUP|py~X1AM89(dHu|2v<=RwZl@VY48<(TaVBKzhYKyS%ny zpZ&06pUuzd>kj=9_n~!%Hi)p~4*i{={;fD`i;HrN$Lux zMt?~24O?38c<>`8&Ybno>ZMwIgyWQ@*KmZX5Wt!>j*WHAKJMKL(i^P_I4yR)f~w-h z(`N^ozjT9ckM|53S!+#wO+RVAb6KB_u&pf31 z@i!gfoM2KNJ;dtH=gtVnqSsck`L#$A$xP{%Uh(0AW|!-;LiVwC0`&wRHH-0;P-XE7ijRd-#&XghI;Irmr+NVt)CO zV#XXQ4_AX>9Gn(RS$x4Ys}JNeHo>32e8ndD<+Evr4=Hm099(PA#gKyfW#?!<}AQRo&f*CHM#hno?P z5y<5@VdsErvG4-BA1C~xn&)&y;<0ipd^|?z{b%^a4`Ijdwwy>5s5gB44Hn;qdJd{# z9PZ2s&+vuec5Nata=AGtY@#M90E7{T5#0TmfTzRFH{lpQYRhRBu#0rUGrVO$cc~9I z6lLacp-$w%QZ)eeHsSj~n-xxleeDET~uReHJP( zd)cGu4r9N*u-qT)M+mhpIX++Wp{j`S3V`J-(Z`ECH$(m15uVy5SsAcD%&0l|vYDDI*ziT)A=uK`~%HW5g#7@L>ZR*cObz8j3q zM?m)rO4!#5ZWUg`eB`7DV4u)3>}y_Kje|Y!K#O-%7>#c#L;6VjOE+whDD`!dscVd| zL^D}3PKXi{d$$N0G1N;`P{ajUlSq>I3TWTfgK=$vYD;W=2b0Snn|k^C1$9ODY(Lb+ z47xqE<}toGyY_w3eZIB8*vw#E!~~<&k+pSI8j98YXf+2~(pH?G5bWK%VddQtG1QKx zg-!I@&ckoGvuAMz?k?6D6+zcNyTS=Sx^6PeK=s?B`6cw}6K>ohKf@#{LZKnJEJm!U zLp%?tq}}U%lk1`f5MCGky6Bf&^ak-CCI%o&`EPAP`vJ7dY1h|nkPE+{jr%g<@vah` zfpm`7ZeG}yROFd(!R+}W(ep?iNe_gJ!JxXRD(GuMF%)^UMHViIq7b=`Of$zJo?bM~ zJefr-3_b{i^dH}jRHEE>Qk_IFOsD{(MQrT!7z&gEx`W!ciBq|rPfS2vZVB&WigaNE z6ABP+h!se(fC(eP3EZf-vGSa_fJBnnT;`%sERoJ!U?L4IC|DLN)0qwMgr9oYBnOZv z5btE9EQ-aQ4iGn@Ko$NAOFlx11xT2(F6Jp#*y5a+fCQ?CKCj(2TBJm0E+CP_{&r*l z2D(VbZkB+xAn zm|$&vV_5w1j+GE9VU#t*84c~y(7tTCJqwCT*eVE$(c6H zdL*wSxG+z%a<1uv1cHS3f+SIdVItJ-fLfRcH59qvQdyH`RIbS1*)sGt1%>#40CiZe zoM^)Zk#j1mh4s@ieAn;0Oj9;@HU0c-z1|FP7}qU|FiZqlRc<^|LgM?oR&>9tkTyO7 zYK^N!O3vqZW?B-aB-UOAPjqSdJ}p(xSu<@tnhI;~D7V(}dj4l@)db6aS}O{ugHVM0p{ce$#SMW7v#zIw=R2MdkIrect;vlJ6?awPGqgDrJnB& zH7_#p|A7Z3%}E?68OX9&;UM#qQ4Xqp3^6@#+xdJY^z+Es>*#L+W=b#Y$QgMr1YPCk zyF}*AP+UL)rHCoa-VL=wc}A zRd~>-w$)ahH8|puU$+hGEBh$FXE@QSk1h?d{E8MAGbmPJneABNh$qJB?j>&HvTfn& z0q|wsewU6q#}{XmdOzOP%>A5328uqHX0rHIj2roSY?$aY(c-J58_P(EyBXUcLG zzibec78~PViP;Y0&G2b-yn}g9NqLXHzJ%F*INQ|%h;2J`W|RgkQ_{_C`yPI*jbcdo4s&}0-uLR? z21h#-F&T)-1Qlo3;j_-0!Bz&Ekg;6Z(m!38;p?>A@j!M^JL2axA0D0@fuF~&(S(XH0Q)=HKdimaA(|d0#F$$8-rDzV%Y6`9~tWL z2=@pxqV6&=Iqxb0P{E}WAo(~rp0NM!(ps#%m%i+%!i4i;ZSW4xp7+(T;(YM){T8%T zDs|&J%Ow;*jHt1e&&VEi5drPdbOJPow~pn6XZ3Dp#<5#imX7`5-79#r;{KYIwf!Kv z0p_oq$%5AtXF(Et2n0s4re%Ci(TEmKpd+$-1OJNC*WgA$iU?Um{eYFpR)L}ynp>bKHr1s z&tJaYgYB2kkYh74+peJ0%awm8?ZYgWPx#Ad<5aSeLOgv)+!kxrGJgBTRCfKXTF8NZ zszq6V5o#Rzu8#V=E*nxotZ-buu+)(>yk*XXha(Y}IV+nq=STeLSl>YkV({HGnnD?};h(yFxD`a^ylXcyYCY&Y| zeQB?qnZ1%GoHiC--8Z1dx}H`0fprI3+`mg?$(W_4ei#qR5uR=3e* zb(^WX5Mb~;fzl*(p;dLCk)1o4MLmp{njqP3>COzanTT&QY&pv_(fN1G@VpWtAKK#` zj$H2o$Q+T%e8r|#2DcmELjYhL0g-t}TP9Qps5D{18pmaGPFlX>1UzNFt_O;=GOP!R zInnh%VIcaw94Jm6lw|5P%&Is1;#-x|c74`S3aV=|A688C_2?j%pP@xRF0_!Dq>+_4 z02Gl_EEjAg2z4Y4)5TipKq;9qv!Ing*e%vBH>wFPKN#9c5}u3osRJD)fEkL8HiD5% zgwX+#*r;-nRPmRjeGvym4CUW*!~_MD5h?3}?U)UpbRw01?^#e|CX~&Vqmi=xI?W07 zpfY9%xN|HAi7{g}gurtPP;ubpSU2Ia%^pOqHf_aP^WyuB@j_y7Mkb3TDL)E358G0Y znA0PLvxA72m)ZT7iln*FGd($>Xx}a;RYh9`25d$22-lJw+aG$zl8PYjST=F$acEIc zB%DA?a7iuFDk2N#akfOIOE1H0;I`42FmkJ8YxdD8?O_>i4e2`7E03n_BAaDDcTG5j zr$hWc(1l97A#i;7T)X^07c$vw=z#d#00@HkOeM-7t_h5?3p-L$;CMcssVF#|3Q9lm zI{EuWiLi4Up^+iv+{Tb-5UvOw5g_a|L$9b$I_B_554O?iAKkgPAqY(k!ALuYrh_2x z3P=G#I+^*UeV{W1rg|Wp+5FNw5E%NTbReCjywf<)&TxR#jde3#H)A8Qbu<2eo3U3~ zMKbe7^DPWeyCl7^bB!R_AVA{2F6~!8M|lFlH#7xNPY9LHs)D{IqmJW!h0U8M+ut_= zT5l`2OLa@hRMqXo*&Z*!GE8Y_$LN;H!^6Zo;*|iZUW05yGItM8)QxRUMH>;)0PetW zl=VA$yn?p`ClYzs6m^%7hS9ZN8UotCoX9<>%2d+?3#vG*1#UF)h%^omf_U6!21X5s z2mbuf!m&bt0EF-hD<`6{%ghC#f+JbC5F~I^Lwdo4IS}GEUUD7;8=Xm~IFY*}bfOEbJA$+mp*v31LQuIQIlz#(zx%tm(a0#UM&ox3n(Y+R;ZXxNC=tlkLN z$o^a?>NPIgG`X~}(zXxaEruakW9cj)jOm3P08Z5}fCWRWMqG4olxjq#P-JS{Nq-+q zC~pMnMEsXWC%BJh^G26OY_TVzv=3)=#R^1^PNwQYE(&!4Z&=YVP2u$4JE_ zAf#g)Uf$@&NIBOaMB}A51BZDPQ=tn&EPjMMG*cCD!c*p2`Zaxa%{E`N&F{i|%{B+i zy_SB>xvr&OzD(EBFJ!)J>DRg$ubc5-ZZrPirC(0eT*t<)qiIIU1tqdW5OQNP&4H-6 zSZorv28@1-!zgKM04TQ?1`dSVyEuShXtr@Y;0U%Te3JGT0I4>0<&6=Q_QF_)NUNjd zLZS^&rgrpMkV3Vi&Zh2NH^OW{uUOE{99D~#njJ~@Zq=Ats#e5U(0p)Y;DVi=5QJE) z$|DFW?1fhxavpA~yx%#_l&)Y$eT~z=FIJ@22M$=e5M5JiHVArafHra=wg!;2OzAK+ zWJYF%X&@KoU}RlT`rIlv@@galJK8Ex)fuxC+z6|Y8~~`Q%V?HB@spv}yAV_3PwY^X z)HqH~WYkFV05sG{!vl_h8X3QKqn@VTsVre`7`A{%css7MoFQ%aMN44sIaVGL#AA+> zhve%e$6*WfjANw-fxY4Q3}GH{EaS${>%Fw`0SPz9q+>!t%n|fG!^)AT5_l*XjS(Px z9C2QOhm0ebK*Pk5X9};-aP;}b5e{BAvUMX%&v4zy-r0dewW!DOC)?jQqHel+*MB40 zt#AcAoW0a=(u;#biQDQlj{7(loYy4kN#1L*Se;Yo(Fc5Kgv*}hhl>5K^)j!WsrOzF zTU)QGXCN+zgh~HNKYI>XxD^&}y*P8AqBb_S)iT;`VytSr&A?>rZCdQ3EF0)1@9OZQ#q7oBSSdCrW_R(?g?>qr>|a!)eGUlaP>mGvs>Ni zLCjWnI-JSsPG2{&bt79hG9y=%jqC%s(@mR(pR+=O^$V@t>95GT(9%NM-_ok+YcB`1 zuk%`$e&;7yOS_>&kZLsTR(A}b#p;>W9RvApId=@MdH+%TF$w%8t7l_95UvM84~FZ3 z@SWZ2*$85`dN$xpR?o(|k*yoqx{(>VqHJUzz_Vcnh4u3woSPc%l6ca6o@AJ`WH!Rq z920Ho4}ug`M{>>DT3(iU^{iC}V||*Z8_QA^>qp5_{pv|bU*&osqA%XOrDCaO^`vB} zTJ>v~p*q!99DNn4<@|I@!BlPP>t=LUmHJUMR*!m;u~dorQ36zh`Vq8LeMYN~Zni1e zbM_9T+G`JwPWP_0G&an|WWB08h@#XY_-RSD3rHRT`EV9TeXUj>Z(;%t@0ge71Q$bHwR2h&X1C5ncYJ*J=8q$8dbxR@f(T| z3;H%`0%J;;mV>9Apl(xnkeRh^dT_Z6vZR^O|ycs)EGH!%U4OW<_1Yc-dQCyPA>h4jAo?gNuXumfPdwm24 zWSmOnIPi+LWXm*xNW0&U^mJb%_A0us|DTu%CjBk=@rG3eeVuUKaJHlQTh3>J<v;1vy-5hyPDb-rjTJ-a1OsG#cspx>wh(QeJ4b4bDoH2eSG6 z_fMZTP96tV6zal*P2Udms*^Wk`b1XmvhF}y#cK2U`hgSf{w1Gh^UNxhKg=Ct@zW>y zZ$c|?sp{{n7)ud0pTGX??_YoU^7ZR)pMLpOFP6};Vq%cFV8eLl=;lBFPw;*D<3IlQ xm;YkLu+R3a`=9^*_vN*Y8Q`mM$fvOS;t|q`QVjVx&_#g`q(jB}clYyGvl`ZfR*@2Plrmm)()-DeA2nQK@36z5I0juY1 zWOp_Dj%p2fl7|cNmV%ZXqAf=Kx7AUn5E_Gi=ov(rksdzHyMm z$Uj45I7LpqovrTtVSRGKY2eG-aH`>c?sos9m7EL4^dL_=(Ldf3isuPl1683{hm+$C z3qeU>wlqu50ze;j$s7D(=a%oP_h_U^TU)+HsdV(7*oq!|WwCwE<;1TwrC6d*kxqt_G;+74`0S6mkvNo zYM(-Y9J462dP@j9LoYaO_uyUTRXdYkqJ)ao$K&CyR_fctm$n?D8PJq^)8v$l*sjC6 zmNmd$m2|vygF*SVv}%ysm;fNEISMtp`fS`Pk?jRaE$)1oLsshI8}G>aY7>%qJ5FZ- z?^m16^HW^Q_H;WBjU)Zv&*guJ`BPx*OL!H!3|i7fh`Xa($i%0bQW;n#LO$2bu`pP; z))AXWz59i9^e0RQwiCuR|JHFda^B`^{V_x3OHjQzE7DpOb$jpOK_L z#nD(bdHFQ4bw13@zRO&V+;CTIerxO3XkX62DFB@CL8ErJOroW|;)frS2XV-!V2df~ zcpg?mOQ&n*sLApCH5H!nWYz);s_2hO#bx=m;0ux6$=11zT_wYxEngNA&JIS{SGL{6 zvQ{`U+o{s7%r-M0tX6*Ptgu62-@FoueLs;aX5^1*48wM8h{d=XHgzU{_AP5|rH0PmsI`;%UmaPv zC)M}0QY!(s20I-;R$rBo$9ivF$vQ?_u4$FXTadt7s~eo`o6-+-TJk!7Y`fiwi)pVa z??jU)OU}Q3r!(oPG3P$;D#Xbs2B(tdD|SgKndaxl^PuVJ!+eRvipn9+m-gppFAt@^ z(eskuZk<1fKV%t%d6laRZhTPZtIu&V)$~!57O$P)CBwP5W9)37g)K!7(TwAH#}j*65^8_h|g1A1y#UpqJ$-;3RGogW&<2~?2yd&3S( zXm_YRhp2yeC8+t7AD^-|D&Q<#cDRj~?we_-58dwS&93^}RInVoNE7eLHwW^upT16L zryI^zvo3A7Hm6pykB{aOsd~S@C1`+K^wi;t1r5!!<(sIb1*itw{s!b7Dt~mF-}%sG zyL$yg$Sa&7&Eo90X_qC>hQph?igRY@maEkkTnn5c$8XvWmuM&Fn}ss()pqZ94O7oH zIrBQi)2dCjOOh%;qb4Hce4*zZTkkl2zxLY6X66SSI)#cmWQF>evrCkd1W8^C6y{tt zd%Fjn$pH^ihS` zkJebqA~kzSa81l^fqJk-`U^^rQW!t4vyFtosI1@s0J<7ZuxINA#63Eiz_yFfs%zuA z^Aj$Y%qUM&w*iEilAMOmL)$7Fr(^JF4VY_$+G%39%iN>v9a6B~MdG+leoJ=wz_@H( zRKhZ%XrF9jb7@Z=AE~!kR&MDw9W!P1TPKaUnsxkGTB&HTqIfzl$y!`SROG0s8QQvZ z>AT`Ec}jiB%!gwvx&cf@i&o9M8N75iXK|9s`zn*KQajo%zAwM~^HW)|-p=ln>M-^N z2&Ca>hoWF*ll1K*d+{db;yQr~`rdIyd>)su639~#sW-8<`1N_`p3a>0D>o zt9S9Ekp`oY^Dk`&bid`~4w_Lm5eu;&-v*X3+sri*>oYmY-rvpMaek|+zM3CT-_mP7 zkeUs!UW|@0SUh;EbK%!K(VryIJ>sf4ZR1{d;Z=|^KXKE%ef&+suo3v7L}AWebSAF4 zyWDNP-%oUw-|FVTNc`AUpou(hq!&=Ma^NY*T@qg znMic_1YGHxN}#5$<~6sPw#^}K&t6lQ+?zOtb7vX(E<>9|p%3SaA}04o)BAd|D_l6+ z^e;vcVE)CI=L)y)P*PXXU@VVoAixl}k)dd*c-*w`MQ@pKVEg(<)m^vC(!v~2NwRR# z+AQR4eQD-!d6&`}rFH4OL`6E0k@T2g&k6$|VsMUt2nAzKybOhHe5TE=LOt5!A{yO3 zGLl%^35VRqa{b@VBmjevxuC9>8zCcM>m#hRniJyx>|EdFgT&pg;O^ahyU@AD>5Ied z&)p!Es#4x&zrDO+c^}$0>~njUOMje#3hPxR^m>xIDkS77eBX%#J^pxw@=p@}{Q&nw z;HzPdOLEhDiLdD-+q6+Vkqe`UF#6gVS$5(CWjcz&vk{O?3lXk2o9nkhsnRHOTv_GP zU*El0(!Wia0jtpdW*;?p9ub}q|CDtOoHwZ%@)M}tKIql3kPi4*EeS1%7n6YcPr_rl z8rsi7*aoC#?Byf=6V5sNzXDn{ChoW$3cWh#xYRSfKl=B8Dy8IqJ!8OIc^fApOqbip zsELu`@BfN$joyWVa&8IcNRT*H8PygJ!h{zMKHH$QZ5$w z3g2=ED-b?f`KQK`i6TQCsM1?HBqxrN;F)BWdg4HL$6S}4iMnVX;(N&e)1fu%(xq53g7qY&ExL#q}wnE!%oBJ)v(VG zX1Cop|8rEt+k)_@g0F7E{w*pzI>+cZ2%VV}eqM|jLEKkujo#YIIXR`Ar@FF)Et876 zTW(S;DLQoqk&iN#6o&5LTmI3YVMIM2y6Fd2IZ|xcbLa>u(&Ko~orU2BSJ7{ho=9Vg z)gXjaJkt*dfAnuWg!IP&@HE|NAwt`IQjJ%gvxv8kIS1dH-gmxjDZFFnHfFIX>SSkC zrozMY-f7rTdkbLfmB$Z-RZ#YTkYt0Oe`dHGB*Ik3*gvfL$8nksnm};cV}6dcosZg#9+t`4doU!Mt>Y{x_ksQ zH>tSWPdLJke_clYe_|^b*N|&?^M#0p*yW8E1WCP%B`-%xky(kll9g;dLs>(Jt~XUq zxMl6K77?x>f9q70=TO{7i8eoVc#q$8pc&6e_Gl|El*jSTRN|c#Tt_zamiO-Mw+NB} zSdZ8QfsaMYk00w_+P>65<$gFd_46I_)lO}1W-bW1LZd(Vdg0RH6Db@Q6Z})``IH(V z8PQV~I+XYxGY&=bWXR9FErzP^i9dvL`SPe_EHndT1Fj?s288@Ysc(qh`N#<6^i8sh z4D#lyK}%&=0TevJHF#a|cKq5re$S70$?xw7XC2m_C-T+TaI!S)atD;-sPotN3d|X@lm)JFCkLEMMbnqW1U_r**aK zdAF37MZNp7x|l?L8MT#I{_)Pu8hkaP7aPJ7bmD&=NQQ3OpvFrn#-VP@kF9V_T*g+*_03q75&H! zO2AZR{kMnMt=tF*!x>VBW>s^7yt}Jsu-ks8 z&)_L}X^%0)PAu>)f=D6bqRn%b{4wfIIaDU%|9Y5`6_t}#G|Enpk3GyS-|(RAl@p(u z7EMdoL+|YI90&`KbBoQgo3Mo7g&HRrhdWjTk_+Qm$>`N+L^SX{VCYy_P8Jk54 zdE`GP@O}|3T#KjH>vDV5fQvp`sc=$9{n3#+jRAf3B(dA=RU&#AVcg`HGuL$g z>qBS03%SeL4`Ob|<;V47(Vd=pzxep~Fy3iwi<77gMtNrmXmk~^uNqkrcoXJq1)VH# zp+0=Bd_Unz%WcO%9iy=G{EolY{+B!NmC)}EJ8PTQw|+L_KR1fYw0@hXCNS#H-Lx(< zeH6a>gyBa4)uX2NMi`kHL;|^BEiBnEBG+7@jK2tG{b`aSOBXP_=U_Rxwd_cZgNQ|t zc)F`4fS()e8pjM4pGlXtQI9sVf#f`g5~WqKI8;W&;g6!cTr{o}NGdrz5+*Com72U= zTmw>Ws_rGF|28uZ7GxHQW>+vQVy}m)9%?=_(kEtKPxNM^IqAm^gxq70vxcs6p@1>5 z-5&4nu47W|c+OM^{8K^#5Rx_A>;YalulDRv4TK;8P~p>v$*dhdm=t!Xx-VX4BteV@ z9qOsW5d=#7UBOM)@!fiUfFq${WcbAMr`TGyR)wzwXAO~EVJfnMfk68&RT)4e7oShU zew>nGEPnJIEv$7IsifQ$fX}2I>;ZZR)d-g2xKI(oC=rS`zzoG1TOa!U3&t}HpS8_> zJ2UFTKGQOcXruCah`xz|H;_H5IVhYu&8ux=ugr`(TE{5($&CNL{!5yu%CoWX-!c8J zp+>L#P!_x&VCJFP52Xjc`?q@r9xQq2wzSX{--401($R-~I2x7jL}sr zBm_@_$-)RvcF7D@1j1HO4-X%G`p@c*^>#{*X z+?oYO@tVB0!CFf%yN+IzUaj3gkXzU9S%RFaqQSbB~ z3i`EVrj%33d54T$TOngoC~H!fMG8x@w<;_Ok!$?Z`))*XMF#kKth&p9n_`)nVssp-kmgG7;V)jT6H^uakw<0_L_JJrjF9r@i?Lxu z)gdL7S1m!73eSuXxw%Y?I$97c?J5vWNFFicSNxbe!Q_N3p2{{m>DqKz^sxyoI-P4* z#`Ljqy+Ph@7Z3M9)wjQsKsw}@U+kQgWD~uw6t1M}Qz$j3XjnkqMoD3IW>CdW_9h7D z)ns8v|1^>ZS#Oo=*5*0Zwa#I0pT$Wh6FOlLIze`_TPx=n`>FMS7;XO^|7ZcO?oeR& zA9n7KKc$NrbT)PqNxBC#*ExzI_7gdZIjO z26`)?ZUwTjaclLVyb0H#f}8>0omk=QedR5dZ_Jk=z zo5Ga+TcQa61Qw%#b>a(ky1}n8TOLS%UE)gu=v^c))=@*3)0Ht)?2CVK4osK8+;FX7 zRx&qmQwW44|3gs!U56^0(tH9$yV)#d9ufQ85&Q*uwZS_jxGF%l$5mmqj7&ymO>z3@ zR)OBf<%!8yxiw7+@wU*w*T341OR?SD)s%-+W2F$%rX@uH;>ImO^!#}6NmCl!nm8wz zKr`fJlqO*W>&9ISI80lN9yWw+R47&|X-DZHKLQjf6_pzcj$G8J)51^x9wdlsqtx)- z19eBVe1&yp++#}`WO7%}gkNE@U$!cn-FoTnhgWv5H-8dKGfTqDh=QT(Rg6>rUe8hZ z%1K6^HE0c6Pw}0ryeUzz)m!VhFP3EMaWX%8TL1A~0^0hc&9cW!?vW{$C!Q_UmN6LO z)`H=r{yI>|y&=dYl*#W10S?mxDTe93z14fyj$U>h!EdyHY_lN%J6?-De)jXg(u;wS z=N%)_D%juVyV*aG!cI~?lnR^ z&h42B^UTKP-P6BbkYG@5*eSPZj=Bk!qXU*X5wMBDO)&P45E@`}365Ju+Zm$^4xn{j zHG>C>Y|jB1aCeQtZyZZe90=)B4P6d`kEVs(nq+&8{oSggtCt+sg(>b9>t8B1(0_cp zn#XZ#^g^Y_!TTqGQ6C2qSD2G^cAA5-AS77@@VYhVE-u>$5sI~q3PZBYut!Wqg(6u( zj5@!cV`?|)`_)8sw>~eeHtIB6+e=uc7-rL{M2eMcjqpNcs`9fMTPbT@Ub<~Q>=`}m z0iw-}1GnDQJWnF|9HAl*S!kXMCetdDGjLt1NP9K^VtiVzy+y-yQIuAQC*1>+NkU_m zq_J-7rvnTv+OIC?d+oAt4Na{~Ift`doK2*Geh>8u{n4+%4czaw(jG`_7{N^1#F?+k z549TW=rQ1DYqhTIjEI&eHjIDzoYTw>C!EzK5C2R1CAI}t|C~BTJf^7JOwm{CYMTn3 zS9;fCJ@*p?H#Q-7B7*x>HWD^O49uj_IEg{>O?kz@fpAx+TJw>ng(m><89MCq4**6O z9UyNKydXPAD!3P&vnOyD5-89~JYRE0T2Q?5*sop44pZ$1}xKq z6}BnBUOAA#3^St+)dw=N;(NtVE)z2A_DwjXDbsqBM^vnE#9l`y+*7biUo46}e#6f* zsw844Qs1d$L(=G6-O&+Uz=SAwhQ_HQ_{z(HQ9H}tPtky1$?c|k>% zk+`woyxz11Rqhj)y42r-$V0sWx>l`0y54``4T z`^Q1{Cc!`-A}6q5qP~B1Zd;4orN2BB!l;F_yBwRxJ#~oG0iPPGgAH!tO=QDyb7$rB z{LrE5AgJlAFy(V+v#KiTbM>1N`#cq54d>)G$MJ_2CY*Dow;1;Mt?%kru$QM!KL6C8 zdT{`5GH4Bo1EawwaGDg34fhuX+P4WW2LmN=P|q;t?iAdk}sHB?PmccRDC3(wINXUX(oaVwNYX} z+Uqzpdwg*+&gRAs)jHZhACBqKc449|vP)}8{nVdqB7dPg(kf*Y3p7|Pakl#-#pY`Y zEzQQy*_v|C7j&_Oi#Y2##^bExG#MQt*JWfZ_H!8`7CdbL!yK9S#F4XX%>2Xi>qXxk1ntP@N_Y}a!y%usboziArZdxcvSy@Z94zqr%5OrEtn6HWZTCeyl&G%5^+g0~U!#z)j0qR!u3<;hy*f zH{cX1HgVIH5dYY)U_qMdPyr;{sPq1+(hj>}21ctUf6u!J8L@YkEIA7>)HVce0(%i zm+GiE4cP0x&EYkmHJ@#gV(%@IkgeYbfMK~&=iBa&@CgOnn$cdUqfI!=!S?qEwLV`k zrr5}0P|h(;j2>OZz3+m}vE=$GI;&&IT*`Uci6srxjb6!D0qrU(M#Pq2Roh*2LFwE5wch z4D^GU@H1a|m}#wHY%Ulj*@!-BaCLe%TWObeGqO>#ckU*+26J6M3P@Uz3nrw?v%My& zv)o8zl7@sZgu4mD*hngL@IMyzG&(aX2}DSds&8nf;{mlkM7-@Cdl)ftK9OswTzFz7 zt(bf$`q(kRZ;OdYcqtc{T2Bk(ZGkLZr!k&yDK$N4nBj61S-(B@V-JDYlfwTQ0K{9& z;JhFP?+cqXoCd0q!CuXf_ykn!^H^WN-PTZrf0t@3c`+EC{r|SR00REh^HKZ4p=ma< z_a2Y!$5XJiTE!2&dPv&XFl~0S3U8nnYWM%UwU|4{oN4?&TZ_qMilM))1-xQ1RsTn8 zq15}c_Z+JyI59V@R;H1VFzKn+>g^q+59_>*+)T|g-&B^T(={#92!0?2H&4HhHqvl$Ill#Vh~9Bpr_J=)A|kfDjKlw0%u2i zT&-~Er%A|gqS`jk;D9wB0l&7%UEs{L9&(3T-fJ%UV@rtNpKVRXy|H*(E1qtB<0|QP zkkdQC6C6s@w$2<>pq;i=$1}2rvyU&B*?4j3QGCZ+epqnSI@G?!+$FfaYFL>e)Fl~; z<`Bzpdk9A8d@EPk)NV{&hWFcg9eq1ycwDT0aUK0;wud+_7@3PIj4?fo(ct_|h;Enl zJB)ri2al9G6SD(&%OJpWVOj1C^2Gqv3?!yv@1`roNJv@L;J!x7Q<}MJZ>@O+Y*90^ zBCU#e?t)KmARUFqheT2EQRaaT#8&_rZ-y=;RjIXo2bZiVu@egPEvEYvW!lV%V~zc8 zKnRd|wJ}nSJ>~&AEC=_sQb1Fn{Mjd#paXYatXCV)GN$(*ie*@lOONixEf^u@Z#vKv4MFK7goJ@;K0=XI(9Tpq9`3G(ck{ z3JvVs){2!WDf#B7QE_7riG;q999F9@u^6|$428d&F{48r9lYSNk5oj0AB*T!j>$eB z{9qmfyQ6MT^b4|Z%g87y-Msuj<|i<~#<3)B_Q_DcKW$q8V)UE{4t!S$u_u3aN1|Az z;X7K*%L zK>QB}6h4|(1^uTKj~C#W2k)$SP1{=;dcAHLrz|<=7hxVuWgd`Et_L~Aov=F{d4BmB zi&X}7n?Zf!&kWaP-x1(v-vH$B|8pcG=WCaY&Y9f;02^4wHY1agVoRBa3nBU=MYQDp z=GH#e>8C+ZxqL8~T?wO%c%P!v{pIK7bsaBJj_F+Jd zz3fY(VwAGl~I58C;HC z6OQQ`Xp7A4wJ5OE$-mH+mSCXQ>OUx)!~de5PDju@<;vSD$0{6oNnrd08I+Dsf$`r+ zj9yp|fdY-6!vE@YKPv!^{vXVL6Yg=wX{QEiXS CHeI6t literal 9500 zcmbuEXIPV4m-iKE#{wdPfC36gCsL&aIHE-9geD!7-aDZQ0!lAZLMN!91PHx11Jb1C zP^34(h|(bh2z74oJkQKC*IYC2JM(43hn;<|wSH^;_u6~2Mp054TWi0(aLM9@#|x01 zyR*}UwHaIYX-GQP=H9_gI{)ga7fIW)2VYAZeZ0)aD#oUYQg`1xj?ZrwqE1uj*;d(a zbS6PxL51iPUV2ZcMNV;Cw7>~3@BsS{WhdovC_V2dT4>%22F>V=AzsbML5fqemSJdI&Rhmj+ve{oiG2I;Z( zM+^&dePD}|wOY!p3Z*Hj7#(<`a3d;7Ga22ZveMH0N3`Wi^GK+jxVqB~r;X*KK_aUY z>-uN5E}S~X#22>M$vQmqeS0_W#6u*{Tv+vG`QyZmt(i6*CJI>g?9A*jbmK_QdBKu+ zyz8{n+(^CgR1VHu!{X;B;!~#vBa|ohdNe!E?G$KFzUuzE|Lt4(%bV6CoGW~67N{o` zRlJzaNM-h<1=;Hu!m#tS=kV*jjg5rH(vlVa=P-2(V%vdgpKrxPePOc2Os8t(ETu{`Z!3g_g@O+$HqLEkV6>lxlIa%;6R9SBtW zwGFN06EQWpZ<_AqGu2ZdGqbYX?8lh zBXSy~3Ivlg;0|F$Q(@x+b%#GerJ0x}Ua{j{IVb0IC7)F758r@?s}wih%ft<+`cA5F7Gn$C;~B>wn%z)@=K z9q3c(Hoa=${j{jNy0Kw0H9JuK+b6$G^8%}w38M{rYS_JL+38i-zS5OELEGB7;icun zDaEDBMW+>yql~B38q$ud!5@coUS%?9NpsSP>9}j^tV{6@I<33h9$qy+YRdu7PEJ`* zeNsIcnu$Ka5l;Pxm=%?(z4Yq-ugiI>B@ar|!;~BnY@Ce?8I>TXI3bGD>Sk;hA1vE> zqa ze2c~Y`<7zaj~e~CyyR08u6^&7w5%TIjHyX)1O}b}merhM&30M`wH>awnH_!*waW_a zkFhy;gN{4J7wvXtrb7&fe)t`=a(7KR-P7YWf~EI1{ch@!WrH1P@<7VhAsqgSrF1#) z)#);8?P;6;F(%w|Zsv3aYZ|z>XwmsV!X^wK770Ie$8JJkHAL)>!HmwGXzMvT&zOvR`nYBa@vUE5Ip8hChb#$ntq`AWkF_vGpCTLWWB)*gN$%ZO2=Mz%dq zY>#zpIugS7YI-YFJv-;1ukLhl3$l5bN41RmK1TEocI>eoHuZwx*mz+}1C;`Kigk@g zE^^ILjf3AS)-R}uNZAQX*?D&MKDsSiQ}0=V^_t1%s~yQuboZ)q>Y?7-$S~~t7&Zm= z>ghjS3!Cb;oIxbQ`(d^H`0qb<_ackj$g7=_R0B`9qTx-lnA8GpcEb3Ro*`T6oJfRT za<~UiEereD`tVdYu2C3*mP@=*Q8R+`+WhvoUcPZ69Ud^eOI)iv9-Fd-8dyG)oXI`- zyt@9$`v>T`|G`bo;mE|==P8MBY+ci*fI8TxUw^__(UtbQLa?~n2Hm#$3>fur{!A%# z!Bru{=C55vzC_8PPoGa~>dZk)KWjO26TZ(SC-T7xe@QmvEzFk4Ss^xjvYZ3*zRIlk zj=b6|UYF?aOQimgyPWarB;DAGtvpG?I$zEZbGrRdzh{PC4m&7O%i(J?m#L^(7lLwAbIaU$YE0Vgcjrz&VW@Py0*+SqC5m9i zdzpGL`C?h|(beiT$qJ{vek@$IWQP%(CZ_1sK%xVRAW|7>=9Zky9Pc^#l5Qs zOXKM>W8ksT?ZqEU(+&1s!|tWc8*4MT#$HXLv%SGoac}dc)8K9!4Nnrjs;| z;>f^hrJtc#^0v7Dm%b;|54Si82Q+(2QF5kDsO(m2yuV^NS0>c<4LwMMCggkHmD}>0 zX$KsNRuRMR-)RW>xr3`El9`|h);9sFS zlEWyyvOFUiV3I62uxQEM`=p^}Y6}0A3Ncq`?|ot;`UJal%hYpkW(I#s^qfk?5Z4Ja z>90=h5bAH%Tk(N*V+6hf^Ux*dZGS3k*isw*8}WQCRs=ORa1=4O?zXo~Y_!HzAp*-H z#0&NoiL-(O73B#`E6U^|)Q}Vva-QIRm3ABXdJ0s_D=M&ks~8qcObE2~?8ASbCvG0m z`hA+u#t|JFmu@qYeihPff1<;b9?_#<$!BG;$;qiT$pU;SY{{A6PmVPm^YZfxuwQpc zX$W14gFN2x_uA5d&B=@|1umnp`EzsViM>)B(GJbH(TbOL=PE4;*xWo)MPPvsodw{` z=-)nI$MKEE>nt*X+33ts{CW5Tz;f;z=6REp%%a~SRv0qr_{r&Pc6vCSjq*BeX>5}F zik|t8@N#2iaBhe@z*2#B*yrK3mw*K~+<+Z|ZFXQN@1mdV!rGAnn03o*chA3Mf+%yk zG%(#=BwiXAyu>~`;Ff>!+=NPV|*SsEY|H zapqb^sugMWr(NfhS%gd+$dL4F_p#|M*UdvxOzXBZd9rO2qy`d0fVI?Di3xXSsQ|k$ zz2Eo*!o8)HGssto;kEc%TmJ1+6La{8>oVQ}*6ZI557MCU~A00 zv;~HCONg2H*^63&|I(DWhMjxI20q>Z$~XrOfGdD;z#1a)Si%GJH(+qPNxj|>sjA0JlU4591Euo2<$Q&?;RO9Jpu zM-UHa{u_%ni1KMHU<@AAUh6EiA>V@kJvnTt*6V+OrH*ip-kOHS$5=e#d~8lB+=H%K zOxHr{y(idHNH%i(LHi`7=_ze0f9^IF0&`En`(#UI?!RYc%a=2o20qQsg5CRcye>BS z8P z?%&fDAEPaVZ_^~j_Hew^7VNZAeIuMq@k7;(pWG0%pb~w(WR)h%h=T$ssO(9+sZD#oMmC@w31ftG(%i&{?O$N)f|t^S@t(w`8$)H z)h;9C{ISlkdOyO^Or@IvR&KmKW8`qlU;X}YQ0^TzbFfSD8;wkcyqkl4T17_mChzW4 zr|?JdM<;)Qpsk-7OwO z&?P9FMr=0~a?mt^F{DKaT1JK$b;LwX_?+fCkGdg7qnToZBDkxx1++_}X zz{$EDeB&Opyj$yP+2PI@dqX5yJJU&GQkG`#PE<*~{foN|j&~cd#y2hvT1#C2YF_d+ zugEi?yGh`*xG}GI{Zai3BRQd8dpSNJLHAthhN?N=>P*#vC^775OxrZPk!T&QMa{B8 zkWf<%by>I+177h(JGvz$`1Mf2XPHb-tnWY9oOoBK5p8#){ssr-GAS+qai_C@A;0E9 zH!?=+SM1`5YHSPSmeXP#JG}OB5%Gn`c1x)1%1zbenWBoXe}YN4hgzN?f>u( zR!mLDynp{fSAoNxEAO0Z=46wJXSC~bt9i3*S3&Dv^Oi*M&3S#b_JRW$(zYt$H~N~8 z;phnNXlvoYyAdL?RyP>(lm8G_r#f48D&_lNXMukgu#m=fR#$;66sN^ZQ1RAM#6H2* zZ^H!>d{`W!XB4mu`7bjgU3t&U|K~plM_D+fOp7b8+^&Qt)94}h;>hDJUzu2~OXf*$ zQDp9t?gc#;-rkRorc`;OK|VX%n;IM~^@l;+aMDih0z+cFzUl;!WpP@jVp-pvoi^5c z*gt-Hs0KEHi`ubg6?=?)xY3i^!+F0DE%1TI(!6OddBH&x9?&Rt{H)slONseNW;N0h zY!C+>e%FPvGb&+uv;OB#syFN7xXrIm8RcR85AS(&xl_#qh0}yHKtq0_A5=%kAGhIj z_#ID2O=m()=LY-+vl89)0t_w_qH)^o&vZr`5`>Zmk}l^JHm2z0#p70UZ&VOwT1_i5 zlapsH^0HLJqJD9}S(Sf5PJcL^FdUzJN~(T@G1WaOi_m?({+TOL4T^Ysd+yt-heJGF z^4-JFmbNF#!(2YmIuE5e+3=R)v@&9nBJ^+BoDQF=u)^bJW&#*=+^u|$Kh{;p zl4>sJTA3gK;lg%treJ6*>ReX# zA#K2(WTowo{6=J?GtCkAh8XWD_?l@8@-omxUxtt&ou-+Azca;iV|+*|+|+dJlh-rf zu>+{MQmLJtVz(}yyTC*#DKcc?TJ+`lOx@ydZX$coCt&9F8Pvz6-AtnbYS;!`GsJGVwWOOB$JX26xuTOwi^@cf=L6RZKML_c9wbB#Y4uj(KO9|w%%rk0 z5GbP$e6~!9UJ!x%{4pywxjw*odk%$;E$qPs73h`WY6d@E6I`yk*n%<}@tnINN`eI0 zy>x~+^0a|GuR&zqLGxQr0c6>e(Fbd=Qn?bjpjH$vi2@l6ZJZzh##R&nd|B%-1e(3R zNBDv|w7|6oU5S1CoAfJkkc8$%V9v-O4+EUtrB$isV&;Vbnt3-UEv3<40)LR)dfnV& zV)np$bob`G{uy18TX&>gS_Mrr)zKgt`tv5G3ir%B+HYNiE1G;96do? z^tayia`~Ir!#)Mo7*Kx6HAT|-2!0$3$?Fo{|jp505#0NPzwR5jh#_j;tQl!J?TI%_=USFzlBELFJ!M(J8(0n ztUe~q7kl2V_y~-Xq%cP*^IrZ~%=^|o_)DJ%%G*8i{X$zostnB{J)hIXp*RA?I}_VS z=sM#`(K~gWec<(aF6B^GCB%W253*R@a>QkG>LO)yqtTgaaB{zFb>x5NxC4A_3XAc2vXCp7{^2Dgx{CERtn zIDTc7Y&Y22QT-x<3>$2H&!si~(ygmI)kI{mv;*CX)SxTA8RfI-#mIQvT?4a&;Yv!Q zGRw6M#uk{BGX>Y_OWm!d{Fh3_@70uEW+skyt&+xd_P)QlrE-7ej$$|8S{1i-Ebfu7 zLZU8?bWbhc$L;+~jjcaW`wa9eVhaQJvGmP)&`QSwu4ayQ5NP!V~yrCmcU2Rs7Z(7q5zZk5i$bb5^0z_9x{H zVCw`nF_IkC73#aqL&18kja9Z1E-`cf%&7CBAnb~Y3>aTP0TkX zti_)trf`cJ4LW&LD!#xxUg9=otMf`r z06Hh@5t|Cl^oYnXBScD*XlXv~IWD*#!O{6YXe6#7Zol+wXIt)H$!1P%s2!A z<2gD)!Eq}}eeclMa^&m;1d+LM(JN6-8oA3@H~)-`6tLi_j3Do8cLT8t0}rSqE~Z3% zyzBC)k%Q9c0?@7mHP7o$X}z{?t(@1VG;&#OA#3w@JUMSq7_8=lXRXk+Rx36ZlHv5J zCp%Y3ZaEkq`jKyDbxq2M5^q&T%E{4*WK?T(Tc@O{RAka{y0P+zeYWbh{fqZ{fenvl z0+g5S4?+pJyFRy%;@Usj5c=#rofl`thHxf%hNbF?oQk^m|5a+e7goaA?};cs+Zk27 z#W==mn^NqOp13NZIQ`|>lONHf1v)#^P{*7q9*f#zc0eER@)jowPxf$Jjp}iEq)my| z=0`>tzLXbkxBg)MFyR?=oeF(3Mk=3HcN)Oehgtw|Ia+HYL*7euMy8^8<5N*a+BGjB z!^I!$`g3GPZg^Y4?$c9qsXkS8zX&^}mw3mk(=rX-3$$;^^6B)_h zr9jHgY^=bcFl>)4BjHT)PgbKRt>=T;>8=B!%S>oN{Xa@$IUtSre@NrIpeqOeqckog zv73YR*GZy;DY66v)GabN_(I&5i|JIha=<$Gl=FD5wHE3rDLbn9#NR}>8z}5W%_*oLo{QyLWQFA<0LV+ z!>3n)4PZ}+i@6CYrT9&g`LiA^C@D_9kt1si{N;%)hg#@QiM?FCAg&uvexzg1%PS6x zO7N#gZm&ffDCsec4JzSt_#V;zR>t)YdiAO1HiP60o|6}VVp$&2A%_?P7mOly%8XuT zz=EO+yK=-H!9eF5+D;cBx9zAq0E0EFE5P_0nfe+v7KAOQ{pWLS9sF$IX>r#=3uA#9xYv*4v$ zI}gp9#SEXa_dcp>*~z!2sWa=HoD+Gy(Kkz_cGiIavKYGG&bB1B|K?Ad>o`e%W;zaE z+K~KU$zD(1eD5#d`?p11jRaex7bfqzwEnpHs3T0P+&4J#|J*lf()a!;TXArmb0vwp zyvJgRJ0=}vf%KfF&;^a(Oyvxvb*d!vTw%)IKNCe4U782-HzKq^S(<8w*>B?}^^_9+ zdgH#}Ky()5eX&UfpPWUO&z^j)MK?r1d4Mz8 zWIz4iN4j8W_9Sccfd{7aIb;U7-n1M!KZI6_*kuD=(9 zGa4Z!x#gh@^dM#r&X^&pWysDCac^75x~Pl<`^J96)vTB4ym>(m`svu|>sJ%2@cWR4 z1)@G$jj_Hkm)6Xjc@(AJk)0hW;yS=>tVUzi{2Z#-9UeQ1|LNGBRlvF>Oa2(1!f&W^ zdD)R8&CZcTZt3~kS9lGl;W6Z?)?TqXy1jmy0m)vVnHHt=q=e6K{XKjUqE#CFu6_Mu zd%wi|j@q<;-+<OA%ps>=ayY$l_N2XXq^H#@=D^4AHlwK!f zwWq6B?_zxLZK3Zi^;{YQTGu|xRoi+D?up7LwpoMY&Aiq0BaMUvZ11By;WTV;Yp}Xy zDqru1;{2l&aQr6fs8^sy6uxN>Tl|*zN1(J>?EEujb6X!1$7s5JhrAo(dSx+1GgKK$ zHrcd(DxBzgS2L*>y&j)U#CcV?u6jwm5ZUnbvy}nG&2aJM8?*S8jfoZPna2!O{sb?I zlG10EtBEKT=O$~=+Yl?E6K;i9*+a=4FS$OXEK0qh$YW&|Q`SjqfF$zUTTB%X{=AEd zEob8_RN~;|y#+5$jc?6Jjeh1uKGra(Q^k9%_s=3Of*cCWjj51O0HgFOG(DC>` zCAOaEJ*|J=Z83^&_Y}&xE}>|G?~8ah+1;LddDJJKQ906EH9Vb*0$7M;hN)wT>_*3) zP%G0?hFMHy+tw=;Wi^RA_U(H@SyN27)E-&%mev##=6IdZ1)OrwuE%F{&pzKp|5nUt zn)civS(yFm?vQU)=c@`g1AFlJO8UrQIb?%BzmuaaVd;&XGox0`(QTy21n#;DmAtFE z$7t)7Td0?@hV0?|xjXppXE9hbM1GbYDTC-O0x3aqZT5qS15=sU0`oH75UKIwXmlSx z#%%A8KX5vWMf>?o^GfQ!BNE^j9$on-=fk7bHHkXg(MQ(tq?(Bi$lf zCo-LHv5Ke7Kc#M$lhPikLbyEOX5At^vZ4hD52IGdZbhz89Pix?EfoY(z{Qkeg{kH zv7@*4N4dAoSFP*)R`h2G40S7m+yQX&;Am9uz*a`joXceK{GA9OVh8---xKk+pKY_j z;yUo~A;z-m2|Rr0A=SVl9kj~OCtXOP>A&U^Z$@%IZlZ}$EUHh0SQIwvf_}6&Sr6EL zWC zGC|gl4z2ibN&7VAnXSWhT%YAb4xY4kY2J|f?N|v*fkm+GncE(tDDM9n`4kf&m2xIT z%xgjH(?>ZgJ@((z&S{^yUss+oiWCEORoYT}@W^?>ccLzR(B*Wj+5>S77_m+3VYNSk zY*=!u)2@GGUXnjl5s5M&G6V~EertR&nn!65P7rfzdVCG=W!-I`_pf>8<&M0IhQN1Z zF=B4TzGaQVWy z=(8X4A)ZYv{`Jeyjr{cSU%y^R_JUWx_2N*o5=v+OtZdp>=?Vr?tx!)(ySd(A45E2^6D^F{Wo;scw@Dj(&2%W^%p=h6K7@%f9`v3mX; zHS@gdMN_Nq{O^}#^GL z&bJ`Tral-}JDi=uBHqsV=ktLV2ZRCku(oby14|4QrUfPK^`CKUb=B$&~%oNK75miF!oA0m$gi?9d^YT@?M>gPp9 zbiVZCj)>CGv{}htc}XSO*c)vY@9MI$_lsZ*^ zBBXtDK)$Y@`FS-vWD~*UFsga`hSHz^z31M(@$|=#$#HDJHAWys{5Q|Ht!0~>S5f#Q zlL5{>DM{&wyo$Iz`mmu+`vU!m4|K=E!`VaD#fJ34tqaT_+h1?i2G$*T&`Q}147&)M zfw37Fn}M+z7(b1H@nrgW7QAWdfyrHW`&yg<9E$3OiXHB7#+njE?6A_AEe{r6I`bCI zPU{aR6_TQkPFesWgpPMX6}%rrZAzDJMHv{GCb=F)k!(d7w@++E87SggQN~u3u@z;U z{=5}sn2Bsf8Cy}t=}TKt1~KWaD8otaCo9S*%G>Sebib4Zk?FRWVeEqZ@=a;ea z&~#!rRYQrnnM0lydqQS_JcrfOOJPD-=jecp){5(L1R zo}%QLy<;t$u&7#cGw9snm}#l$qZeIMjP{gPdON1RU8(Lca^3%sFD@t|STbY$88}YU z`eegx?`<4GdJACiI&JWv_lo6}N6re~W&4sH;!_3$*(~YVTVilHI)98RdtxqPS%3Y~ zyF`HKOgME{q$ZpyAZimqe7!Bx@xCcv<;=$gMJ1WGse4Yd-Yw&g%|Y@Ub!en(PUl_M zpfc+uvr3@Qq!ZU$67**}6&vzn*3{*DHt;u3(ptKN`Aar*%v@M^rujpZMb7TEYF@EG z0=Z3B<5JcN4nz9knKgERqnAt1`p|SHTN_D;Vln0&XJ+r+5eiu3ilrsho68+h!W3g? z%NZh)47`pUjx8dv_AVE-irK}!oEPDq517J{Zbliu+8lr{NJ8TvT~-2Lp02u?^#sbM z<9lg;S-Qq>ltllM^?73s7oSP(%Z7Q*h7Uqn-E^kFF@{>tr4j8JpMQ(@@-R^NO_5Xv z=1yq(HJl>isR}r+GTfK)AmcR$3OfbMZA2kOPi!YV`LLOMLFGxV@O3NBf<}XX z3u$+@=!Q{>Z*LGI7{Qqd!5jrEke(kZxml8Lb&pD!@w%g;51+>)Qwio$>^j!jXCl-C zEcbCGJSb0yzsa*AEH`;WY2S1&HisBT=|GTpu$AHCsBDFmf1koiQBx!%D^YQqH;B$x z5fF)wQIKru=Eo8M08eErTiwc5|0ZRtw|PFZ*@(y#PH;{c$rhr?^bwr$Z{NTEGiW!L z`ZAqC1v~W(H?*s{@5i`sm{batyE_^tokhYE58XDOE+8fpOvhmq{ZSgrNr$2u^-FUu zorVDbZG)DHR9j;$()sKvC#GWInXVqO1BM|u1-nyqGh}RUPB_f?!^7N)&#=BY<$vJ0Z6M3_J(tTAUN> ziC*R@L%baV+aa(W0)BbU>6PfM?zXC*Og~>&4Vl75&axpQppCm-#pj)oW7FeG*4>UW zvIeRhL(Kcp(pfMQpqJ`G>ZGaQ(-`YkPpEpNJyXRM@;l$Ye*GHh$Vr9x5;CNYGiz}S z44OKF=N(_cLII0ZpT>{%sB+N6N& zlB7S`N;d4xEn(qA$%}!bQ*NjgNt4}aeVJ!N%IeAV^UN6?29$SxYHSMGVY{ZV?yo!R zGG#~G7l;={2aCh4f_2&bxhWxI6ou3vAr!vw9d?S}{7 z4{280)CdQ1wI~3a(}AA6h#jkMyDi*N$&#C!dHr`;=`I{6s&CNCP`5PoLt{h|gGi>I zDR^rv=U9k_-&6^4f<;~Mw`9~iGs*)E@jF)@??$P=R7iSPHVW;AT0^+jQ-B(CfT8Hm zAxqraxeJg<%N0D?W)PMcTX^+AG%WZd>_;KbSp`8rJkgg*Jrh3Ci|4^>wku(VU>Nk)b-8{vcF4cWLx4T3pBv3>Qbu8&lcFWI zd}98)ZG3xH_B?BwUa0=5bYAejYe4o&nAcHN|G@f|NIw|{Lz6WH%;scS#OinG9L#<47wrh0X&wqdY=RdHX zRx##DJ*|@~ly1K(lgEyINJL+)xlB%S*NKE7FwII)NI9M%B{I=)R5k_03AR^T#ZqvG zm)9ZlL_~#vO@)Rbvg>EP=Pyop&j2(rk77*26cX$R2K`^c#i# zh=pcDa_s2d!8G9$XnZUnZ^gUH=~{mB7f&514OW5<9UrCyl%KsXn;%(%JlBPG0_Am~ zJ@%_r!}l%Al{pxHRSTff`TWS7&Xeiq>q_U6?f4@6YRyi}zS$gZzve2CHe8qE`_A;a z8BcB|Cei!SccezYD>*NoTa-X#ru1~kj9F$H^4)0w0#;CPuXud0GTe`~t9+Au`@i?g zJ)%GHH(%z$VHyF!onT(ur+bTSCmM)dj{yO5dkTM_>2|Mg?_pxTa`z&k^}5|_8s_GC zckfNxmcj~jfm6G-J}buyyd|Mv)fBAOyEzhslV`eli3mRZhxEFlug?~&W_A=UoJ^v& zpf>w7Inm!UnJ|1Vk-b^+LwuK$$0I@^l8ycFtiMO?ej@7-ov?iXCIar7^)6J{W zyPAo8v^Wrh$TOhXNeAn7v8$a5+=_bQpE*?nvo~MK4k4hQVWn8y-a32WQ9B#%Y>913 z*doI7iug?%wi^NI4cqPVwGG?tmkrx(X{T=|^k+PWwiDVS!kQ=aPl9T=;mED#f|*Xy z4M)xtMzGwFgj?CZOCatMN#J#1?5-ov8=C1nnSP$R`CWeG{m`*tA*Lr*>$-;(vk%moMI`6Q+5tBA&Gj?XFR)52B$`bWD!c+)g&syJ& zbI(5R-4QApZAdt+c725IQl`^qN1ng*qv?!CCXJlEv{=_qx$mXwKPNcXPSlJ&xQ=?V zMU92EGw}|FG0BaGuF86(H4}f6BaH=<%Jh)CJ6{GPJe%Iw$%Y$|B9ht4Em8U5f@hcO z!9vcqLMTL|L>G(kH_`jAaNjje$yp5}sRpaC9EkH=DXOQZTh$Uo{y3|s$A4+l@KJ>q zqz~QbfXWJc`>(9@58O@>WjU$Z3sx7IrH1Xo&J5FgOO2QNknNf5DrZ%klF6ruuPV&j zksj$&Lx!jzx?X%_3sl`L+4X9?Yq=F*UJC|R(kS{1?nwQLa{Y5-=1$>x{ zI9EBaDGLQ)fiIGq3YC+;fu_&oi%v$~u?gde-@hYER4MjuKh?*1$<`AHq-l0w6`n`& zdz!_oHb>JYLZ6efnGnSxDZ)y7eD6ih%Bp(HNoG%|AU(3_iwfnK9FQz za2|212Wle_#WFiHLO#IhwVD}WljjV#&B($#eY~MJ$WZk&zz`g4_!e>Xm!09b8F_06 z#YNs5nu{Mn;=2q0A-i}c?(iJ}eNQNF#v!2J-Wk@L5f55Lt`HxBdayy`3H!}Nu;0*S zj5irWfchg)SjZm=oCz1Sc)Vf3cov=zAprwGxNyb=zX{x-!x?c$2*Sm|8AjAkKyfW# z;l!EDQRo)F&>|ftPneO85h&z1W9NZuaqt3%A7}ichUX1M(y{Vvd^$$t^Jn=4hp^)g zTh1g3)Ek(1gCn+~pMz=`PdIbNGkjzCLz_g5d|}QRo2W$!0AZwI1P^~E;OPnT%{Yd? zwe>U$I7B+*89p*#xD>;UM45R)s55z}R1HAA&GWc~iBl{@nKyDde`~H7wGdNqUp#qtyok34(Y79`o=iVqw?HX;ntoW2cJ@G75-;8bVME09 zkHauK`b$!BIYC~UxI}GS0|RQU9mQ+>Qj-rw(;(?Qv;o`4uhEh%YYv=C)K;#B8jk`! zif{Id18uQ@%tue@^3G;=+oSD{X53sj?nlNEBJE3#uh)F2E@HX{V6{rj`6A?DXudlF zsa=w_0pnpt?a3e9n2bB&xs32Bvs92ZwjRn>C6Qt(%6BL zWwA1y*#Is4)UQo(0Eq&LPDaY2S=`BhxDf@a@LxpoH>6pBgsGZhSz<*k&WQ;~pk^HM z#_yv=N_6G|5=owKuL1x;9Vyd=4M-@dI{W0B=fngikmz63of-ZfhFGCK7$F7AYQTMB zWYSICU|u-?W3*JlQOb5<0~AWSv0_MVC0*FSgo5;V#Y&`k!2B>^=|!YO7dCfu4sQh_ z@DMK~eH{oGPjODv$=|9$gBY7srKwIe$bqs|H3Ot|NGzrKav}{{SNiTJ)#$M+oBRcI zfdW=06AO&Pk{K(5;-~3~JW8A$x%Qlivjb{PMAKf7JlFJrPKrXKAUA`xFlX8zLH4Zd zdDb?)7{sf;vLHyHTOKgMx`y0TnWRyK@8mIQ0d$o;On|C_!IlKc@j)FXjJnqQbrE|f zlI2VsB#OO_BVLj-ZJ77S-X?J2GR@AprVkPblHLoFL=%RI(D)N-VInk8Zn!|CgRmgy# zYj(|10T!|zn&VUZXGq;__$w3}tm5iUid0tiHGP~wAk?xrnI;xsvGnyPX&nFq8<=>e z_HP1ln)_uiCIP?!<*_YN29&SJqr^!RP`)A$5(ik58zjq__EL~1*YBq4ed8reLFpak zRPD>=M|L9XWUcgke{9Mklm8#fprSd21EmsK4lf+!ehOZKx*tPa&)ac+xf1%z#2ID! z+klzU3p;TpJ_|utxnZ}+au}KmNT3usg@ruu{KCMy${OH#yv41>ef(uTi5b73-OBdK z^z*Dyp&7gTbjogEMT>tMSdr<3-N1^jNN-?8m#=MLMZaudMf$l7tjO{E?LarMq7AHw zl;Q?fL`-@ED{_+i*}#fgcDqWQ{#OW)k5>po4TqiN?2D$#3#a_N?nHjH-Sv0jYPsXj ze<0U|kmjbn4jXhal=CV)Xja?lt1bo{amlaWhsDZ1$&XAYI`z?IAdXMT@?sXnDm}Bk zC_MSZINiO*ZCtJ+Tt5K5%o}&yfvA3f% z7@3l+>Q`3cCK<*x>;QM^Gbf70LdtMGojb0tBV1Nx|yPt;N1yE%`TTPI$5R`Owh zo3&94dAX-N-hlVL;>X|_hbE>HF`1;|ojQ8fc{{q=Koc@nJ6px43p;$Bk$b$-rR zi8@&Kr8MMtZR$zHa6WPde94HltDk2(0l0;C_7&9YRR27q(?s)pBn5HFmTFRXMV4R2L*fe0khG8a{w#*Xyx zy?~JF(cdK#E;mIn=8!1Ey$>;f%S%woHr>LNt){kCJ1fVPtn|0})ti^rf()u>Ktm!g+7Xl2PCs3NCA+)*~GPd^@v*?HEN)r^@9pBr5HWTq}$6YD% zOiunCD?G1+$j9#W4kp)!0IEi$wqLPnl))dyM+gASBM>t0c_*X_0ktN~Skr`T&PB_2 zT!5#_*Ug|vE5l|`EQxLgg@x$PVo;nSD9O}om{)K41lyI<^+VQk4(e;N8!I;YW^|Ct z$Izx9A6m#l(#TF60E$Q|mMe}DggTOj>1w0&pp-0_InYWV>{c6>AJv4G9}Mjz3D4E$ z)Ps%^z>GvkTfs^V?l zW|Zxoqm{DzI?WmNpf+X*xN|HAg)!rFgurVH&~bnYtlMze77r44n|5-qq4@p~WuY)Q zBa6k7v>%1p!>-aJ;f+YqoFMYc>zw{;Wzu};nMh6~+PBYHUGq+Z0Xq>r!nI^C4#&Z> zr6S0CwoROR96J&e2`|tRTvCU$n#js^oE_8U((5oAOV`_&G4iWqXZ6vk>|q^lE$e$d zE03n_B8TO+yJnoi%OU<4`AV(b5I8=1u6=&wE4gegbU=D;00cpLrV3?{)&(Xxg*_!H za6BJmDGJS`g33=&mOm~_L|xkm%?uIOHil$_XhlF|fT+t1QCXjK%;A|HYNr#Q-MP0T z2rUhvC_9IigAnivC;=g|%y4BN-33{bmbgYL*l1yN53l`iUnza^ti z6McoPn_Rg4zn-WY z+q{Z4BBcR5f#E3YcZ_%i9|>M0@~|c9Zy^n%>%24sbbfh}dq|b3VF(UXao7m_XyOrN z93TYoxWf#L8V*nV;n2dVQh)%2@Cz$1qOs4-1)+iydAATGa8gHl#ez8y;x}D!9t7o^ zAe;t6_NG~l5HxQDC)`12(M(R|B4!9gZ2|=cL(#r40wBUVtM4Gl**N$D6k;}Uco&G0O&r{XA!E}r zMWJC6cC(@pu!-ZjP}FN$wrOfeGHsrcz)t4UR&4>>uGx~F|8k~m4uZS`O(3=Nrv7aReZ zsOJujdQ8+z0zx{b;e|#wCaSpxAsVlJ7(~6*t=fF6 zHopt=t=b$c_tyHgX{K-FtRTweQlK=c{P%O8*LS+>a0}?euULX4ggfubu>$$_{mV~ zeTb>?7CRIrHI9=P88wnT01Y+L@qiC}~XbBuW zr>aAOM9himkg!p58n(d5I8}KNI2umR5EcQaa&B-@@3ox|NV++u921gaj-ZcBD@R%- z@KiDxBS88%!d`)=j3byp)5MWB1u8WhZNGTZ!P`N$9c1YlZU@;1J8-I&)Es|u{eC0r zbKmTTAIx|ZZh(i2FSR8bq~TD}jyg@?J}v}@T0{fO2O}1%cL_cE3tt=Ia^U%~7JnF{ z%-dv|qu1lkHEa4ANXQ{!(SOo>HsPg_6z9v6>E|oipNI1HCAePCb1vGW@$7Vu>2_n~ zr7(Le-d&Ze5BCapmUi7<(arLsJsucF-O*G9+70eL<49M<-CG<@+}%%{ytu;^Pz-yS z|7QP1tJysPmVo*QZ%`$GrgYM!1?_Uuf=>*Gs~k$RRVf_lQ%(wt)01DxiJ9c0@|dCCPCa}i)?HL z;bssb7;XmP2fHn@5yWhZY`~dpk&W#j+YYkrAhU8MImkXiWWx>$i;EzfhZ-J|cryJw zt1#)<;te}%OpK*J1X9#J%ME*LeOZ>>vrZYT{b|TBmZK{cN6FFsiX>$2a=j2SH*elj zadfjHDLJ}UaRak-r((r1ccD7$r&|iPZc}WS(ce{yqiF3OMUruJiQ*^$xZ4yAO75DyBdZTaOn^5f66|F zt|_exiMphn%9z;i)n!-a?AoSPtW-CxvT2o{P^;LsAKo3XZ8;nz+cvw0YI>-3f_kf# zErWX%As+N?(FDbkux$sVpP+A3d61R09eQxNEVAh@Z-?IJRAy@*dQYaGZ$0yj9hRXn zcYKe(3S$i1t5a4y3rKV@BvJ<>6co+avr3d!=+t0^NlNgAcQvOanXK=gl<4V8bOhb^ z=27pD;Dn4Y;9AGmK(M8+ZNv67QRLW{ss zb#;GZDXHjxMj+D(rZz-jzVfcOXo-?_~RaL~57SU+!nYv1Z-`;cKKRPPbwg%~jX3*EJ^0F`- z8>Ogp9mw;y|M~j$*~?=plE|8U)*o4?2|j;&Ha;MAntT!L^vL}6KdJxzdisBGCxX*4 GVgdjy(vj}~ literal 9739 zcmYkB19KhV7HDJJ)`@M~X^h6UZQDArZCedb(%3c|n{Cv*-uv#%d*95rf5M)%)+UXE zgEO$wFa?8jFmZOaakO${wsmuI1iLnnPXh3y^shCtx_1z+2rZHLDIqd8{iuYt41s>h zC}O9JQJ;=_EN*xh{0fk(C{v?kYoITu2vuOmn@jrgeLIL!{92u#*$Y~4 zswryBB|R=o3`c!%`ob89HiEY*E8RIpJyb4)t901=Twd_E*7@@M+gBvOe)so6^`7;z z#p#edCxP+rUx_`Gr>A>T5kpze_zV#bdn0rdsy56dLs}t&g{{28b$`?E%6ei;#$UR<;Xd9*^EnS?uL&&b@w3 zFLjuCPz^jzRn8R_@MNNXWhrVettQ}s=l0^?{|>s$Hr`2X=`IfpT!B76$3AAL9xu@R zyk0|!U$)u`xi@{Wde2-w^_Y55=*79N)N1LU?{xCev$vf4xHo_Qv{i{EazJS-V>-ZJ zinX)wl1B;&*C$z!R6-p?_c3xUa4&EdF=DC6CO>eUY zhqAG@&ywqJmENJBTJxSMN|yYidfY=GqX zEIUn7CCh7yqvt?Z#)U{FyH;MUOkkfLR07_GyoO3V>lqi87v$+51na%kFc&8=M)u3? zp;9r$#@S$J?gI}qIm9Qc3UAjs))n8o6+}=!AAD*(C1@j00g6wu-Cb?d;1;x8gou5O)2FEFIc2j;VxP;e&eed2s;pQ6$c zzNyMEFoNNTk98LzY(Cd5mdme|AdzNEcaum1>!k2ImH!RHk+qWec3Ly0N#0K%rpb9s zojCT@lK7>})eP9*vCWYb;M~VdVizdD(N@< z=Lh<>9yj*R_AFO5@L@sIrn=2b0WJ^~c88&eWUnXjgqRdGhPGK++WHYsJM>JI(pFd< z<(3%Hxe{C7?q8tu@4*?YJ_SJ7iQ7$8%b(aEi73A~xQAGbf<~!dpC57cQw+qa-S{Yc zh-inpBcC>*k7Oui<5IarnhdQ8Kk4>yBml8*YKED})w?P-IBipUtD>P`ffY1IoE%IUai{G5aG&&hpNTeHjoDP3NOF{81j24C*E!QhA?!D}e zqK|Zf`BpB$p!%|C8~PNL6*+VqnUfZ>i;i2u&+VdxJ(fOhk+ssiQDhvivGmJBMbCz- zcS_lUsv%0a(MK}3*qN?v1F<+raj~$?b9ydlXMK6%MYv(*+!`eS25$TgTbm56iR`s- z2#UNe`Q&0j73wL%BIX`H_gV`2`BPRwRMSQ+{ZwzD9*!BOcwD~jwHm%R2>4&r2G^M_ z8P0eh)VcU(ND<%eySm>VwXN+!@Ft9-z2%a(eR|^Ht@_PPW&8o%rJ1Zh=W|>mN#tO3 zrBKGO(6xOtTfdE7nHtR^w$8yS9Ki8}jWu#3?#Iy-7vnNBgt8=Q?I`d=Su8E)Hw_>e zm<6V3|4#AnsYk^vl)Ls;B(fS7D6Xr~GuU5`(v^uozs4){i}N~UP5z?@)bKS`w(jA{ zq%!LdulC4wv4`}`c--Fs>a1i8%N|C!*|raDWm5;0zXrQT{}#W$y58y~@VKrg+;_jy zt=SfTk_`~w8~&W!P}umP8yO)*;HL&O?5()or!|1wlC}dOi7j%ObWl&Lbs=@v_T%q+ zsHg+73YUg-n+e#6T1qTra$Gil!@*D;|3GU`XOy6nkL9@Mbd8eTMbg<;tpw0(Rn;YH z#KAcnEo$k2pK3%3Fr%>RsunewHEl~`sXa+{OaoDK$8j|Z%n^;o)H};e4`bAI_60NW zAhpjh>ABy`v}7N6RJ^ji7beE_xJ?9I+PW7$ur>EaC$aoGOX)|?DDCc;mN}A%ncaJ; zFHAvWLHCn0syr1VL6k1)5YEzXiP)(yhcgHDW_$-aeOu=ex#Rx)F9Oz_f1Sv$XDuB> z;O=LTP2m8%9Aa6e#?Xrk0d(B*5A`=HbeM)-8czq0-el}U(;FRB^*+?%LZba~3d5twl_WG%tX{_;x@O5k#bhacOw$*GnI(ZZ179Xq}pC+qppj^8$;e$KhHfFp%Q$qx4D#as`R} zlgCxd#0;8oJ_>@Z6N3~q+Hg^!^Fi)T z)o9vRlpHH2c;FBI(XgZsAT?bYncui$LP}yP7y@eSiMgC9&fZl}M%H?&`YC^C2x&f* z_Aqdiu=?1=|kfO5JQ{GjR@!wqhO-pjGAYgTc)y7z;j-x4`1+}xlL zeI}&A6ZayT|M1L~Bch*1ih2M?<2y6MN`#rzGqNxKLqQQF8pWBq=#UMkJ>t=lBR8136qW&6`?v<*-bTAha!j^MGn55`$CKk%)>&7{{NA z62=k}ETpTs+gI9XAjp1XhF6gnB%q-w2-&KcJ^xaOFAwg}yDZn9gic{QIvAvhG_T5A zLP%cjJg!$%k&K5&Ugdlyz-%6GWgK1yqHv0A5kE8z>&j`K4^|(-&njwU?b0S|w*2*J zfR3nv1LFZXyAkcNHm+PQWV4(YJ_rOD39CME|J zCyPq(DGHypZciJzq?%=zc){BcQG571Pa1qub*faJO`~mY^W7SU)!SBhMDJ`yi#JId zg&qnA)bQEBiAF-v6z`H{gr&dx+HucD7Wo%5D)>v2uj72DoJ~Z#Qf$ilUCiIxiu-H0 zTkF+j;?z&|^WJ>}y|Alv*2YrCC`0DX@IXc7Ei>`U{MNjSR6gD(`3L?(R9mOCg8RG9 z-johVjrQhWoCV$zy5xS11F+_!hr~b`WhLf}04hcImiq5|b4Gm^OE8CYzV1Y3m(0=y@bYlpw>-|4puo4~4dr!e`|XDPdAD*pwP*`20{ ze!l14u`Ttvu-zixMLX?>LNn`F+B=5|%)!$cm~RK(puavv6r%UK=6Er!^w%-|Ckw5@ z7U&@SdA>gKrAz+QWLcU718%9s=&9^LP;}hzV-g@Rh*MogMWhQfgExf=hf^t<7r|oR6R@7rsbv`OS~Xf zz!VM{9fNq#Y-6f#IED?nUk_O{rm|kRH~A*|wjp2`G==f2o4E~>wh{5VSg($t-%gf! zX6R3?E?D~PsZ7Oxd0dO%t);y6u9i)J?bWhx;4`OQRM*WQ@lxRE)K{=OCQJ5Keg{bK z*)Y<}>!=C=#9vw>=djGO8TT3kXVn%P>N~t2nsnyT=y*_ky8c?7`5pZ>vg4o45oZGh zsd#~aGwbtzLwggefJ4SCgL-io0+`7Gyvbo@b+deO>Zi_@Q@nw9cI}wGg8nMW913R z`f$(t{%qNz+Qc-?3QHm&*<0K{d+DuL>m;`_A#Zm`cODi|VA&J+onMQiB?$-C6wL=_ zS?|D@*y_h-&0sSc8TGHG+e*-GC2I|v4y8}X2*e%%qS zOv}D=YE9F=ll0&3&hIanC%q@qRkjX;lP{RxgReP=1$1U1RtQLG5%C$|4LM7s>`i(a zK)58yLewKTcoF;Vswkhz{eB#oae9Qje#>~`U?XnU-4fakES-9t$8P|7RZ~5E+T2)t z!U@P3sIHtPaPGC0KcgUun1W8~qBL1n4xM0`tm{>b)%s5Y;V|MyyH;` zJ&V3dF3g}*Kq7SN1(m#pag+M_I7IsA|3nXB`N#| ztYgKQ1>eLbq8fH~apW~f33bU&KHCWd}dKhu~$fs4gH!^dVAgsyA=K}ro!vOuw)76;e-2?vo59ttI9_5+WqC=_i2s9zgLB6M0{ugs%=1)iLsU=-x zW1AL!dh&M19M? z>nSZkEgXZispd@SWU`szK)*P@%ZH}N6VR-Q5_pNGih+*Ms1-VF!Dk)tmt!+SPM4_~ zx+BRWQPA=K=~0XN|Jg$8c{nQN26meC_QgRvda`MDJqbDK{u9AWeA7g0yH3b%h)*TvMT^LBVITevCU7HQ^Z64|{0OmMP+Ag13Rl#qTv8qWdRypt31 zrD8N}J%wKMT$pc@*b&^jq9p6ye0c(ojYRMTKt+3;7$FZ1x6M1xWXKyQ5se%IQnbq% z^fL%^T_OteFas?-A&e`&sv!7KcMueVnO!>1q+cv0sFcVGAOWb1yMSiF*GvzfQACvf zNFmSR$BvjCumACmpFf?uiy;Li3fc1kX;qy5j1V$5BR>;q4D*7rF$)d|hlw{OO@AS{ zDXmK~se!=(PypV4IKY)jCi@=~6Mp%W#OP-7L5_2_ zO4hgygn18F+v#J$d%_Lvy%HzWBFB zJP^}HD$n2WZR}9f$nOvJ(WoUAe)Dp+miALp;$r?PJI}02urt2XwS)DHH?ln}wkDv> zkD@ky|HCyo(R(u3BdK_7Yj^}KlDVG=bSYwjNq#+}%ir6uck<)hS-9fC&3FfX!WF+z zjjh=UYv8gOLN@7Bz4-%AO{xY2KgAqEo|U>9K| zD^wd~elkc0A2MI0oUy3h`5rKgQbU|$n>{|X{~J-aFlLYF^PiyzFYW(}Dd*tpG;K(y zIcpk6#&ZJmhdWr(r@519!i%qq$a<9D0(Q(V2mp)=3I*|)B_kudOrg?9&s6lZhoN@I6KIcr}RZ2u*z`ldp(kDVVT$2%BoPl*)~ickEr!%khTS}rgJ%&XYgVr%_9 zAb61;NA^0==Z(^zR8Pm4j@uL>W0dp~#pGR#(sefCg2)<+H_y!z9!rpLjk$q4Px6kw zVSi9lA(QJrI)#gO!0KH$pFIe%KrGfr{qf$twUVb~`H$_nxE)m3uaaHHFXt2sl|aE1 z5;)v2F%Q-6^K0Rzv=7#~jQVI&{Hr!rf6`-BK?@vlHIo@{4vJb zD&R_SyzA#|ShUdYu&ZUQ=lWvC) z7;>6!*}3A)lKgHVF`R8h<~nr>R@|b9BF}cIM4~04C8@v`S{yHSGDpb}`VZWU4`ezV zGLy-ktW0f_e-laC#K`Xsmc%&S0)mrO0W5l&z~QMhtTOc<^Rfn^On2gX zIfi$J(HNEm0Lnux2+oCgIqm?SEL^1^LZo!(tQLcHum}uC#!GcQIb3emVlpAaW(+zK zRuLp^Uv#>Kh?$+dVZ9JE!4X6`E!fbcN?t14W2faJv3QRfHLXg5U!vJHIpB-BlsJq2 z{6Ls`29MvV_D|~%{6XLbMf(h?i4jS|DK-pYhn45Cy_vjv?$>ym5Vi30GzPza4ogl)@tGp?>{sZ2CkC1g&_zM>?FUJLZGj&vSsUDc?iC<#^(73t>^KNnJ3kihbqmnz=WCBj#`*y zUgy}wVp)uJnE8m_bt91AdMq=Eu#L%Ui*20!_qZ&<$#lu{S~(~XmIU} zN)a~}_bN>rQMbT5DdyQxB-;DfmT^)I$z5kIiDt%5WuL?WR;vOV+LsLq${E2ty zyPNp9ls;c5J!@Do0G~zI*95To)r#hxo_mXvjVj>+4xdI*$(jmVV031qM*u09)Y+BZ zl{8Q%!*+z*X0RdrPh2_amnHlPl{T>JtY4g<8UDp32JHpxbL_gae@m?HEDF0$R;V)+ zTPJXU_zXd_Efl+0O@AmhS+3DoO4D_ZP+L{PQLB383jzGW1);e5fHgJGUscP;B)w3R zX$rOC$6scBWUr5I)L%8mpl!}IlQ8DU7Q+bG8C$)8m5pEk^Q*^uQ-4d+gI@(^{o5Hk z+Ek^r4_lZicELs>cA~#&-i_fQH}e*H0|}VX$+g#c)#K-3FO`oD;^f(bKGLjqU!c#8 zpy1=rYaAOHrGUMbYvcYz)6prRbR5sc7@|uDV%)Xr$^cLQp%?LM%kbOFjAvMNEPi|0 zL=!sGr}T3cIo6h))x#(19L_XUJ-+Og{GW{kP!=PjA6ljz!`>YJ1jM(-yXtDYW(_Ph zo}{y$?O>)`DEav1VpCWa-$~(r=bGcKH9>Ay%+=B2aL^4uyE}IGqW~4fxXwzYg;ixj zE>8r_9hn3c$c%mtV>~9rzbP{Yep8ZSXCy$KS`xJSP+*+>5euf_d=4f1mxM@pUBt_V zNpn};$Wy}vfgN=nenaMQNKf99DIb3>*WD|qtub2NF+xo_~dT2 zIq$A+_j;Pw6LD&>d9Tk3+UfhYQEN*F=?LfGVLFG-o#i2;Ea*2-W5Ubb>09q~fnvR_ z(DGl&0#tL(vVcrjiZ~kHt&XyY8GcW!q+7=BL%7j1bm}+yE39CbNX+*2m1{)Grr2S{ zASQnZ;fzoueWhTjW9%2YxW=imHPl~^LRo>YZ;wLjfuBfhxxfui=yJp24*?re%Wi*= zku+gtN;($Cxx`_|dQ(Qs7E49TrPGQy_>(%zBt*Q*0%=XD^B0Nq}6^V~>()m^bY5##DttFw;*&?pG{^#tZ2|2^( z^W|e>RZiCLw)$wv5e)%zAD&c?on-2KYODa8+x+Z*CeQC+-`nFPp5P%=`vQ51($@8r zHlD#Iul&g_?!COiOcTx95E;9}!(8uJI(5OV?nOSh?*maTM<8B%Aq)`kVvF;|YvdGb z*;&lTx8^|K`@j9|;a2;=he}eoZA>6b{}y7L066LYvqBTb3vW*X%$Nim=p&;?tpQiX zWdb6gF?VCu?LUGL{NCRHBc+S48D^^F&P9tPIe;5>h$SXhVL06DJ51AWz~``LMO~?=KzSX49erLrrlgKIx# z|ECPI`on52lz^ky{)H$Z?Rw2dsS388O=2+Lj88TmzZL*xs27|F**Mq(pXu;I=Gtk) z@REbO_ph>JgmJWJGs?ZIW(nCd2${}CW6i=-^!HAm4`@)&L=>D)z)QV*@>SDs(z6;f zl*n{tT&@3|7wFJ{@)_%hQBRVH3#B-K(hZfj1!|(}bS1V@)#&@;TS9jBCBOqZzjbJm z=1-Dj9IPfcsJA{?Z!)`Td;YKk(e)A?c7`x7Zz6{r4WDiv!}cUkSxUf;oWh8rjabpE z!nhb1r>U>2ra+86-ZM%KWoxLs#$+DfW>SF~9KtOllkEs4TEx02yv>7~rjYttgwL~v zIfT}gDuX;vcN59lIH5|fh;&gx2N{Z>peEQK8Qw*$5E!3Dn>Ics1<>%`(%i{5)OIb1 zthtGb+|7jI*M`?+?StiFpGrK*m_^n+GsQm=m#g4cJZZr?B!2w{ z_PsIFUV`0gprDFyD=(Itq8wZ>D&rbV z?lQ*KT-k;zWW+kxR;xNajvkuk}*%F+k+tlO%kpNT6amfl#C#!@Dx zxakRIAxKD!9ze^8N?&w`m68CsQiGEdUA%ot2>~nmVQ2}N!bn6n-xZ>m;hO)TU(72Q z48qq>w?EsxUFkaXYv%4nyy`ljWcWFJ=C~Q?4ftm_)EkZaudOydvl^lviR_qs4CxKz zRdpwZ8OSWfry-2&L8=kj6@*YID zSN%ho&jP0Aa4|~=pudi2MgeNdOhiZ$7AahxwUR~H%H%5w{csH52-ayX%H`|DVd4@< zx!UPrYuBicZZ3M>uNs7g6(Wag*3Oqf(^yArz;lFsspEEEABhG{~Y^wg#)D z-GgSJZpRJhYZuJzO5OpJ+{Sp$%4IHX8DT?)dtz*Er1G{&&+}kQ2`R8iud;$)i?t0` z3nT)os+qFR5WP<=Uj2cle#GQS`^}b!hKAji(g%#dw(JO8Ml@mC=!dGZdZ9)JCl8X| zAfS(!M`t#T`=f-)`H83&1s;PY2>01=^EJ3E z*KCrFobCkft-$73wic@3o3t5}EUC;o!(EyS6sC0!?_$N>H)NSDmPGeE-{g8&9pA`H zv6oqtyc#L4u|oTSz386v3xlr&>m(5No%2h1WU3(^3e`>364cF+xU*qg6^&2hPg4)( zVws==d4T@^z&6b%Um@VhcYV`RQ3w*AkCO3CIcH9~mlNa2-{+qTQoUE4cCrk;_;T~v zp=`D;30G_-q`Ou8*3SDupGkT{dBb?q(^ryv4C^My8sAf1jnGDT6)*~Hi0$j-&FShZ z1R@3Kcg2a#yhi+%22EQ-W zLpJzBYCCf?E;y~Q`LF|9wEk8-0HN6Q>Nrf^1i^+jYRfa39x6LPD=q8_Qj&YRs3{OV z{843C0z$gfD)+{o^5pQuG`&?>%9{5aI zcI(=Qw!q`GtJdt^>Z{h`UjK$XJ)eKUkl2s7Vk)wu#_Ey#7Ky)OP#%btuv1!K&S+2) zm0e`o>V>B`%t4_-URy9o*YJhRV6MO;?OeE+Z z!Z>|#A3qzE8Gz(4lYj{Sql|beYyJ9{h1T2EX3h6_4XyeX+w@zN+N_Sk0RiJp*T6J| ztt*n6^WxQ*H9anS`~fZWL$N&^MPo+zCC|Gj%w-39)RvZ22~_4xG4AK%=lS=nN&9n& z+ka)H-l_ojDadJ=bh`jp8Xe5%w-{;#lY*6?@L&}eTZkv7!{-sNgkez72xV!jr)8cn z5BOarscZxSSP$@^`UOGJnCE>v=DmyUFrVU#k@ee7dU7`cj0CwU^*Gjzj>wX(JE=a! zOK3>YQNu>ygqGW0{(UK@Ed9a{RiMe*M2qk#`>raZVdr~Y#JraI!dt$bGF#2(*-M>c z2hpbK;bHDzFaec`PRqaPeFm$8jVCwnzsLupevBj5uEN$%4(sNoz^R&O2BwnvO%9)QxW2wqP=I?{nGNUS zvqxitL3!^y5n*0>4T|~WtJB;ZAx#n_1A@oFP=yn$FV{qFvp+?xmyQ=%y=nWa|K}jQ z*O%H{>+1;pcadr0+rvt$S0`}>r|O*BIy^me?!utImwSPK-=-c}f?kiEjGj#4i5YNU zvJtuCy%HV&-immBJ!XEr{r>tG3ksrZ5hM+4{QCWROuD9g;k8ejU^~73@$m8b`nqPs XQ{nMN`SpJE1#Y-!$WCVl0rr0YQDEFA diff --git a/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts b/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts index 2785fece6d90a..b5c72abac6329 100644 --- a/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts +++ b/x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts @@ -19,7 +19,6 @@ import { waitForSignalsToBePresent, waitForRuleSuccessOrStatus, } from '../../../../detection_engine_api_integration/utils'; -import { ID } from '../../../../detection_engine_api_integration/security_and_spaces/group1/generating_signals'; import { obsOnlySpacesAllEsRead, obsOnlySpacesAll, @@ -31,6 +30,8 @@ type RuleRegistrySearchResponseWithErrors = RuleRegistrySearchResponse & { message: string; }; +const ID = 'BhbXBmkBR346wHgn4PeZ'; + // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver');