diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh index 39e299251cac04..26501fd5555d0f 100755 --- a/.buildkite/scripts/steps/artifacts/docker_context.sh +++ b/.buildkite/scripts/steps/artifacts/docker_context.sh @@ -11,7 +11,7 @@ KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}" echo "--- Create contexts" mkdir -p target -node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact +node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --skip-cdn-assets --docker-context-use-local-artifact echo "--- Setup context" DOCKER_BUILD_FOLDER=$(mktemp -d) diff --git a/packages/kbn-management/cards_navigation/src/consts.tsx b/packages/kbn-management/cards_navigation/src/consts.tsx index 502e4dc3c03ad6..b5544d917d9dc5 100644 --- a/packages/kbn-management/cards_navigation/src/consts.tsx +++ b/packages/kbn-management/cards_navigation/src/consts.tsx @@ -93,7 +93,7 @@ export const appDefinitions: Record = { [AppIds.SAVED_OBJECTS]: { category: appCategories.CONTENT, description: i18n.translate('management.landing.withCardNavigation.objectsDescription', { - defaultMessage: 'Manage your saved dashboards, maps, data views, and Canvas workpads.', + defaultMessage: 'Manage your saved dashboards, visualizations, maps, and data views.', }), icon: 'save', }, diff --git a/packages/kbn-search-api-panels/components/integrations_panel.tsx b/packages/kbn-search-api-panels/components/integrations_panel.tsx index dc4c4724775785..3f6005f26cebd6 100644 --- a/packages/kbn-search-api-panels/components/integrations_panel.tsx +++ b/packages/kbn-search-api-panels/components/integrations_panel.tsx @@ -127,7 +127,7 @@ export const IntegrationsPanel: React.FC = ({

{i18n.translate('searchApiPanels.welcomeBanner.ingestData.connectorsTitle', { - defaultMessage: 'Connector Client', + defaultMessage: 'Connector clients', })}

@@ -135,7 +135,7 @@ export const IntegrationsPanel: React.FC = ({ {i18n.translate('searchApiPanels.welcomeBanner.ingestData.connectorsDescription', { defaultMessage: - 'Specialized integrations for syncing data from third-party sources to Elasticsearch. Use Elastic Connectors to sync content from a range of databases and object stores.', + 'Specialized integrations for syncing data from third-party sources to Elasticsearch. Use Elastic connectors to sync content from a range of databases and object stores.', })} @@ -153,7 +153,7 @@ export const IntegrationsPanel: React.FC = ({ label={i18n.translate( 'searchApiPanels.welcomeBanner.ingestData.connectorsPythonLink', { - defaultMessage: 'connectors-python', + defaultMessage: 'elastic/connectors', } )} assetBasePath={assetBasePath} diff --git a/packages/kbn-search-api-panels/components/try_in_console_button.tsx b/packages/kbn-search-api-panels/components/try_in_console_button.tsx index 93012c58a036d7..fe109e025e2e5a 100644 --- a/packages/kbn-search-api-panels/components/try_in_console_button.tsx +++ b/packages/kbn-search-api-panels/components/try_in_console_button.tsx @@ -43,7 +43,7 @@ export const TryInConsoleButton = ({ ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx index e8feefbfd25339..b1d9145a9e6128 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx @@ -95,7 +95,7 @@ describe('API tests', () => { const result = await fetchConnectorExecuteAction(testProps); - expect(result).toEqual({ response: API_ERROR, isError: true }); + expect(result).toEqual({ response: API_ERROR, isStream: false, isError: true }); }); it('returns API_ERROR when there are no choices', async () => { @@ -109,7 +109,7 @@ describe('API tests', () => { const result = await fetchConnectorExecuteAction(testProps); - expect(result).toEqual({ response: API_ERROR, isError: true }); + expect(result).toEqual({ response: API_ERROR, isStream: false, isError: true }); }); it('returns the value of the action_input property when assistantLangChain is true, and `content` has properly prefixed and suffixed JSON with the action_input property', async () => { @@ -129,7 +129,11 @@ describe('API tests', () => { const result = await fetchConnectorExecuteAction(testProps); - expect(result).toEqual({ response: 'value from action_input', isError: false }); + expect(result).toEqual({ + response: 'value from action_input', + isStream: false, + isError: false, + }); }); it('returns the original content when assistantLangChain is true, and `content` has properly formatted JSON WITHOUT the action_input property', async () => { @@ -149,7 +153,7 @@ describe('API tests', () => { const result = await fetchConnectorExecuteAction(testProps); - expect(result).toEqual({ response, isError: false }); + expect(result).toEqual({ response, isStream: false, isError: false }); }); it('returns the original when assistantLangChain is true, and `content` is not JSON', async () => { @@ -169,7 +173,7 @@ describe('API tests', () => { const result = await fetchConnectorExecuteAction(testProps); - expect(result).toEqual({ response, isError: false }); + expect(result).toEqual({ response, isStream: false, isError: false }); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx index d6b61ee70b1ae8..69e6d39d85e11d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx @@ -8,7 +8,6 @@ import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/public/common'; import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; - import type { Conversation, Message } from '../assistant_context/types'; import { API_ERROR } from './translations'; import { MODEL_GPT_3_5_TURBO } from '../connectorland/models/model_selector/model_selector'; @@ -24,8 +23,9 @@ export interface FetchConnectorExecuteAction { } export interface FetchConnectorExecuteResponse { - response: string; + response: string | ReadableStreamDefaultReader; isError: boolean; + isStream: boolean; } export const fetchConnectorExecuteAction = async ({ @@ -54,15 +54,60 @@ export const fetchConnectorExecuteAction = async ({ messages: outboundMessages, }; - const requestBody = { - params: { - subActionParams: body, - subAction: 'invokeAI', - }, - assistantLangChain, - }; + // TODO: Remove in part 2 of streaming work for security solution + // tracked here: https://github.com/elastic/security-team/issues/7363 + // My "Feature Flag", turn to false before merging + // In part 2 I will make enhancements to invokeAI to make it work with both openA, but to keep it to a Security Soltuion only review on this PR, + // I'm calling the stream action directly + const isStream = !assistantLangChain && false; + const requestBody = isStream + ? { + params: { + subActionParams: body, + subAction: 'stream', + }, + assistantLangChain, + } + : { + params: { + subActionParams: body, + subAction: 'invokeAI', + }, + assistantLangChain, + }; try { + if (isStream) { + const response = await http.fetch( + `/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute`, + { + method: 'POST', + body: JSON.stringify(requestBody), + signal, + asResponse: isStream, + rawResponse: isStream, + } + ); + + const reader = response?.response?.body?.getReader(); + + if (!reader) { + return { + response: `${API_ERROR}\n\nCould not get reader from response`, + isError: true, + isStream: false, + }; + } + return { + response: reader, + isStream: true, + isError: false, + }; + } + + // TODO: Remove in part 2 of streaming work for security solution + // tracked here: https://github.com/elastic/security-team/issues/7363 + // This is a temporary code to support the non-streaming API const response = await http.fetch<{ connector_id: string; status: string; @@ -70,10 +115,8 @@ export const fetchConnectorExecuteAction = async ({ service_message?: string; }>(`/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute`, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, body: JSON.stringify(requestBody), + headers: { 'Content-Type': 'application/json' }, signal, }); @@ -82,21 +125,25 @@ export const fetchConnectorExecuteAction = async ({ return { response: `${API_ERROR}\n\n${response.service_message}`, isError: true, + isStream: false, }; } return { response: API_ERROR, isError: true, + isStream: false, }; } return { response: assistantLangChain ? getFormattedMessageContent(response.data) : response.data, isError: false, + isStream: false, }; } catch (error) { return { response: `${API_ERROR}\n\n${error?.body?.message ?? error?.message}`, isError: true, + isStream: false, }; } }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.test.tsx index bd54136fae4f15..a7eac5c362ca07 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.test.tsx @@ -9,50 +9,28 @@ import React from 'react'; import { fireEvent, render, waitFor } from '@testing-library/react'; import { ChatSend, Props } from '.'; import { TestProviders } from '../../mock/test_providers/test_providers'; -import { useChatSend } from './use_chat_send'; -import { defaultSystemPrompt, mockSystemPrompt } from '../../mock/system_prompt'; -import { emptyWelcomeConvo } from '../../mock/conversation'; -import { HttpSetup } from '@kbn/core-http-browser'; jest.mock('./use_chat_send'); -const testProps: Props = { - selectedPromptContexts: {}, - allSystemPrompts: [defaultSystemPrompt, mockSystemPrompt], - currentConversation: emptyWelcomeConvo, - http: { - basePath: { - basePath: '/mfg', - serverBasePath: '/mfg', - }, - anonymousPaths: {}, - externalUrl: {}, - } as unknown as HttpSetup, - editingSystemPromptId: defaultSystemPrompt.id, - setEditingSystemPromptId: () => {}, - setPromptTextPreview: () => {}, - setSelectedPromptContexts: () => {}, - setUserPrompt: () => {}, - isDisabled: false, - shouldRefocusPrompt: false, - userPrompt: '', -}; const handleButtonSendMessage = jest.fn(); const handleOnChatCleared = jest.fn(); const handlePromptChange = jest.fn(); const handleSendMessage = jest.fn(); -const chatSend = { +const handleRegenerateResponse = jest.fn(); +const testProps: Props = { handleButtonSendMessage, handleOnChatCleared, handlePromptChange, handleSendMessage, + handleRegenerateResponse, isLoading: false, + isDisabled: false, + shouldRefocusPrompt: false, + userPrompt: '', }; - describe('ChatSend', () => { beforeEach(() => { jest.clearAllMocks(); - (useChatSend as jest.Mock).mockReturnValue(chatSend); }); it('the prompt updates when the text area changes', async () => { const { getByTestId } = render(, { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.tsx index 88db2e124ceab5..fe8a2be756047d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/index.tsx @@ -8,11 +8,11 @@ import React, { useCallback, useEffect, useMemo, useRef } from 'react'; import { css } from '@emotion/react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { useChatSend, UseChatSendProps } from './use_chat_send'; +import { UseChatSend } from './use_chat_send'; import { ChatActions } from '../chat_actions'; import { PromptTextArea } from '../prompt_textarea'; -export interface Props extends UseChatSendProps { +export interface Props extends UseChatSend { isDisabled: boolean; shouldRefocusPrompt: boolean; userPrompt: string | null; @@ -23,18 +23,15 @@ export interface Props extends UseChatSendProps { * Allows the user to clear the chat and switch between different system prompts. */ export const ChatSend: React.FC = ({ + handleButtonSendMessage, + handleOnChatCleared, + handlePromptChange, + handleSendMessage, isDisabled, - userPrompt, + isLoading, shouldRefocusPrompt, - ...rest + userPrompt, }) => { - const { - handleButtonSendMessage, - handleOnChatCleared, - handlePromptChange, - handleSendMessage, - isLoading, - } = useChatSend(rest); // For auto-focusing prompt within timeline const promptTextAreaRef = useRef(null); useEffect(() => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx index 0b139177d02b06..8dfa8699048ac0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.test.tsx @@ -23,6 +23,7 @@ const setSelectedPromptContexts = jest.fn(); const setUserPrompt = jest.fn(); const sendMessages = jest.fn(); const appendMessage = jest.fn(); +const removeLastMessage = jest.fn(); const appendReplacements = jest.fn(); const clearConversation = jest.fn(); @@ -55,6 +56,7 @@ describe('use chat send', () => { (useConversation as jest.Mock).mockReturnValue({ appendMessage, appendReplacements, + removeLastMessage, clearConversation, }); }); @@ -106,4 +108,17 @@ describe('use chat send', () => { expect(appendMessage.mock.calls[0][0].message.content).toEqual(`\n\n${promptText}`); }); }); + it('handleRegenerateResponse removes the last message of the conversation, resends the convo to GenAI, and appends the message received', async () => { + const { result } = renderHook(() => + useChatSend({ ...testProps, currentConversation: welcomeConvo }) + ); + + result.current.handleRegenerateResponse(); + expect(removeLastMessage).toHaveBeenCalledWith('Welcome'); + + await waitFor(() => { + expect(sendMessages).toHaveBeenCalled(); + expect(appendMessage.mock.calls[0][0].message.content).toEqual(robotMessage.response); + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx index 3e1c1940978888..83a2ad33e55fb9 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/chat_send/use_chat_send.tsx @@ -29,11 +29,12 @@ export interface UseChatSendProps { setUserPrompt: React.Dispatch>; } -interface UseChatSend { +export interface UseChatSend { handleButtonSendMessage: (m: string) => void; handleOnChatCleared: () => void; handlePromptChange: (prompt: string) => void; handleSendMessage: (promptText: string) => void; + handleRegenerateResponse: () => void; isLoading: boolean; } @@ -54,7 +55,9 @@ export const useChatSend = ({ setUserPrompt, }: UseChatSendProps): UseChatSend => { const { isLoading, sendMessages } = useSendMessages(); - const { appendMessage, appendReplacements, clearConversation } = useConversation(); + + const { appendMessage, appendReplacements, clearConversation, removeLastMessage } = + useConversation(); const handlePromptChange = (prompt: string) => { setPromptTextPreview(prompt); @@ -112,6 +115,24 @@ export const useChatSend = ({ ] ); + const handleRegenerateResponse = useCallback(async () => { + const updatedMessages = removeLastMessage(currentConversation.id); + const rawResponse = await sendMessages({ + http, + apiConfig: currentConversation.apiConfig, + messages: updatedMessages, + }); + const responseMessage: Message = getMessageFromRawResponse(rawResponse); + appendMessage({ conversationId: currentConversation.id, message: responseMessage }); + }, [ + appendMessage, + currentConversation.apiConfig, + currentConversation.id, + http, + removeLastMessage, + sendMessages, + ]); + const handleButtonSendMessage = useCallback( (message: string) => { handleSendMessage(message); @@ -146,6 +167,7 @@ export const useChatSend = ({ handleOnChatCleared, handlePromptChange, handleSendMessage, + handleRegenerateResponse, isLoading, }; }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index b4eb89a092600a..61001d95e8a3ec 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -12,12 +12,14 @@ import type { Message } from '../assistant_context/types'; import { enterpriseMessaging, WELCOME_CONVERSATION } from './use_conversation/sample_conversations'; export const getMessageFromRawResponse = (rawResponse: FetchConnectorExecuteResponse): Message => { - const { response, isError } = rawResponse; + const { response, isStream, isError } = rawResponse; const dateTimeString = new Date().toLocaleString(); // TODO: Pull from response if (rawResponse) { return { role: 'assistant', - content: response, + ...(isStream + ? { reader: response as ReadableStreamDefaultReader } + : { content: response as string }), timestamp: dateTimeString, isError, }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index cdfd8187e7a2f3..86e0f3a460055d 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -31,6 +31,7 @@ import { css } from '@emotion/react'; import { OpenAiProviderType } from '@kbn/stack-connectors-plugin/common/openai/constants'; import { ActionConnectorProps } from '@kbn/triggers-actions-ui-plugin/public/types'; +import { useChatSend } from './chat_send/use_chat_send'; import { ChatSend } from './chat_send'; import { BlockBotCallToAction } from './block_bot/cta'; import { AssistantHeader } from './assistant_header'; @@ -51,6 +52,7 @@ import { ConnectorMissingCallout } from '../connectorland/connector_missing_call export interface Props { conversationId?: string; + embeddedLayout?: boolean; promptContextId?: string; shouldRefocusPrompt?: boolean; showTitle?: boolean; @@ -63,6 +65,7 @@ export interface Props { */ const AssistantComponent: React.FC = ({ conversationId, + embeddedLayout = false, promptContextId = '', shouldRefocusPrompt = false, showTitle = true, @@ -93,7 +96,7 @@ const AssistantComponent: React.FC = ({ [selectedPromptContexts] ); - const { createConversation } = useConversation(); + const { amendMessage, createConversation } = useConversation(); // Connector details const { data: connectors, isSuccess: areConnectorsFetched } = useLoadConnectors({ http }); @@ -166,17 +169,11 @@ const AssistantComponent: React.FC = ({ const { comments: connectorComments, prompt: connectorPrompt } = useConnectorSetup({ conversation: blockBotConversation, - onSetupComplete: () => { - bottomRef.current?.scrollIntoView({ behavior: 'auto' }); - }, }); const currentTitle: string | JSX.Element = isWelcomeSetup && blockBotConversation.theme?.title ? blockBotConversation.theme?.title : title; - const bottomRef = useRef(null); - const lastCommentRef = useRef(null); - const [promptTextPreview, setPromptTextPreview] = useState(''); const [autoPopulatedOnce, setAutoPopulatedOnce] = useState(false); const [userPrompt, setUserPrompt] = useState(null); @@ -188,7 +185,10 @@ const AssistantComponent: React.FC = ({ const [messageCodeBlocks, setMessageCodeBlocks] = useState(); const [_, setCodeBlockControlsVisible] = useState(false); useLayoutEffect(() => { - setMessageCodeBlocks(augmentMessageCodeBlocks(currentConversation)); + // need in order for code block controls to be added to the DOM + setTimeout(() => { + setMessageCodeBlocks(augmentMessageCodeBlocks(currentConversation)); + }, 0); }, [augmentMessageCodeBlocks, currentConversation]); const isSendingDisabled = useMemo(() => { @@ -213,17 +213,22 @@ const AssistantComponent: React.FC = ({ }, []); // End drill in `Add To Timeline` action - // Scroll to bottom on conversation change - useEffect(() => { - bottomRef.current?.scrollIntoView({ behavior: 'auto' }); - }, []); + // Start Scrolling + const commentsContainerRef = useRef(null); + useEffect(() => { - setTimeout(() => { - bottomRef.current?.scrollIntoView({ behavior: 'auto' }); - }, 0); - }, [currentConversation.messages.length, selectedPromptContextsCount]); - //// - // + const parent = commentsContainerRef.current?.parentElement; + if (!parent) { + return; + } + // when scrollHeight changes, parent is scrolled to bottom + parent.scrollTop = parent.scrollHeight; + }); + + const getWrapper = (children: React.ReactNode, isCommentContainer: boolean) => + isCommentContainer ? {children} : <>{children}; + + // End Scrolling const selectedSystemPrompt = useMemo( () => getDefaultSystemPrompt({ allSystemPrompts, conversation: currentConversation }), @@ -321,24 +326,53 @@ const AssistantComponent: React.FC = ({ const createCodeBlockPortals = useCallback( () => - messageCodeBlocks?.map((codeBlocks: CodeBlockDetails[]) => { - return codeBlocks.map((codeBlock: CodeBlockDetails) => { - const getElement = codeBlock.getControlContainer; - const element = getElement?.(); - return element ? createPortal(codeBlock.button, element) : <>; - }); + messageCodeBlocks?.map((codeBlocks: CodeBlockDetails[], i: number) => { + return ( + + {codeBlocks.map((codeBlock: CodeBlockDetails, j: number) => { + const getElement = codeBlock.getControlContainer; + const element = getElement?.(); + return ( + + {element ? createPortal(codeBlock.button, element) : <>} + + ); + })} + + ); }), [messageCodeBlocks] ); + const { + handleButtonSendMessage, + handleOnChatCleared, + handlePromptChange, + handleSendMessage, + handleRegenerateResponse, + isLoading: isLoadingChatSend, + } = useChatSend({ + allSystemPrompts, + currentConversation, + setPromptTextPreview, + setUserPrompt, + editingSystemPromptId, + http, + setEditingSystemPromptId, + selectedPromptContexts, + setSelectedPromptContexts, + }); + const chatbotComments = useMemo( () => ( <> = ({ setSelectedPromptContexts={setSelectedPromptContexts} /> )} - -
), [ + amendMessage, currentConversation, editingSystemPromptId, getComments, handleOnSystemPromptSelectionChange, + handleRegenerateResponse, + isLoadingChatSend, isSettingsModalVisible, promptContexts, promptTextPreview, @@ -384,15 +419,12 @@ const AssistantComponent: React.FC = ({ const comments = useMemo(() => { if (isDisabled) { return ( - <> - - - + ); } @@ -409,7 +441,7 @@ const AssistantComponent: React.FC = ({ [assistantTelemetry, selectedConversationId] ); - return ( + return getWrapper( <> = ({ )} - {comments} - - {!isDisabled && showMissingConnectorCallout && areConnectorsFetched && ( + {getWrapper( <> - - - - 0} - isSettingsModalVisible={isSettingsModalVisible} - setIsSettingsModalVisible={setIsSettingsModalVisible} - /> - - - + {comments} + + {!isDisabled && showMissingConnectorCallout && areConnectorsFetched && ( + <> + + + + 0} + isSettingsModalVisible={isSettingsModalVisible} + setIsSettingsModalVisible={setIsSettingsModalVisible} + /> + + + + )} + , + !embeddedLayout )} = ({ isWelcomeSetup={isWelcomeSetup} /> {!isDisabled && ( = ({ /> )} - + , + embeddedLayout ); }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx index d52460bd95a0fe..562a252bf81110 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.test.tsx @@ -16,10 +16,15 @@ const message = { role: 'user' as ConversationRole, timestamp: '10/04/2023, 1:00:36 PM', }; +const anotherMessage = { + content: 'I am a robot', + role: 'assistant' as ConversationRole, + timestamp: '10/04/2023, 1:00:46 PM', +}; const mockConvo = { id: 'new-convo', - messages: [message], + messages: [message, anotherMessage], apiConfig: { defaultSystemPromptId: 'default-system-prompt' }, theme: { title: 'Elastic AI Assistant', @@ -253,4 +258,79 @@ describe('useConversation', () => { }); }); }); + + it('should remove the last message from a conversation when called with valid conversationId', async () => { + await act(async () => { + const setConversations = jest.fn(); + const { result, waitForNextUpdate } = renderHook(() => useConversation(), { + wrapper: ({ children }) => ( + ({ + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, + [mockConvo.id]: mockConvo, + }), + setConversations, + }} + > + {children} + + ), + }); + await waitForNextUpdate(); + + const removeResult = result.current.removeLastMessage('new-convo'); + + expect(removeResult).toEqual([message]); + expect(setConversations).toHaveBeenCalledWith({ + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, + [mockConvo.id]: { ...mockConvo, messages: [message] }, + }); + }); + }); + + it('amendMessage updates the last message of conversation[] for a given conversationId with provided content', async () => { + await act(async () => { + const setConversations = jest.fn(); + const { result, waitForNextUpdate } = renderHook(() => useConversation(), { + wrapper: ({ children }) => ( + ({ + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, + [mockConvo.id]: mockConvo, + }), + setConversations, + }} + > + {children} + + ), + }); + await waitForNextUpdate(); + + result.current.amendMessage({ + conversationId: 'new-convo', + content: 'hello world', + }); + + expect(setConversations).toHaveBeenCalledWith({ + [alertConvo.id]: alertConvo, + [welcomeConvo.id]: welcomeConvo, + [mockConvo.id]: { + ...mockConvo, + messages: [ + message, + { + ...anotherMessage, + content: 'hello world', + }, + ], + }, + }); + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx index 0cf4a4bdc94399..3bd9f3fcbff716 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/index.tsx @@ -35,6 +35,10 @@ interface AppendMessageProps { conversationId: string; message: Message; } +interface AmendMessageProps { + conversationId: string; + content: string; +} interface AppendReplacementsProps { conversationId: string; @@ -56,7 +60,8 @@ interface SetConversationProps { } interface UseConversation { - appendMessage: ({ conversationId: string, message: Message }: AppendMessageProps) => Message[]; + appendMessage: ({ conversationId, message }: AppendMessageProps) => Message[]; + amendMessage: ({ conversationId, content }: AmendMessageProps) => void; appendReplacements: ({ conversationId, replacements, @@ -64,6 +69,7 @@ interface UseConversation { clearConversation: (conversationId: string) => void; createConversation: ({ conversationId, messages }: CreateConversationProps) => Conversation; deleteConversation: (conversationId: string) => void; + removeLastMessage: (conversationId: string) => Message[]; setApiConfig: ({ conversationId, apiConfig }: SetApiConfigProps) => void; setConversation: ({ conversation }: SetConversationProps) => void; } @@ -71,6 +77,64 @@ interface UseConversation { export const useConversation = (): UseConversation => { const { allSystemPrompts, assistantTelemetry, setConversations } = useAssistantContext(); + /** + * Removes the last message of conversation[] for a given conversationId + */ + const removeLastMessage = useCallback( + (conversationId: string) => { + let messages: Message[] = []; + setConversations((prev: Record) => { + const prevConversation: Conversation | undefined = prev[conversationId]; + + if (prevConversation != null) { + messages = prevConversation.messages.slice(0, prevConversation.messages.length - 1); + const newConversation = { + ...prevConversation, + messages, + }; + return { + ...prev, + [conversationId]: newConversation, + }; + } else { + return prev; + } + }); + return messages; + }, + [setConversations] + ); + + /** + * Updates the last message of conversation[] for a given conversationId with provided content + */ + const amendMessage = useCallback( + ({ conversationId, content }: AmendMessageProps) => { + setConversations((prev: Record) => { + const prevConversation: Conversation | undefined = prev[conversationId]; + + if (prevConversation != null) { + const { messages, ...rest } = prevConversation; + const message = messages[messages.length - 1]; + const updatedMessages = message + ? [...messages.slice(0, -1), { ...message, content }] + : [...messages]; + const newConversation = { + ...rest, + messages: updatedMessages, + }; + return { + ...prev, + [conversationId]: newConversation, + }; + } else { + return prev; + } + }); + }, + [setConversations] + ); + /** * Append a message to the conversation[] for a given conversationId */ @@ -262,11 +326,13 @@ export const useConversation = (): UseConversation => { ); return { + amendMessage, appendMessage, appendReplacements, clearConversation, createConversation, deleteConversation, + removeLastMessage, setApiConfig, setConversation, }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index f296246736a787..dd508b54ce9066 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -64,12 +64,22 @@ export interface AssistantProviderProps { docLinks: Omit; children: React.ReactNode; getComments: ({ + amendMessage, currentConversation, - lastCommentRef, + isFetchingResponse, + regenerateMessage, showAnonymizedValues, }: { + amendMessage: ({ + conversationId, + content, + }: { + conversationId: string; + content: string; + }) => void; currentConversation: Conversation; - lastCommentRef: React.MutableRefObject; + isFetchingResponse: boolean; + regenerateMessage: (conversationId: string) => void; showAnonymizedValues: boolean; }) => EuiCommentProps[]; http: HttpSetup; @@ -102,12 +112,20 @@ export interface UseAssistantContext { conversations: Record; getComments: ({ currentConversation, - lastCommentRef, showAnonymizedValues, + amendMessage, + isFetchingResponse, }: { currentConversation: Conversation; - lastCommentRef: React.MutableRefObject; - + isFetchingResponse: boolean; + amendMessage: ({ + conversationId, + content, + }: { + conversationId: string; + content: string; + }) => void; + regenerateMessage: () => void; showAnonymizedValues: boolean; }) => EuiCommentProps[]; http: HttpSetup; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx index 651eeee17f21e5..f9ade4b9abc1ee 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/types.tsx @@ -15,7 +15,8 @@ export interface MessagePresentation { } export interface Message { role: ConversationRole; - content: string; + reader?: ReadableStreamDefaultReader; + content?: string; timestamp: string; isError?: boolean; presentation?: MessagePresentation; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx index e51d356679bc30..c80cfe0ed10077 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/connectorland/connector_setup/index.tsx @@ -131,7 +131,7 @@ export const useConnectorSetup = ({ (message?.presentation?.stream ?? false) && currentMessageIndex !== length - 1; return ( ; export type CasesBulkGetResponse = rt.TypeOf; export type GetRelatedCasesByAlertResponse = rt.TypeOf; export type CaseRequestCustomFields = rt.TypeOf; +export type BulkCreateCasesRequest = rt.TypeOf; +export type BulkCreateCasesResponse = rt.TypeOf; diff --git a/x-pack/plugins/cases/server/client/attachments/delete.ts b/x-pack/plugins/cases/server/client/attachments/delete.ts index b21c6b5ae753f1..e67a5788b34766 100644 --- a/x-pack/plugins/cases/server/client/attachments/delete.ts +++ b/x-pack/plugins/cases/server/client/attachments/delete.ts @@ -124,13 +124,15 @@ export async function deleteComment( const attachmentRequestAttributes = decodeOrThrow(AttachmentRequestRt)(attachment.attributes); await userActionService.creator.createUserAction({ - type: UserActionTypes.comment, - action: UserActionActions.delete, - caseId: id, - attachmentId: attachmentID, - payload: { attachment: attachmentRequestAttributes }, - user, - owner: attachment.attributes.owner, + userAction: { + type: UserActionTypes.comment, + action: UserActionActions.delete, + caseId: id, + attachmentId: attachmentID, + payload: { attachment: attachmentRequestAttributes }, + user, + owner: attachment.attributes.owner, + }, }); await handleAlerts({ alertsService, attachments: [attachment.attributes], caseId: id }); diff --git a/x-pack/plugins/cases/server/client/cases/bulk_create.test.ts b/x-pack/plugins/cases/server/client/cases/bulk_create.test.ts new file mode 100644 index 00000000000000..fa0b4e3f584e58 --- /dev/null +++ b/x-pack/plugins/cases/server/client/cases/bulk_create.test.ts @@ -0,0 +1,1231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MAX_DESCRIPTION_LENGTH, + MAX_TAGS_PER_CASE, + MAX_LENGTH_PER_TAG, + MAX_TITLE_LENGTH, + MAX_ASSIGNEES_PER_CASE, + MAX_CUSTOM_FIELDS_PER_CASE, +} from '../../../common/constants'; +import type { CasePostRequest } from '../../../common'; +import { SECURITY_SOLUTION_OWNER } from '../../../common'; +import { mockCases } from '../../mocks'; +import { createCasesClientMock, createCasesClientMockArgs } from '../mocks'; +import { bulkCreate } from './bulk_create'; +import { CaseSeverity, ConnectorTypes, CustomFieldTypes } from '../../../common/types/domain'; + +import type { CaseCustomFields } from '../../../common/types/domain'; +import { omit } from 'lodash'; + +jest.mock('@kbn/core-saved-objects-utils-server', () => { + const actual = jest.requireActual('@kbn/core-saved-objects-utils-server'); + + return { + ...actual, + SavedObjectsUtils: { + generateId: () => 'mock-saved-object-id', + }, + }; +}); + +describe('bulkCreate', () => { + const getCases = (overrides = {}) => [ + { + title: 'My Case', + tags: [], + description: 'testing sir', + connector: { + id: '.none', + name: 'None', + type: ConnectorTypes.none, + fields: null, + }, + settings: { syncAlerts: true }, + severity: CaseSeverity.LOW, + owner: SECURITY_SOLUTION_OWNER, + assignees: [{ uid: '1' }], + ...overrides, + }, + ]; + + const caseSO = mockCases[0]; + const casesClientMock = createCasesClientMock(); + casesClientMock.configure.get = jest.fn().mockResolvedValue([]); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('execution', () => { + const createdAtDate = new Date('2023-11-05'); + + beforeAll(() => { + jest.useFakeTimers(); + jest.setSystemTime(createdAtDate); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + const clientArgs = createCasesClientMockArgs(); + + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [caseSO], + }); + + it('create the cases correctly', async () => { + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [ + caseSO, + { ...caseSO, attributes: { ...caseSO.attributes, severity: CaseSeverity.CRITICAL } }, + ], + }); + + const res = await bulkCreate( + { cases: [getCases()[0], getCases({ severity: CaseSeverity.CRITICAL })[0]] }, + clientArgs, + casesClientMock + ); + + expect(res).toMatchInlineSnapshot(` + Object { + "cases": Array [ + Object { + "assignees": Array [], + "category": null, + "closed_at": null, + "closed_by": null, + "comments": Array [], + "connector": Object { + "fields": null, + "id": "none", + "name": "none", + "type": ".none", + }, + "created_at": "2019-11-25T21:54:48.952Z", + "created_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "customFields": Array [], + "description": "This is a brand new case of a bad meanie defacing data", + "duration": null, + "external_service": null, + "id": "mock-id-1", + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "low", + "status": "open", + "tags": Array [ + "defacement", + ], + "title": "Super Bad Security Issue", + "totalAlerts": 0, + "totalComment": 0, + "updated_at": "2019-11-25T21:54:48.952Z", + "updated_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "version": "WzAsMV0=", + }, + Object { + "assignees": Array [], + "category": null, + "closed_at": null, + "closed_by": null, + "comments": Array [], + "connector": Object { + "fields": null, + "id": "none", + "name": "none", + "type": ".none", + }, + "created_at": "2019-11-25T21:54:48.952Z", + "created_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "customFields": Array [], + "description": "This is a brand new case of a bad meanie defacing data", + "duration": null, + "external_service": null, + "id": "mock-id-1", + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "critical", + "status": "open", + "tags": Array [ + "defacement", + ], + "title": "Super Bad Security Issue", + "totalAlerts": 0, + "totalComment": 0, + "updated_at": "2019-11-25T21:54:48.952Z", + "updated_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "version": "WzAsMV0=", + }, + ], + } + `); + }); + + it('accepts an ID in the request correctly', async () => { + await bulkCreate({ cases: getCases({ id: 'my-id' }) }, clientArgs, casesClientMock); + + expect(clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].id).toBe( + 'my-id' + ); + }); + + it('generates an ID if not provided in the request', async () => { + await bulkCreate({ cases: getCases() }, clientArgs, casesClientMock); + + expect(clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].id).toBe( + 'mock-saved-object-id' + ); + }); + + it('calls bulkCreateCases correctly', async () => { + await bulkCreate( + { cases: [getCases()[0], getCases({ severity: CaseSeverity.CRITICAL })[0]] }, + clientArgs, + casesClientMock + ); + + expect(clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0]) + .toMatchInlineSnapshot(` + Object { + "cases": Array [ + Object { + "assignees": Array [ + Object { + "uid": "1", + }, + ], + "category": null, + "closed_at": null, + "closed_by": null, + "connector": Object { + "fields": null, + "id": ".none", + "name": "None", + "type": ".none", + }, + "created_at": "2023-11-05T00:00:00.000Z", + "created_by": Object { + "email": "damaged_raccoon@elastic.co", + "full_name": "Damaged Raccoon", + "profile_uid": "u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0", + "username": "damaged_raccoon", + }, + "customFields": Array [], + "description": "testing sir", + "duration": null, + "external_service": null, + "id": "mock-saved-object-id", + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "low", + "status": "open", + "tags": Array [], + "title": "My Case", + "updated_at": null, + "updated_by": null, + }, + Object { + "assignees": Array [ + Object { + "uid": "1", + }, + ], + "category": null, + "closed_at": null, + "closed_by": null, + "connector": Object { + "fields": null, + "id": ".none", + "name": "None", + "type": ".none", + }, + "created_at": "2023-11-05T00:00:00.000Z", + "created_by": Object { + "email": "damaged_raccoon@elastic.co", + "full_name": "Damaged Raccoon", + "profile_uid": "u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0", + "username": "damaged_raccoon", + }, + "customFields": Array [], + "description": "testing sir", + "duration": null, + "external_service": null, + "id": "mock-saved-object-id", + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "critical", + "status": "open", + "tags": Array [], + "title": "My Case", + "updated_at": null, + "updated_by": null, + }, + ], + "refresh": false, + } + `); + }); + + it('throws an error if bulkCreateCases returns at least one error ', async () => { + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [ + caseSO, + { + id: '2', + type: 'cases', + error: { + error: 'My error', + message: 'not found', + statusCode: 404, + }, + references: [], + }, + { + id: '3', + type: 'cases', + error: { + error: 'My second error', + message: 'conflict', + statusCode: 409, + }, + references: [], + }, + ], + }); + + await expect(bulkCreate({ cases: getCases() }, clientArgs, casesClientMock)).rejects.toThrow( + `Failed to bulk create cases: Error: My error` + ); + }); + + it('constructs the case error correctly', async () => { + expect.assertions(1); + + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [ + caseSO, + { + id: '1', + type: 'cases', + error: { + error: 'My error', + message: 'not found', + statusCode: 404, + }, + references: [], + }, + ], + }); + + try { + await bulkCreate({ cases: getCases() }, clientArgs, casesClientMock); + } catch (error) { + expect(error.wrappedError.output).toEqual({ + headers: {}, + payload: { error: 'Not Found', message: 'My error', statusCode: 404 }, + statusCode: 404, + }); + } + }); + }); + + describe('authorization', () => { + const clientArgs = createCasesClientMockArgs(); + + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [caseSO], + }); + + it('validates the cases correctly', async () => { + await bulkCreate( + { cases: [getCases()[0], getCases({ owner: 'cases' })[0]] }, + clientArgs, + casesClientMock + ); + + expect(clientArgs.authorization.ensureAuthorized).toHaveBeenCalledWith({ + entities: [ + { id: 'mock-saved-object-id', owner: 'securitySolution' }, + { id: 'mock-saved-object-id', owner: 'cases' }, + ], + operation: { + action: 'case_create', + docType: 'case', + ecsType: 'creation', + name: 'createCase', + savedObjectType: 'cases', + verbs: { past: 'created', present: 'create', progressive: 'creating' }, + }, + }); + }); + }); + + describe('Assignees', () => { + const clientArgs = createCasesClientMockArgs(); + + it('notifies single assignees', async () => { + const caseSOWithAssignees = { + ...caseSO, + attributes: { ...caseSO.attributes, assignees: [{ uid: '1' }] }, + }; + + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [caseSOWithAssignees], + }); + + const cases = getCases(); + + await bulkCreate({ cases }, clientArgs, casesClientMock); + + expect(clientArgs.services.notificationService.bulkNotifyAssignees).toHaveBeenCalledWith([ + { + assignees: cases[0].assignees, + theCase: caseSOWithAssignees, + }, + ]); + }); + + it('notifies multiple assignees', async () => { + const caseSOWithAssignees = { + ...caseSO, + attributes: { ...caseSO.attributes, assignees: [{ uid: '1' }, { uid: '2' }] }, + }; + + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [caseSOWithAssignees], + }); + + await bulkCreate( + { cases: getCases({ assignees: [{ uid: '1' }, { uid: '2' }] }) }, + clientArgs, + casesClientMock + ); + + expect(clientArgs.services.notificationService.bulkNotifyAssignees).toHaveBeenCalledWith([ + { + assignees: [{ uid: '1' }, { uid: '2' }], + theCase: caseSOWithAssignees, + }, + ]); + }); + + it('does not notify when there are no assignees', async () => { + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [caseSO], + }); + + await bulkCreate({ cases: getCases({ assignees: [] }) }, clientArgs, casesClientMock); + + expect(clientArgs.services.notificationService.bulkNotifyAssignees).not.toHaveBeenCalled(); + }); + + it('does not notify the current user', async () => { + const caseSOWithAssignees = { + ...caseSO, + attributes: { + ...caseSO.attributes, + assignees: [{ uid: '1' }, { uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0' }], + }, + }; + + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ + saved_objects: [caseSOWithAssignees], + }); + + await bulkCreate( + { + cases: getCases({ + assignees: [{ uid: '1' }, { uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0' }], + }), + }, + clientArgs, + casesClientMock + ); + + expect(clientArgs.services.notificationService.bulkNotifyAssignees).toHaveBeenCalledWith([ + { + assignees: [{ uid: '1' }], + theCase: caseSOWithAssignees, + }, + ]); + }); + + it('should throw an error if the assignees array length is too long', async () => { + const assignees = Array(MAX_ASSIGNEES_PER_CASE + 1).fill({ uid: 'foo' }); + + await expect( + bulkCreate({ cases: getCases({ assignees }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + `Failed to bulk create cases: Error: The length of the field assignees is too long. Array must be of length <= ${MAX_ASSIGNEES_PER_CASE}.` + ); + }); + + it('should throw if the user does not have the correct license', async () => { + clientArgs.services.licensingService.isAtLeastPlatinum.mockResolvedValue(false); + + await expect(bulkCreate({ cases: getCases() }, clientArgs, casesClientMock)).rejects.toThrow( + `Failed to bulk create cases: Error: In order to assign users to cases, you must be subscribed to an Elastic Platinum license` + ); + }); + }); + + describe('Attributes', () => { + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + + it('should throw an error when an excess field exists', async () => { + await expect( + bulkCreate({ cases: getCases({ foo: 'bar' }) }, clientArgs, casesClientMock) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: invalid keys \\"foo\\""` + ); + }); + }); + + describe('title', () => { + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + + it(`should not throw an error if the title is non empty and less than ${MAX_TITLE_LENGTH} characters`, async () => { + await expect( + bulkCreate( + { cases: getCases({ title: 'This is a test case!!' }) }, + clientArgs, + casesClientMock + ) + ).resolves.not.toThrow(); + }); + + it('should throw an error if the title length is too long', async () => { + await expect( + bulkCreate( + { + cases: getCases({ + title: + 'This is a very long title with more than one hundred and sixty characters!! To confirm the maximum limit error thrown for more than one hundred and sixty characters!!', + }), + }, + clientArgs, + casesClientMock + ) + ).rejects.toThrow( + `Failed to bulk create cases: Error: The length of the title is too long. The maximum length is ${MAX_TITLE_LENGTH}.` + ); + }); + + it('should throw an error if the title is an empty string', async () => { + await expect( + bulkCreate({ cases: getCases({ title: '' }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The title field cannot be an empty string.' + ); + }); + + it('should throw an error if the title is a string with empty characters', async () => { + await expect( + bulkCreate({ cases: getCases({ title: ' ' }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The title field cannot be an empty string.' + ); + }); + + it('should trim title', async () => { + await bulkCreate( + { cases: getCases({ title: 'title with spaces ' }) }, + clientArgs, + casesClientMock + ); + + const title = clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].title; + + expect(title).toBe('title with spaces'); + }); + }); + + describe('description', () => { + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + + it(`should not throw an error if the description is non empty and less than ${MAX_DESCRIPTION_LENGTH} characters`, async () => { + await expect( + bulkCreate( + { cases: getCases({ description: 'This is a test description!!' }) }, + clientArgs, + casesClientMock + ) + ).resolves.not.toThrow(); + }); + + it('should throw an error if the description length is too long', async () => { + const description = Array(MAX_DESCRIPTION_LENGTH + 1) + .fill('x') + .toString(); + + await expect( + bulkCreate({ cases: getCases({ description }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + `Failed to bulk create cases: Error: The length of the description is too long. The maximum length is ${MAX_DESCRIPTION_LENGTH}.` + ); + }); + + it('should throw an error if the description is an empty string', async () => { + await expect( + bulkCreate({ cases: getCases({ description: '' }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The description field cannot be an empty string.' + ); + }); + + it('should throw an error if the description is a string with empty characters', async () => { + await expect( + bulkCreate({ cases: getCases({ description: ' ' }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The description field cannot be an empty string.' + ); + }); + + it('should trim description', async () => { + await bulkCreate( + { cases: getCases({ description: 'this is a description with spaces!! ' }) }, + clientArgs, + casesClientMock + ); + + const description = + clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].description; + + expect(description).toBe('this is a description with spaces!!'); + }); + }); + + describe('tags', () => { + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + + it('should not throw an error if the tags array is empty', async () => { + await expect( + bulkCreate({ cases: getCases({ tags: [] }) }, clientArgs, casesClientMock) + ).resolves.not.toThrow(); + }); + + it('should not throw an error if the tags array has non empty string within limit', async () => { + await expect( + bulkCreate({ cases: getCases({ tags: ['abc'] }) }, clientArgs, casesClientMock) + ).resolves.not.toThrow(); + }); + + it('should throw an error if the tags array length is too long', async () => { + const tags = Array(MAX_TAGS_PER_CASE + 1).fill('foo'); + + await expect( + bulkCreate({ cases: getCases({ tags }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + `Failed to bulk create cases: Error: The length of the field tags is too long. Array must be of length <= ${MAX_TAGS_PER_CASE}.` + ); + }); + + it('should throw an error if the tags array has empty string', async () => { + await expect( + bulkCreate({ cases: getCases({ tags: [''] }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The tag field cannot be an empty string.' + ); + }); + + it('should throw an error if the tags array has string with empty characters', async () => { + await expect( + bulkCreate({ cases: getCases({ tags: [' '] }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The tag field cannot be an empty string.' + ); + }); + + it('should throw an error if the tag length is too long', async () => { + const tag = Array(MAX_LENGTH_PER_TAG + 1) + .fill('f') + .toString(); + + await expect( + bulkCreate({ cases: getCases({ tags: [tag] }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + `Failed to bulk create cases: Error: The length of the tag is too long. The maximum length is ${MAX_LENGTH_PER_TAG}.` + ); + }); + + it('should trim tags', async () => { + await bulkCreate( + { cases: getCases({ tags: ['pepsi ', 'coke'] }) }, + clientArgs, + casesClientMock + ); + + const tags = clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].tags; + + expect(tags).toEqual(['pepsi', 'coke']); + }); + }); + + describe('Category', () => { + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + + it('should not throw an error if the category is null', async () => { + await expect( + bulkCreate({ cases: getCases({ category: null }) }, clientArgs, casesClientMock) + ).resolves.not.toThrow(); + }); + + it('should throw an error if the category length is too long', async () => { + await expect( + bulkCreate( + { + cases: getCases({ category: 'A very long category with more than fifty characters!' }), + }, + clientArgs, + casesClientMock + ) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The length of the category is too long.' + ); + }); + + it('should throw an error if the category is an empty string', async () => { + await expect( + bulkCreate({ cases: getCases({ category: '' }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The category field cannot be an empty string.,Invalid value "" supplied to "cases,category"' + ); + }); + + it('should throw an error if the category is a string with empty characters', async () => { + await expect( + bulkCreate({ cases: getCases({ category: ' ' }) }, clientArgs, casesClientMock) + ).rejects.toThrow( + 'Failed to bulk create cases: Error: The category field cannot be an empty string.,Invalid value " " supplied to "cases,category"' + ); + }); + + it('should trim category', async () => { + await bulkCreate( + { cases: getCases({ category: 'reporting ' }) }, + clientArgs, + casesClientMock + ); + + const category = + clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].category; + + expect(category).toEqual('reporting'); + }); + }); + + describe('Custom Fields', () => { + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + const theCase = getCases()[0]; + + const casesClient = createCasesClientMock(); + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: theCase.owner, + customFields: [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + label: 'foo', + required: true, + }, + { + key: 'second_key', + type: CustomFieldTypes.TOGGLE, + label: 'foo', + required: false, + }, + ], + }, + ]); + + const theCustomFields: CaseCustomFields = [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + value: 'this is a text field value', + }, + { + key: 'second_key', + type: CustomFieldTypes.TOGGLE, + value: true, + }, + ]; + + it('should bulkCreate customFields correctly', async () => { + await expect( + bulkCreate({ cases: getCases({ customFields: theCustomFields }) }, clientArgs, casesClient) + ).resolves.not.toThrow(); + + const customFields = + clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].customFields; + + expect(customFields).toEqual(theCustomFields); + }); + + it('should not throw an error and fill out missing customFields when they are undefined', async () => { + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: theCase.owner, + customFields: [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + label: 'foo', + required: false, + }, + { + key: 'second_key', + type: CustomFieldTypes.TOGGLE, + label: 'foo', + required: false, + }, + ], + }, + ]); + + await expect( + bulkCreate({ cases: getCases() }, clientArgs, casesClient) + ).resolves.not.toThrow(); + + const customFields = + clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases[0].customFields; + + expect(customFields).toEqual([ + { key: 'first_key', type: 'text', value: null }, + { key: 'second_key', type: 'toggle', value: null }, + ]); + }); + + it('should throw an error when required customFields are undefined', async () => { + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: theCase.owner, + customFields: [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + label: 'missing field 1', + required: true, + }, + { + key: 'second_key', + type: CustomFieldTypes.TOGGLE, + label: 'foo', + required: false, + }, + ], + }, + ]); + + await expect( + bulkCreate({ cases: getCases() }, clientArgs, casesClient) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: Missing required custom fields: \\"missing field 1\\""` + ); + }); + + it('should throw an error when required customFields are null', async () => { + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: theCase.owner, + customFields: [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + label: 'missing field 1', + required: true, + }, + { + key: 'second_key', + type: CustomFieldTypes.TOGGLE, + label: 'missing field 2', + required: true, + }, + ], + }, + ]); + + await expect( + bulkCreate( + { + cases: getCases({ + customFields: [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + value: null, + }, + { + key: 'second_key', + type: CustomFieldTypes.TOGGLE, + value: null, + }, + ], + }), + }, + clientArgs, + casesClient + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: Missing required custom fields: \\"missing field 1\\", \\"missing field 2\\""` + ); + }); + + it('throws error when the customFields array is too long', async () => { + await expect( + bulkCreate( + { + cases: getCases({ + customFields: Array(MAX_CUSTOM_FIELDS_PER_CASE + 1).fill(theCustomFields[0]), + }), + }, + clientArgs, + casesClient + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: The length of the field customFields is too long. Array must be of length <= 10."` + ); + }); + + it('throws with duplicated customFields keys', async () => { + await expect( + bulkCreate( + { + cases: getCases({ + customFields: [ + { + key: 'duplicated_key', + type: CustomFieldTypes.TEXT, + value: 'this is a text field value', + }, + { + key: 'duplicated_key', + type: CustomFieldTypes.TOGGLE, + value: true, + }, + ], + }), + }, + clientArgs, + casesClient + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: Invalid duplicated custom field keys in request: duplicated_key"` + ); + }); + + it('throws error when customFields keys are not present in configuration', async () => { + await expect( + bulkCreate( + { + cases: getCases({ + customFields: [ + { + key: 'missing_key', + type: CustomFieldTypes.TEXT, + value: null, + }, + ], + }), + }, + clientArgs, + casesClient + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: Invalid custom field keys: missing_key"` + ); + }); + + it('throws error when required custom fields are missing', async () => { + await expect( + bulkCreate( + { + cases: getCases({ + customFields: [ + { + key: 'second_key', + type: CustomFieldTypes.TEXT, + value: 'this is a text field value', + }, + ], + }), + }, + clientArgs, + casesClient + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: Missing required custom fields: \\"missing field 1\\""` + ); + }); + + it('throws when the customField types do not match the configuration', async () => { + await expect( + bulkCreate( + { + cases: getCases({ + customFields: [ + { + key: 'first_key', + type: CustomFieldTypes.TOGGLE, + value: true, + }, + { + key: 'second_key', + type: CustomFieldTypes.TEXT, + value: 'foobar', + }, + ], + }), + }, + clientArgs, + casesClient + ) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: The following custom fields have the wrong type in the request: first_key,second_key"` + ); + }); + + it('should get all configurations', async () => { + await expect( + bulkCreate({ cases: getCases({ customFields: theCustomFields }) }, clientArgs, casesClient) + ).resolves.not.toThrow(); + + expect(casesClient.configure.get).toHaveBeenCalledWith(); + }); + + it('validate required custom fields from different owners', async () => { + const casesWithDifferentOwners = [getCases()[0], getCases({ owner: 'cases' })[0]]; + + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: theCase.owner, + customFields: [ + { + key: 'sec_first_key', + type: CustomFieldTypes.TEXT, + label: 'sec custom field', + required: false, + }, + ], + }, + { + owner: 'cases', + customFields: [ + { + key: 'cases_first_key', + type: CustomFieldTypes.TEXT, + label: 'stack cases custom field', + required: true, + }, + ], + }, + ]); + + await expect( + bulkCreate({ cases: casesWithDifferentOwners }, clientArgs, casesClient) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failed to bulk create cases: Error: Missing required custom fields: \\"stack cases custom field\\""` + ); + }); + + it('should fill out missing custom fields from different owners correctly', async () => { + const casesWithDifferentOwners = [getCases()[0], getCases({ owner: 'cases' })[0]]; + + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: theCase.owner, + customFields: [ + { + key: 'sec_first_key', + type: CustomFieldTypes.TEXT, + label: 'sec custom field', + required: false, + }, + ], + }, + { + owner: 'cases', + customFields: [ + { + key: 'cases_first_key', + type: CustomFieldTypes.TEXT, + label: 'stack cases custom field', + required: false, + }, + ], + }, + ]); + + await bulkCreate({ cases: casesWithDifferentOwners }, clientArgs, casesClient); + + const cases = clientArgs.services.caseService.bulkCreateCases.mock.calls[0][0].cases; + + expect(cases[0].owner).toBe('securitySolution'); + expect(cases[1].owner).toBe('cases'); + + expect(cases[0].customFields).toEqual([{ key: 'sec_first_key', type: 'text', value: null }]); + expect(cases[1].customFields).toEqual([ + { key: 'cases_first_key', type: 'text', value: null }, + ]); + }); + }); + + describe('User actions', () => { + const theCase = getCases()[0]; + + const caseWithOnlyRequiredFields = omit(theCase, [ + 'assignees', + 'category', + 'severity', + 'customFields', + ]) as CasePostRequest; + + const caseWithOptionalFields: CasePostRequest = { + ...theCase, + category: 'My category', + severity: CaseSeverity.CRITICAL, + customFields: [ + { + key: 'first_customField_key', + type: CustomFieldTypes.TEXT, + value: 'this is a text field value', + }, + { + key: 'second_customField_key', + type: CustomFieldTypes.TOGGLE, + value: true, + }, + ], + }; + + const casesClient = createCasesClientMock(); + const clientArgs = createCasesClientMockArgs(); + clientArgs.services.caseService.bulkCreateCases.mockResolvedValue({ saved_objects: [caseSO] }); + + casesClient.configure.get = jest.fn().mockResolvedValue([ + { + owner: caseWithOptionalFields.owner, + customFields: [ + { + key: 'first_customField_key', + type: CustomFieldTypes.TEXT, + label: 'foo', + required: false, + }, + { + key: 'second_customField_key', + type: CustomFieldTypes.TOGGLE, + label: 'foo', + required: false, + }, + ], + }, + ]); + + it('should bulkCreate a user action with defaults correctly', async () => { + await bulkCreate({ cases: [caseWithOnlyRequiredFields] }, clientArgs, casesClient); + + expect( + clientArgs.services.userActionService.creator.bulkCreateUserAction + ).toHaveBeenCalledWith({ + userActions: [ + { + caseId: 'mock-id-1', + owner: 'securitySolution', + payload: { + assignees: [], + category: null, + connector: { fields: null, id: 'none', name: 'none', type: '.none' }, + customFields: [], + description: 'This is a brand new case of a bad meanie defacing data', + owner: 'securitySolution', + settings: { syncAlerts: true }, + severity: 'low', + tags: ['defacement'], + title: 'Super Bad Security Issue', + }, + type: 'create_case', + user: { + email: 'damaged_raccoon@elastic.co', + full_name: 'Damaged Raccoon', + profile_uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0', + username: 'damaged_raccoon', + }, + }, + ], + }); + }); + + it('should bulkCreate a user action with optional fields set correctly', async () => { + await bulkCreate({ cases: [caseWithOptionalFields] }, clientArgs, casesClient); + + expect( + clientArgs.services.userActionService.creator.bulkCreateUserAction + ).toHaveBeenCalledWith({ + userActions: [ + { + caseId: 'mock-id-1', + owner: 'securitySolution', + payload: { + assignees: [], + category: null, + connector: { fields: null, id: 'none', name: 'none', type: '.none' }, + customFields: [], + description: 'This is a brand new case of a bad meanie defacing data', + owner: 'securitySolution', + settings: { syncAlerts: true }, + severity: 'low', + tags: ['defacement'], + title: 'Super Bad Security Issue', + }, + type: 'create_case', + user: { + email: 'damaged_raccoon@elastic.co', + full_name: 'Damaged Raccoon', + profile_uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0', + username: 'damaged_raccoon', + }, + }, + ], + }); + }); + }); +}); diff --git a/x-pack/plugins/cases/server/client/cases/bulk_create.ts b/x-pack/plugins/cases/server/client/cases/bulk_create.ts new file mode 100644 index 00000000000000..fea7986a9169d6 --- /dev/null +++ b/x-pack/plugins/cases/server/client/cases/bulk_create.ts @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import { partition } from 'lodash'; + +import type { SavedObject } from '@kbn/core/server'; +import { SavedObjectsUtils } from '@kbn/core/server'; + +import type { Case, CustomFieldsConfiguration, User } from '../../../common/types/domain'; +import { CaseSeverity, UserActionTypes } from '../../../common/types/domain'; +import { decodeWithExcessOrThrow } from '../../../common/api'; + +import { Operations } from '../../authorization'; +import { createCaseError } from '../../common/error'; +import { flattenCaseSavedObject, isSOError, transformNewCase } from '../../common/utils'; +import type { CasesClient, CasesClientArgs } from '..'; +import { LICENSING_CASE_ASSIGNMENT_FEATURE } from '../../common/constants'; +import { decodeOrThrow } from '../../../common/api/runtime_types'; +import type { + BulkCreateCasesRequest, + BulkCreateCasesResponse, + CasePostRequest, +} from '../../../common/types/api'; +import { BulkCreateCasesResponseRt, BulkCreateCasesRequestRt } from '../../../common/types/api'; +import { validateCustomFields } from './validators'; +import { normalizeCreateCaseRequest } from './utils'; +import type { BulkCreateCasesArgs } from '../../services/cases/types'; +import type { NotifyAssigneesArgs } from '../../services/notifications/types'; +import type { CaseTransformedAttributes } from '../../common/types/case'; + +export const bulkCreate = async ( + data: BulkCreateCasesRequest, + clientArgs: CasesClientArgs, + casesClient: CasesClient +): Promise => { + const { + services: { caseService, userActionService, licensingService, notificationService }, + user, + logger, + authorization: auth, + } = clientArgs; + + try { + const decodedData = decodeWithExcessOrThrow(BulkCreateCasesRequestRt)(data); + const configurations = await casesClient.configure.get(); + + const customFieldsConfigurationMap: Map = new Map( + configurations.map((conf) => [conf.owner, conf.customFields]) + ); + + const casesWithIds = getCaseWithIds(decodedData); + + await auth.ensureAuthorized({ + operation: Operations.createCase, + entities: casesWithIds.map((theCase) => ({ owner: theCase.owner, id: theCase.id })), + }); + + const hasPlatinumLicenseOrGreater = await licensingService.isAtLeastPlatinum(); + + const bulkCreateRequest: BulkCreateCasesArgs['cases'] = []; + + for (const theCase of casesWithIds) { + const customFieldsConfiguration = customFieldsConfigurationMap.get(theCase.owner); + + validateRequest({ theCase, customFieldsConfiguration, hasPlatinumLicenseOrGreater }); + + bulkCreateRequest.push( + createBulkCreateCaseRequest({ theCase, user, customFieldsConfiguration }) + ); + } + + const bulkCreateResponse = await caseService.bulkCreateCases({ + cases: bulkCreateRequest, + refresh: false, + }); + + const userActions = []; + const assigneesPerCase: NotifyAssigneesArgs[] = []; + const res: Case[] = []; + + const [errors, casesSOs] = partition(bulkCreateResponse.saved_objects, isSOError); + + if (errors.length > 0) { + const firstError = errors[0]; + throw new Boom.Boom(firstError.error.error, { + statusCode: firstError.error.statusCode, + message: firstError.error.message, + }); + } + + for (const theCase of casesSOs) { + userActions.push(createBulkCreateUserActionsRequest({ theCase, user })); + + if (theCase.attributes.assignees && theCase.attributes.assignees.length !== 0) { + const assigneesWithoutCurrentUser = theCase.attributes.assignees.filter( + (assignee) => assignee.uid !== user.profile_uid + ); + + assigneesPerCase.push({ assignees: assigneesWithoutCurrentUser, theCase }); + } + + res.push( + flattenCaseSavedObject({ + savedObject: theCase, + }) + ); + } + + await userActionService.creator.bulkCreateUserAction({ userActions }); + + if (assigneesPerCase.length > 0) { + licensingService.notifyUsage(LICENSING_CASE_ASSIGNMENT_FEATURE); + await notificationService.bulkNotifyAssignees(assigneesPerCase); + } + + return decodeOrThrow(BulkCreateCasesResponseRt)({ cases: res }); + } catch (error) { + throw createCaseError({ message: `Failed to bulk create cases: ${error}`, error, logger }); + } +}; + +const getCaseWithIds = ( + req: BulkCreateCasesRequest +): Array<{ id: string } & BulkCreateCasesRequest['cases'][number]> => + req.cases.map((theCase) => ({ + ...theCase, + id: theCase.id ?? SavedObjectsUtils.generateId(), + })); + +const validateRequest = ({ + theCase, + customFieldsConfiguration, + hasPlatinumLicenseOrGreater, +}: { + theCase: BulkCreateCasesRequest['cases'][number]; + customFieldsConfiguration?: CustomFieldsConfiguration; + hasPlatinumLicenseOrGreater: boolean; +}) => { + const customFieldsValidationParams = { + requestCustomFields: theCase.customFields, + customFieldsConfiguration, + }; + + validateCustomFields(customFieldsValidationParams); + validateAssigneesUsage({ assignees: theCase.assignees, hasPlatinumLicenseOrGreater }); +}; + +const validateAssigneesUsage = ({ + assignees, + hasPlatinumLicenseOrGreater, +}: { + assignees?: BulkCreateCasesRequest['cases'][number]['assignees']; + hasPlatinumLicenseOrGreater: boolean; +}) => { + /** + * Assign users to a case is only available to Platinum+ + */ + + if (assignees && assignees.length !== 0) { + if (!hasPlatinumLicenseOrGreater) { + throw Boom.forbidden( + 'In order to assign users to cases, you must be subscribed to an Elastic Platinum license' + ); + } + } +}; + +const createBulkCreateCaseRequest = ({ + theCase, + customFieldsConfiguration, + user, +}: { + theCase: { id: string } & BulkCreateCasesRequest['cases'][number]; + customFieldsConfiguration?: CustomFieldsConfiguration; + user: User; +}): BulkCreateCasesArgs['cases'][number] => { + const { id, ...caseWithoutId } = theCase; + + /** + * Trim title, category, description and tags + * and fill out missing custom fields + * before saving to ES + */ + + const normalizedCase = normalizeCreateCaseRequest(caseWithoutId, customFieldsConfiguration); + + return { + id, + ...transformNewCase({ + user, + newCase: normalizedCase, + }), + }; +}; + +const createBulkCreateUserActionsRequest = ({ + theCase, + user, +}: { + theCase: SavedObject; + user: User; +}) => { + const userActionPayload: CasePostRequest = { + title: theCase.attributes.title, + tags: theCase.attributes.tags, + connector: theCase.attributes.connector, + settings: theCase.attributes.settings, + owner: theCase.attributes.owner, + description: theCase.attributes.description, + severity: theCase.attributes.severity ?? CaseSeverity.LOW, + assignees: theCase.attributes.assignees ?? [], + category: theCase.attributes.category ?? null, + customFields: theCase.attributes.customFields ?? [], + }; + + return { + type: UserActionTypes.create_case, + caseId: theCase.id, + user, + payload: userActionPayload, + owner: theCase.attributes.owner, + }; +}; diff --git a/x-pack/plugins/cases/server/client/cases/client.ts b/x-pack/plugins/cases/server/client/cases/client.ts index bbe91b1ad791ed..4a3ed9e6c4b055 100644 --- a/x-pack/plugins/cases/server/client/cases/client.ts +++ b/x-pack/plugins/cases/server/client/cases/client.ts @@ -18,6 +18,8 @@ import type { AllReportersFindRequest, GetRelatedCasesByAlertResponse, CasesBulkGetResponse, + BulkCreateCasesRequest, + BulkCreateCasesResponse, } from '../../../common/types/api'; import type { CasesClient } from '../client'; import type { CasesClientInternal } from '../client_internal'; @@ -31,6 +33,7 @@ import { get, resolve, getCasesByAlertID, getReporters, getTags, getCategories } import type { PushParams } from './push'; import { push } from './push'; import { update } from './update'; +import { bulkCreate } from './bulk_create'; /** * API for interacting with the cases entities. @@ -40,6 +43,10 @@ export interface CasesSubClient { * Creates a case. */ create(data: CasePostRequest): Promise; + /** + * Bulk create cases. + */ + bulkCreate(data: BulkCreateCasesRequest): Promise; /** * Returns cases that match the search criteria. * @@ -103,6 +110,7 @@ export const createCasesSubClient = ( ): CasesSubClient => { const casesSubClient: CasesSubClient = { create: (data: CasePostRequest) => create(data, clientArgs, casesClient), + bulkCreate: (data: BulkCreateCasesRequest) => bulkCreate(data, clientArgs, casesClient), find: (params: CasesFindRequest) => find(params, clientArgs), get: (params: GetParams) => get(params, clientArgs), resolve: (params: GetParams) => resolve(params, clientArgs), diff --git a/x-pack/plugins/cases/server/client/cases/create.test.ts b/x-pack/plugins/cases/server/client/cases/create.test.ts index b7cc876c476558..b65061d403fec2 100644 --- a/x-pack/plugins/cases/server/client/cases/create.test.ts +++ b/x-pack/plugins/cases/server/client/cases/create.test.ts @@ -51,7 +51,7 @@ describe('create', () => { describe('Assignees', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); beforeEach(() => { jest.clearAllMocks(); @@ -108,11 +108,19 @@ describe('create', () => { `Failed to create case: Error: The length of the field assignees is too long. Array must be of length <= ${MAX_ASSIGNEES_PER_CASE}.` ); }); + + it('should throw if the user does not have the correct license', async () => { + clientArgs.services.licensingService.isAtLeastPlatinum.mockResolvedValue(false); + + await expect(create(theCase, clientArgs, casesClientMock)).rejects.toThrow( + `Failed to create case: Error: In order to assign users to cases, you must be subscribed to an Elastic Platinum license` + ); + }); }); describe('Attributes', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); beforeEach(() => { jest.clearAllMocks(); @@ -130,7 +138,7 @@ describe('create', () => { describe('title', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); beforeEach(() => { jest.clearAllMocks(); @@ -173,7 +181,7 @@ describe('create', () => { it('should trim title', async () => { await create({ ...theCase, title: 'title with spaces ' }, clientArgs, casesClientMock); - expect(clientArgs.services.caseService.postNewCase).toHaveBeenCalledWith( + expect(clientArgs.services.caseService.createCase).toHaveBeenCalledWith( expect.objectContaining({ attributes: { ...theCase, @@ -199,7 +207,7 @@ describe('create', () => { describe('description', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); beforeEach(() => { jest.clearAllMocks(); @@ -250,7 +258,7 @@ describe('create', () => { casesClientMock ); - expect(clientArgs.services.caseService.postNewCase).toHaveBeenCalledWith( + expect(clientArgs.services.caseService.createCase).toHaveBeenCalledWith( expect.objectContaining({ attributes: { ...theCase, @@ -276,7 +284,7 @@ describe('create', () => { describe('tags', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); beforeEach(() => { jest.clearAllMocks(); @@ -329,7 +337,7 @@ describe('create', () => { it('should trim tags', async () => { await create({ ...theCase, tags: ['pepsi ', 'coke'] }, clientArgs, casesClientMock); - expect(clientArgs.services.caseService.postNewCase).toHaveBeenCalledWith( + expect(clientArgs.services.caseService.createCase).toHaveBeenCalledWith( expect.objectContaining({ attributes: { ...theCase, @@ -355,7 +363,7 @@ describe('create', () => { describe('Category', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); beforeEach(() => { jest.clearAllMocks(); @@ -396,7 +404,7 @@ describe('create', () => { it('should trim category', async () => { await create({ ...theCase, category: 'reporting ' }, clientArgs, casesClientMock); - expect(clientArgs.services.caseService.postNewCase).toHaveBeenCalledWith( + expect(clientArgs.services.caseService.createCase).toHaveBeenCalledWith( expect.objectContaining({ attributes: { ...theCase, @@ -421,7 +429,7 @@ describe('create', () => { describe('Custom Fields', () => { const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); const casesClient = createCasesClientMock(); casesClient.configure.get = jest.fn().mockResolvedValue([ @@ -473,7 +481,7 @@ describe('create', () => { ) ).resolves.not.toThrow(); - expect(clientArgs.services.caseService.postNewCase).toHaveBeenCalledWith( + expect(clientArgs.services.caseService.createCase).toHaveBeenCalledWith( expect.objectContaining({ attributes: { ...theCase, @@ -517,7 +525,7 @@ describe('create', () => { ]); await expect(create({ ...theCase }, clientArgs, casesClient)).resolves.not.toThrow(); - expect(clientArgs.services.caseService.postNewCase).toHaveBeenCalledWith( + expect(clientArgs.services.caseService.createCase).toHaveBeenCalledWith( expect.objectContaining({ attributes: { ...theCase, @@ -758,7 +766,7 @@ describe('create', () => { const casesClient = createCasesClientMock(); const clientArgs = createCasesClientMockArgs(); - clientArgs.services.caseService.postNewCase.mockResolvedValue(caseSO); + clientArgs.services.caseService.createCase.mockResolvedValue(caseSO); casesClient.configure.get = jest.fn().mockResolvedValue([ { @@ -784,26 +792,28 @@ describe('create', () => { await create(caseWithOnlyRequiredFields, clientArgs, casesClient); expect(clientArgs.services.userActionService.creator.createUserAction).toHaveBeenCalledWith({ - caseId: 'mock-id-1', - owner: 'securitySolution', - payload: { - assignees: [], - category: null, - connector: { fields: null, id: '.none', name: 'None', type: '.none' }, - customFields: [], - description: 'testing sir', + userAction: { + caseId: 'mock-id-1', owner: 'securitySolution', - settings: { syncAlerts: true }, - severity: 'low', - tags: [], - title: 'My Case', - }, - type: 'create_case', - user: { - email: 'damaged_raccoon@elastic.co', - full_name: 'Damaged Raccoon', - profile_uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0', - username: 'damaged_raccoon', + payload: { + assignees: [], + category: null, + connector: { fields: null, id: '.none', name: 'None', type: '.none' }, + customFields: [], + description: 'testing sir', + owner: 'securitySolution', + settings: { syncAlerts: true }, + severity: 'low', + tags: [], + title: 'My Case', + }, + type: 'create_case', + user: { + email: 'damaged_raccoon@elastic.co', + full_name: 'Damaged Raccoon', + profile_uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0', + username: 'damaged_raccoon', + }, }, }); }); @@ -812,26 +822,28 @@ describe('create', () => { await create(caseWithOptionalFields, clientArgs, casesClient); expect(clientArgs.services.userActionService.creator.createUserAction).toHaveBeenCalledWith({ - caseId: 'mock-id-1', - owner: 'securitySolution', - payload: { - assignees: [{ uid: '1' }], - category: 'My category', - connector: { fields: null, id: '.none', name: 'None', type: '.none' }, - customFields: caseWithOptionalFields.customFields, - description: 'testing sir', + userAction: { + caseId: 'mock-id-1', owner: 'securitySolution', - settings: { syncAlerts: true }, - severity: 'critical', - tags: [], - title: 'My Case', - }, - type: 'create_case', - user: { - email: 'damaged_raccoon@elastic.co', - full_name: 'Damaged Raccoon', - profile_uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0', - username: 'damaged_raccoon', + payload: { + assignees: [{ uid: '1' }], + category: 'My category', + connector: { fields: null, id: '.none', name: 'None', type: '.none' }, + customFields: caseWithOptionalFields.customFields, + description: 'testing sir', + owner: 'securitySolution', + settings: { syncAlerts: true }, + severity: 'critical', + tags: [], + title: 'My Case', + }, + type: 'create_case', + user: { + email: 'damaged_raccoon@elastic.co', + full_name: 'Damaged Raccoon', + profile_uid: 'u_J41Oh6L9ki-Vo2tOogS8WRTENzhHurGtRc87NgEAlkc_0', + username: 'damaged_raccoon', + }, }, }); }); diff --git a/x-pack/plugins/cases/server/client/cases/create.ts b/x-pack/plugins/cases/server/client/cases/create.ts index d48eaa080dee8c..4e548d07a2ef6d 100644 --- a/x-pack/plugins/cases/server/client/cases/create.ts +++ b/x-pack/plugins/cases/server/client/cases/create.ts @@ -23,7 +23,7 @@ import type { CasePostRequest } from '../../../common/types/api'; import { CasePostRequestRt } from '../../../common/types/api'; import {} from '../utils'; import { validateCustomFields } from './validators'; -import { fillMissingCustomFields } from './utils'; +import { normalizeCreateCaseRequest } from './utils'; /** * Creates a new case. @@ -82,39 +82,31 @@ export const create = async ( * before saving to ES */ - const normalizedQuery = { - ...query, - title: query.title.trim(), - description: query.description.trim(), - category: query.category?.trim() ?? null, - tags: query.tags?.map((tag) => tag.trim()) ?? [], - customFields: fillMissingCustomFields({ - customFields: query.customFields, - customFieldsConfiguration, - }), - }; + const normalizedCase = normalizeCreateCaseRequest(query, customFieldsConfiguration); - const newCase = await caseService.postNewCase({ + const newCase = await caseService.createCase({ attributes: transformNewCase({ user, - newCase: normalizedQuery, + newCase: normalizedCase, }), id: savedObjectID, refresh: false, }); await userActionService.creator.createUserAction({ - type: UserActionTypes.create_case, - caseId: newCase.id, - user, - payload: { - ...query, - severity: query.severity ?? CaseSeverity.LOW, - assignees: query.assignees ?? [], - category: query.category ?? null, - customFields: query.customFields ?? [], + userAction: { + type: UserActionTypes.create_case, + caseId: newCase.id, + user, + payload: { + ...query, + severity: query.severity ?? CaseSeverity.LOW, + assignees: query.assignees ?? [], + category: query.category ?? null, + customFields: query.customFields ?? [], + }, + owner: newCase.attributes.owner, }, - owner: newCase.attributes.owner, }); if (query.assignees && query.assignees.length !== 0) { diff --git a/x-pack/plugins/cases/server/client/cases/push.ts b/x-pack/plugins/cases/server/client/cases/push.ts index cf3732c065e23a..fba7908ed2762d 100644 --- a/x-pack/plugins/cases/server/client/cases/push.ts +++ b/x-pack/plugins/cases/server/client/cases/push.ts @@ -261,11 +261,13 @@ export const push = async ( if (shouldMarkAsClosed) { await userActionService.creator.createUserAction({ - type: UserActionTypes.status, - payload: { status: CaseStatuses.closed }, - user, - caseId, - owner: myCase.attributes.owner, + userAction: { + type: UserActionTypes.status, + payload: { status: CaseStatuses.closed }, + user, + caseId, + owner: myCase.attributes.owner, + }, refresh: false, }); @@ -275,11 +277,13 @@ export const push = async ( } await userActionService.creator.createUserAction({ - type: UserActionTypes.pushed, - payload: { externalService }, - user, - caseId, - owner: myCase.attributes.owner, + userAction: { + type: UserActionTypes.pushed, + payload: { externalService }, + user, + caseId, + owner: myCase.attributes.owner, + }, }); /* End of update case with push information */ diff --git a/x-pack/plugins/cases/server/client/cases/update.ts b/x-pack/plugins/cases/server/client/cases/update.ts index b2baff721302fc..1f7a489127a087 100644 --- a/x-pack/plugins/cases/server/client/cases/update.ts +++ b/x-pack/plugins/cases/server/client/cases/update.ts @@ -369,7 +369,7 @@ export const update = async ( ); } - const configurations = await casesClient.configure.get({}); + const configurations = await casesClient.configure.get(); const customFieldsConfigurationMap: Map = new Map( configurations.map((conf) => [conf.owner, conf.customFields]) ); diff --git a/x-pack/plugins/cases/server/client/cases/utils.test.ts b/x-pack/plugins/cases/server/client/cases/utils.test.ts index a1017b20b4286f..2d697b597baacf 100644 --- a/x-pack/plugins/cases/server/client/cases/utils.test.ts +++ b/x-pack/plugins/cases/server/client/cases/utils.test.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { omit } from 'lodash'; + import { comment as commentObj, userActions, @@ -28,9 +30,16 @@ import { formatComments, addKibanaInformationToDescription, fillMissingCustomFields, + normalizeCreateCaseRequest, } from './utils'; import type { CaseCustomFields } from '../../../common/types/domain'; -import { CaseStatuses, CustomFieldTypes, UserActionActions } from '../../../common/types/domain'; +import { + CaseStatuses, + CustomFieldTypes, + UserActionActions, + CaseSeverity, + ConnectorTypes, +} from '../../../common/types/domain'; import { flattenCaseSavedObject } from '../../common/utils'; import { SECURITY_SOLUTION_OWNER } from '../../../common/constants'; import { casesConnectors } from '../../connectors'; @@ -1471,3 +1480,96 @@ describe('utils', () => { }); }); }); + +describe('normalizeCreateCaseRequest', () => { + const theCase = { + title: 'My Case', + tags: [], + description: 'testing sir', + connector: { + id: '.none', + name: 'None', + type: ConnectorTypes.none, + fields: null, + }, + settings: { syncAlerts: true }, + severity: CaseSeverity.LOW, + owner: SECURITY_SOLUTION_OWNER, + assignees: [{ uid: '1' }], + category: 'my category', + customFields: [], + }; + + it('should trim title', async () => { + expect(normalizeCreateCaseRequest({ ...theCase, title: 'title with spaces ' })).toEqual({ + ...theCase, + title: 'title with spaces', + }); + }); + + it('should trim description', async () => { + expect( + normalizeCreateCaseRequest({ + ...theCase, + description: 'this is a description with spaces!! ', + }) + ).toEqual({ + ...theCase, + description: 'this is a description with spaces!!', + }); + }); + + it('should trim tags', async () => { + expect( + normalizeCreateCaseRequest({ + ...theCase, + tags: ['pepsi ', 'coke'], + }) + ).toEqual({ + ...theCase, + tags: ['pepsi', 'coke'], + }); + }); + + it('should trim category', async () => { + expect( + normalizeCreateCaseRequest({ + ...theCase, + category: 'reporting ', + }) + ).toEqual({ + ...theCase, + category: 'reporting', + }); + }); + + it('should set the category to null if missing', async () => { + expect(normalizeCreateCaseRequest(omit(theCase, 'category'))).toEqual({ + ...theCase, + category: null, + }); + }); + + it('should fill out missing custom fields', async () => { + expect( + normalizeCreateCaseRequest(omit(theCase, 'customFields'), [ + { + key: 'first_key', + type: CustomFieldTypes.TEXT, + label: 'foo', + required: false, + }, + ]) + ).toEqual({ + ...theCase, + customFields: [{ key: 'first_key', type: CustomFieldTypes.TEXT, value: null }], + }); + }); + + it('should set the customFields to an empty array if missing', async () => { + expect(normalizeCreateCaseRequest(omit(theCase, 'customFields'))).toEqual({ + ...theCase, + customFields: [], + }); + }); +}); diff --git a/x-pack/plugins/cases/server/client/cases/utils.ts b/x-pack/plugins/cases/server/client/cases/utils.ts index 9e7f1ab73af206..474871af02c5dc 100644 --- a/x-pack/plugins/cases/server/client/cases/utils.ts +++ b/x-pack/plugins/cases/server/client/cases/utils.ts @@ -25,6 +25,7 @@ import type { } from '../../../common/types/domain'; import { CaseStatuses, UserActionTypes, AttachmentType } from '../../../common/types/domain'; import type { + CasePostRequest, CaseRequestCustomFields, CaseUserActionsDeprecatedResponse, } from '../../../common/types/api'; @@ -480,3 +481,18 @@ export const fillMissingCustomFields = ({ return [...customFields, ...missingCustomFields]; }; + +export const normalizeCreateCaseRequest = ( + request: CasePostRequest, + customFieldsConfiguration?: CustomFieldsConfiguration +) => ({ + ...request, + title: request.title.trim(), + description: request.description.trim(), + category: request.category?.trim() ?? null, + tags: request.tags?.map((tag) => tag.trim()) ?? [], + customFields: fillMissingCustomFields({ + customFields: request.customFields, + customFieldsConfiguration, + }), +}); diff --git a/x-pack/plugins/cases/server/client/configure/client.ts b/x-pack/plugins/cases/server/client/configure/client.ts index 1482ea501ca87d..1261f1061a3717 100644 --- a/x-pack/plugins/cases/server/client/configure/client.ts +++ b/x-pack/plugins/cases/server/client/configure/client.ts @@ -69,7 +69,7 @@ export interface ConfigureSubClient { /** * Retrieves the external connector configuration for a particular case owner. */ - get(params: GetConfigurationFindRequest): Promise; + get(params?: GetConfigurationFindRequest): Promise; /** * Retrieves the valid external connectors supported by the cases plugin. */ @@ -120,7 +120,7 @@ export const createConfigurationSubClient = ( casesInternalClient: CasesClientInternal ): ConfigureSubClient => { return Object.freeze({ - get: (params: GetConfigurationFindRequest) => get(params, clientArgs, casesInternalClient), + get: (params?: GetConfigurationFindRequest) => get(params, clientArgs, casesInternalClient), getConnectors: () => getConnectors(clientArgs), update: (configurationId: string, configuration: ConfigurationPatchRequest) => update(configurationId, configuration, clientArgs, casesInternalClient), @@ -130,7 +130,7 @@ export const createConfigurationSubClient = ( }; export async function get( - params: GetConfigurationFindRequest, + params: GetConfigurationFindRequest = {}, clientArgs: CasesClientArgs, casesClientInternal: CasesClientInternal ): Promise { diff --git a/x-pack/plugins/cases/server/client/mocks.ts b/x-pack/plugins/cases/server/client/mocks.ts index 8fd0a61c35f304..7d4015d15a085d 100644 --- a/x-pack/plugins/cases/server/client/mocks.ts +++ b/x-pack/plugins/cases/server/client/mocks.ts @@ -47,6 +47,7 @@ type CasesSubClientMock = jest.Mocked; const createCasesSubClientMock = (): CasesSubClientMock => { return { create: jest.fn(), + bulkCreate: jest.fn(), find: jest.fn(), resolve: jest.fn(), get: jest.fn(), diff --git a/x-pack/plugins/cases/server/common/models/case_with_comments.ts b/x-pack/plugins/cases/server/common/models/case_with_comments.ts index 5e48c38e67d0d8..e1b89d7af791e1 100644 --- a/x-pack/plugins/cases/server/common/models/case_with_comments.ts +++ b/x-pack/plugins/cases/server/common/models/case_with_comments.ts @@ -193,13 +193,15 @@ export class CaseCommentModel { const { id, version, ...queryRestAttributes } = updateRequest; await this.params.services.userActionService.creator.createUserAction({ - type: UserActionTypes.comment, - action: UserActionActions.update, - caseId: this.caseInfo.id, - attachmentId: comment.id, - payload: { attachment: queryRestAttributes }, - user: this.params.user, - owner, + userAction: { + type: UserActionTypes.comment, + action: UserActionActions.update, + caseId: this.caseInfo.id, + attachmentId: comment.id, + payload: { attachment: queryRestAttributes }, + user: this.params.user, + owner, + }, }); } @@ -403,15 +405,17 @@ export class CaseCommentModel { req: AttachmentRequest ) { await this.params.services.userActionService.creator.createUserAction({ - type: UserActionTypes.comment, - action: UserActionActions.create, - caseId: this.caseInfo.id, - attachmentId: comment.id, - payload: { - attachment: req, + userAction: { + type: UserActionTypes.comment, + action: UserActionActions.create, + caseId: this.caseInfo.id, + attachmentId: comment.id, + payload: { + attachment: req, + }, + user: this.params.user, + owner: comment.attributes.owner, }, - user: this.params.user, - owner: comment.attributes.owner, }); } diff --git a/x-pack/plugins/cases/server/common/types.ts b/x-pack/plugins/cases/server/common/types.ts index c79cb96d0d0b66..8855d73e9f4b39 100644 --- a/x-pack/plugins/cases/server/common/types.ts +++ b/x-pack/plugins/cases/server/common/types.ts @@ -52,3 +52,7 @@ export type FileAttachmentRequest = Omit< export type AttachmentSavedObject = SavedObject; export type SOWithErrors = Omit, 'attributes'> & { error: SavedObjectError }; + +export interface SavedObjectsBulkResponseWithErrors { + saved_objects: Array | SOWithErrors>; +} diff --git a/x-pack/plugins/cases/server/services/attachments/index.test.ts b/x-pack/plugins/cases/server/services/attachments/index.test.ts index fc12e26b67ed6e..b089616717e76b 100644 --- a/x-pack/plugins/cases/server/services/attachments/index.test.ts +++ b/x-pack/plugins/cases/server/services/attachments/index.test.ts @@ -21,9 +21,10 @@ import { persistableStateAttachmentAttributes, persistableStateAttachmentAttributesWithoutInjectedId, } from '../../attachment_framework/mocks'; -import { createAlertAttachment, createErrorSO, createUserAttachment } from './test_utils'; +import { createAlertAttachment, createUserAttachment } from './test_utils'; import { AttachmentType } from '../../../common/types/domain'; -import { createSOFindResponse } from '../test_utils'; +import { createErrorSO, createSOFindResponse } from '../test_utils'; +import { CASE_COMMENT_SAVED_OBJECT } from '../../../common'; describe('AttachmentService', () => { const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); @@ -135,9 +136,10 @@ describe('AttachmentService', () => { it('returns error objects unmodified', async () => { const userAttachment = createUserAttachment({ foo: 'bar' }); - const errorResponseObj = createErrorSO(); + const errorResponseObj = createErrorSO(CASE_COMMENT_SAVED_OBJECT); unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + // @ts-expect-error: SO client types are wrong saved_objects: [errorResponseObj, userAttachment], }); @@ -411,9 +413,10 @@ describe('AttachmentService', () => { it('returns error objects unmodified', async () => { const userAttachment = createUserAttachment({ foo: 'bar' }); - const errorResponseObj = createErrorSO(); + const errorResponseObj = createErrorSO(CASE_COMMENT_SAVED_OBJECT); unsecuredSavedObjectsClient.bulkUpdate.mockResolvedValue({ + // @ts-expect-error: SO client types are wrong saved_objects: [errorResponseObj, userAttachment], }); diff --git a/x-pack/plugins/cases/server/services/attachments/operations/get.test.ts b/x-pack/plugins/cases/server/services/attachments/operations/get.test.ts index 5ee0cdce357693..590cabcae67a13 100644 --- a/x-pack/plugins/cases/server/services/attachments/operations/get.test.ts +++ b/x-pack/plugins/cases/server/services/attachments/operations/get.test.ts @@ -12,13 +12,9 @@ import type { SavedObjectsFindResponse } from '@kbn/core/server'; import { loggerMock } from '@kbn/logging-mocks'; import { createPersistableStateAttachmentTypeRegistryMock } from '../../../attachment_framework/mocks'; import { AttachmentGetter } from './get'; -import { - createAlertAttachment, - createErrorSO, - createFileAttachment, - createUserAttachment, -} from '../test_utils'; -import { mockPointInTimeFinder, createSOFindResponse } from '../../test_utils'; +import { createAlertAttachment, createFileAttachment, createUserAttachment } from '../test_utils'; +import { mockPointInTimeFinder, createSOFindResponse, createErrorSO } from '../../test_utils'; +import { CASE_COMMENT_SAVED_OBJECT } from '../../../../common'; describe('AttachmentService getter', () => { const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); @@ -50,12 +46,15 @@ describe('AttachmentService getter', () => { it('does not modified the error saved objects', async () => { unsecuredSavedObjectsClient.bulkGet.mockResolvedValue({ - saved_objects: [createUserAttachment(), createErrorSO()], + // @ts-expect-error: SO client types are not correct + saved_objects: [createUserAttachment(), createErrorSO(CASE_COMMENT_SAVED_OBJECT)], }); const res = await attachmentGetter.bulkGet(['1', '2']); - expect(res).toStrictEqual({ saved_objects: [createUserAttachment(), createErrorSO()] }); + expect(res).toStrictEqual({ + saved_objects: [createUserAttachment(), createErrorSO(CASE_COMMENT_SAVED_OBJECT)], + }); }); it('strips excess fields', async () => { diff --git a/x-pack/plugins/cases/server/services/attachments/test_utils.ts b/x-pack/plugins/cases/server/services/attachments/test_utils.ts index ab3c7a6f710be9..b90fac81b54428 100644 --- a/x-pack/plugins/cases/server/services/attachments/test_utils.ts +++ b/x-pack/plugins/cases/server/services/attachments/test_utils.ts @@ -22,19 +22,6 @@ import { } from '../../../common/constants'; import { CASE_REF_NAME, EXTERNAL_REFERENCE_REF_NAME } from '../../common/constants'; -export const createErrorSO = () => - ({ - id: '1', - type: CASE_COMMENT_SAVED_OBJECT, - error: { - error: 'error', - message: 'message', - statusCode: 500, - }, - references: [], - // casting because this complains about attributes not being there - } as unknown as SavedObject); - export const createUserAttachment = ( attributes?: object ): SavedObject => { diff --git a/x-pack/plugins/cases/server/services/cases/index.test.ts b/x-pack/plugins/cases/server/services/cases/index.test.ts index 712e22732b9ff2..db6776b7524747 100644 --- a/x-pack/plugins/cases/server/services/cases/index.test.ts +++ b/x-pack/plugins/cases/server/services/cases/index.test.ts @@ -41,10 +41,15 @@ import { createCaseSavedObjectResponse, basicCaseFields, createSOFindResponse, + createErrorSO, } from '../test_utils'; import { AttachmentService } from '../attachments'; import { PersistableStateAttachmentTypeRegistry } from '../../attachment_framework/persistable_state_registry'; -import type { CaseSavedObjectTransformed, CasePersistedAttributes } from '../../common/types/case'; +import type { + CaseSavedObjectTransformed, + CasePersistedAttributes, + CaseTransformedAttributes, +} from '../../common/types/case'; import { CasePersistedSeverity, CasePersistedStatus, @@ -175,6 +180,101 @@ describe('CasesService', () => { }); }); + describe('execution', () => { + describe('bulkCreateCases', () => { + it('return cases with the SO errors correctly', async () => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + // @ts-expect-error: SO client types are not correct + saved_objects: [createCaseSavedObjectResponse(), createErrorSO('cases')], + }); + + const res = await service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ + connector: getNoneCaseConnector(), + severity: CaseSeverity.MEDIUM, + }), + id: '1', + }, + ], + }); + + expect(res).toMatchInlineSnapshot(` + Object { + "saved_objects": Array [ + Object { + "attributes": Object { + "assignees": Array [], + "category": null, + "closed_at": null, + "closed_by": null, + "connector": Object { + "fields": null, + "id": "none", + "name": "none", + "type": ".none", + }, + "created_at": "2019-11-25T21:54:48.952Z", + "created_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "customFields": Array [], + "description": "This is a brand new case of a bad meanie defacing data", + "duration": null, + "external_service": Object { + "connector_id": "none", + "connector_name": ".jira", + "external_id": "100", + "external_title": "awesome", + "external_url": "http://www.google.com", + "pushed_at": "2019-11-25T21:54:48.952Z", + "pushed_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "low", + "status": "open", + "tags": Array [ + "defacement", + ], + "title": "Super Bad Security Issue", + "updated_at": "2019-11-25T21:54:48.952Z", + "updated_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "id": "1", + "references": Array [], + "type": "cases", + }, + Object { + "error": Object { + "error": "error", + "message": "message", + "statusCode": 500, + }, + "id": "1", + "references": Array [], + "type": "cases", + }, + ], + } + `); + }); + }); + }); + describe('transforms the external model to the Elasticsearch model', () => { describe('patch', () => { it('includes the passed in fields', async () => { @@ -663,11 +763,11 @@ describe('CasesService', () => { }); }); - describe('post', () => { + describe('createCase', () => { it('creates a null external_service field when the attribute was null in the creation parameters', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector() }), id: '1', }); @@ -680,7 +780,7 @@ describe('CasesService', () => { it('includes the creation attributes excluding the connector.id and connector_id', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector(), externalService: createExternalService(), @@ -780,7 +880,7 @@ describe('CasesService', () => { it('includes default values for total_alerts and total_comments', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector(), }), @@ -797,7 +897,7 @@ describe('CasesService', () => { it('moves the connector.id and connector_id to the references', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector(), externalService: createExternalService(), @@ -826,7 +926,7 @@ describe('CasesService', () => { it('sets fields to an empty array when it is not included with the connector', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector({ setFieldsToNull: true }), externalService: createExternalService(), @@ -842,7 +942,7 @@ describe('CasesService', () => { it('does not create a reference for a none connector', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector() }), id: '1', }); @@ -855,7 +955,7 @@ describe('CasesService', () => { it('does not create a reference for an external_service field that is null', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector() }), id: '1', }); @@ -875,7 +975,7 @@ describe('CasesService', () => { async (postParamsSeverity, expectedSeverity) => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector(), severity: postParamsSeverity, @@ -898,7 +998,7 @@ describe('CasesService', () => { async (postParamsStatus, expectedStatus) => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); - await service.postNewCase({ + await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector(), status: postParamsStatus, @@ -912,6 +1012,103 @@ describe('CasesService', () => { } ); }); + + describe('bulkCreateCases', () => { + it('creates a null external_service field when the attribute was null in the creation parameters', async () => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [createCaseSavedObjectResponse()], + }); + + await service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ + connector: getNoneCaseConnector(), + severity: CaseSeverity.MEDIUM, + }), + id: '1', + }, + ], + }); + + const bulkCreateRequest = unsecuredSavedObjectsClient.bulkCreate.mock.calls[0]; + + expect(bulkCreateRequest).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "attributes": Object { + "assignees": Array [], + "category": null, + "closed_at": null, + "closed_by": null, + "connector": Object { + "fields": Array [], + "name": "none", + "type": ".none", + }, + "created_at": "2019-11-25T21:54:48.952Z", + "created_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "customFields": Array [], + "description": "This is a brand new case of a bad meanie defacing data", + "duration": null, + "external_service": null, + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": 10, + "status": 0, + "tags": Array [ + "defacement", + ], + "title": "Super Bad Security Issue", + "total_alerts": -1, + "total_comments": -1, + "updated_at": "2019-11-25T21:54:48.952Z", + "updated_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "id": "1", + "references": Array [], + "type": "cases", + }, + ], + Object { + "refresh": undefined, + }, + ] + `); + }); + }); + + it('includes default values for total_alerts and total_comments', async () => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [createCaseSavedObjectResponse({})], + }); + + await service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ connector: getNoneCaseConnector() }), + id: '1', + }, + ], + }); + + const postAttributes = unsecuredSavedObjectsClient.bulkCreate.mock.calls[0][0][0] + .attributes as CasePersistedAttributes; + + expect(postAttributes.total_alerts).toEqual(-1); + expect(postAttributes.total_comments).toEqual(-1); + }); }); describe('transforms the Elasticsearch model to the external model', () => { @@ -1364,7 +1561,7 @@ describe('CasesService', () => { }); }); - describe('post', () => { + describe('createCase', () => { it('includes the connector.id and connector_id fields in the response', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue( createCaseSavedObjectResponse({ @@ -1373,7 +1570,7 @@ describe('CasesService', () => { }) ); - const res = await service.postNewCase({ + const res = await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector() }), id: '1', }); @@ -1394,7 +1591,7 @@ describe('CasesService', () => { createCaseSavedObjectResponse({ overrides: { severity: internalSeverityValue } }) ); - const res = await service.postNewCase({ + const res = await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector() }), id: '1', }); @@ -1414,7 +1611,7 @@ describe('CasesService', () => { createCaseSavedObjectResponse({ overrides: { status: internalStatusValue } }) ); - const res = await service.postNewCase({ + const res = await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector() }), id: '1', }); @@ -1426,7 +1623,7 @@ describe('CasesService', () => { it('does not include total_alerts and total_comments fields in the response', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse({})); - const res = await service.postNewCase({ + const res = await service.createCase({ attributes: createCasePostParams({ connector: getNoneCaseConnector() }), id: '1', }); @@ -1436,6 +1633,109 @@ describe('CasesService', () => { }); }); + describe('bulkCreateCases', () => { + it('includes the connector.id and connector_id fields in the response', async () => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [ + createCaseSavedObjectResponse({ + overrides: { severity: CasePersistedSeverity.MEDIUM }, + }), + ], + }); + + const res = await service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ connector: getNoneCaseConnector() }), + id: '1', + }, + ], + }); + + expect(res).toMatchInlineSnapshot(` + Object { + "saved_objects": Array [ + Object { + "attributes": Object { + "assignees": Array [], + "category": null, + "closed_at": null, + "closed_by": null, + "connector": Object { + "fields": null, + "id": "none", + "name": "none", + "type": ".none", + }, + "created_at": "2019-11-25T21:54:48.952Z", + "created_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "customFields": Array [], + "description": "This is a brand new case of a bad meanie defacing data", + "duration": null, + "external_service": Object { + "connector_id": "none", + "connector_name": ".jira", + "external_id": "100", + "external_title": "awesome", + "external_url": "http://www.google.com", + "pushed_at": "2019-11-25T21:54:48.952Z", + "pushed_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "medium", + "status": "open", + "tags": Array [ + "defacement", + ], + "title": "Super Bad Security Issue", + "updated_at": "2019-11-25T21:54:48.952Z", + "updated_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "id": "1", + "references": Array [], + "type": "cases", + }, + ], + } + `); + }); + + it('does not include total_alerts and total_comments fields in the response', async () => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [createCaseSavedObjectResponse({})], + }); + + const res = await service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ connector: getNoneCaseConnector() }), + id: '1', + }, + ], + }); + + const theCase = res.saved_objects[0] as SavedObject; + + expect(theCase.attributes).not.toHaveProperty('total_alerts'); + expect(theCase.attributes).not.toHaveProperty('total_comments'); + }); + }); + describe('find', () => { it('includes the connector.id and connector_id field in the response', async () => { const findMockReturn = createSOFindResponse([ @@ -2469,12 +2769,12 @@ describe('CasesService', () => { }); }); - describe('post', () => { + describe('createCase', () => { it('decodes correctly', async () => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); await expect( - service.postNewCase({ + service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector() }), id: '1', }) @@ -2489,7 +2789,7 @@ describe('CasesService', () => { unsecuredSavedObjectsClient.create.mockResolvedValue({ ...theCase, attributes }); await expect( - service.postNewCase({ + service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector() }), id: '1', }) @@ -2503,7 +2803,7 @@ describe('CasesService', () => { unsecuredSavedObjectsClient.create.mockResolvedValue({ ...theCase, attributes }); await expect( - service.postNewCase({ + service.createCase({ attributes: createCasePostParams({ connector: createJiraConnector() }), id: '1', }) @@ -2567,6 +2867,126 @@ describe('CasesService', () => { }); }); + describe('bulkCreateCases', () => { + it('decodes correctly', async () => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [createCaseSavedObjectResponse()], + }); + + await expect( + service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ connector: createJiraConnector() }), + id: '1', + }, + ], + }) + ).resolves.not.toThrow(); + }); + + it.each(Object.keys(attributesToValidateIfMissing))( + 'throws if %s is omitted', + async (key) => { + const theCase = createCaseSavedObjectResponse(); + const attributes = omit({ ...theCase.attributes }, key); + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [{ ...theCase, attributes }], + }); + + await expect( + service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ connector: createJiraConnector() }), + id: '1', + }, + ], + }) + ).rejects.toThrow(`Invalid value "undefined" supplied to "${key}"`); + } + ); + + it('strips out excess attributes', async () => { + const theCase = createCaseSavedObjectResponse(); + const attributes = { ...theCase.attributes, 'not-exists': 'not-exists' }; + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [{ ...theCase, attributes }], + }); + + await expect( + service.bulkCreateCases({ + cases: [ + { + ...createCasePostParams({ connector: createJiraConnector() }), + id: '1', + }, + ], + }) + ).resolves.toMatchInlineSnapshot(` + Object { + "saved_objects": Array [ + Object { + "attributes": Object { + "assignees": Array [], + "category": null, + "closed_at": null, + "closed_by": null, + "connector": Object { + "fields": null, + "id": "none", + "name": "none", + "type": ".none", + }, + "created_at": "2019-11-25T21:54:48.952Z", + "created_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + "customFields": Array [], + "description": "This is a brand new case of a bad meanie defacing data", + "duration": null, + "external_service": Object { + "connector_id": "none", + "connector_name": ".jira", + "external_id": "100", + "external_title": "awesome", + "external_url": "http://www.google.com", + "pushed_at": "2019-11-25T21:54:48.952Z", + "pushed_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "owner": "securitySolution", + "settings": Object { + "syncAlerts": true, + }, + "severity": "low", + "status": "open", + "tags": Array [ + "defacement", + ], + "title": "Super Bad Security Issue", + "updated_at": "2019-11-25T21:54:48.952Z", + "updated_by": Object { + "email": "testemail@elastic.co", + "full_name": "elastic", + "username": "elastic", + }, + }, + "id": "1", + "references": Array [], + "type": "cases", + }, + ], + } + `); + }); + }); + describe('patchCase', () => { it('decodes correctly', async () => { unsecuredSavedObjectsClient.update.mockResolvedValue(createUpdateSOResponse()); @@ -2732,7 +3152,7 @@ describe('CasesService', () => { }); describe('Decoding requests', () => { - describe('create case', () => { + describe('createCase', () => { beforeEach(() => { unsecuredSavedObjectsClient.create.mockResolvedValue(createCaseSavedObjectResponse()); }); @@ -2740,7 +3160,7 @@ describe('CasesService', () => { it('decodes correctly the requested attributes', async () => { const attributes = createCasePostParams({ connector: createJiraConnector() }); - await expect(service.postNewCase({ id: 'a', attributes })).resolves.not.toThrow(); + await expect(service.createCase({ id: 'a', attributes })).resolves.not.toThrow(); }); it('throws if title is omitted', async () => { @@ -2748,7 +3168,7 @@ describe('CasesService', () => { unset(attributes, 'title'); await expect( - service.postNewCase({ + service.createCase({ attributes, id: '1', }) @@ -2761,13 +3181,54 @@ describe('CasesService', () => { foo: 'bar', }; - await expect(service.postNewCase({ id: 'a', attributes })).resolves.not.toThrow(); + await expect(service.createCase({ id: 'a', attributes })).resolves.not.toThrow(); const persistedAttributes = unsecuredSavedObjectsClient.create.mock.calls[0][1]; expect(persistedAttributes).not.toHaveProperty('foo'); }); }); + describe('bulkCreateCases', () => { + beforeEach(() => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [createCaseSavedObjectResponse()], + }); + }); + + it('decodes correctly the requested attributes', async () => { + const attributes = createCasePostParams({ connector: createJiraConnector() }); + + await expect( + service.bulkCreateCases({ cases: [{ id: 'a', ...attributes }] }) + ).resolves.not.toThrow(); + }); + + it('throws if title is omitted', async () => { + const attributes = createCasePostParams({ connector: createJiraConnector() }); + unset(attributes, 'title'); + + await expect( + service.bulkCreateCases({ cases: [{ id: '1', ...attributes }] }) + ).rejects.toThrow(`Invalid value "undefined" supplied to "title"`); + }); + + it('remove excess fields', async () => { + const attributes = { + ...createCasePostParams({ connector: createJiraConnector() }), + foo: 'bar', + }; + + await expect( + service.bulkCreateCases({ cases: [{ id: 'a', ...attributes }] }) + ).resolves.not.toThrow(); + + const persistedAttributes = + unsecuredSavedObjectsClient.bulkCreate.mock.calls[0][0][0].attributes; + + expect(persistedAttributes).not.toHaveProperty('foo'); + }); + }); + describe('patch case', () => { beforeEach(() => { unsecuredSavedObjectsClient.update.mockResolvedValue( diff --git a/x-pack/plugins/cases/server/services/cases/index.ts b/x-pack/plugins/cases/server/services/cases/index.ts index 8ead5738d51957..5942f5ce1096cc 100644 --- a/x-pack/plugins/cases/server/services/cases/index.ts +++ b/x-pack/plugins/cases/server/services/cases/index.ts @@ -31,7 +31,11 @@ import { MAX_DOCS_PER_PAGE, } from '../../../common/constants'; import { decodeOrThrow } from '../../../common/api'; -import type { SavedObjectFindOptionsKueryNode, SOWithErrors } from '../../common/types'; +import type { + SavedObjectFindOptionsKueryNode, + SavedObjectsBulkResponseWithErrors, + SOWithErrors, +} from '../../common/types'; import { defaultSortField, flattenCaseSavedObject, isSOError } from '../../common/utils'; import { DEFAULT_PAGE, DEFAULT_PER_PAGE } from '../../routes/api'; import { combineFilters } from '../../client/utils'; @@ -67,10 +71,11 @@ import type { FindCaseCommentsArgs, GetReportersArgs, GetTagsArgs, - PostCaseArgs, + CreateCaseArgs, PatchCaseArgs, PatchCasesArgs, GetCategoryArgs, + BulkCreateCasesArgs, } from './types'; import type { AttachmentTransformedAttributes } from '../../common/types/attachments'; import { bulkDecodeSOAttributes } from '../utils'; @@ -589,13 +594,13 @@ export class CasesService { } } - public async postNewCase({ + public async createCase({ attributes, id, refresh, - }: PostCaseArgs): Promise { + }: CreateCaseArgs): Promise { try { - this.log.debug(`Attempting to POST a new case`); + this.log.debug(`Attempting to create a new case`); const decodedAttributes = decodeOrThrow(CaseTransformedAttributesRt)(attributes); const transformedAttributes = transformAttributesToESModel(decodedAttributes); @@ -614,7 +619,57 @@ export class CasesService { return { ...res, attributes: decodedRes }; } catch (error) { - this.log.error(`Error on POST a new case: ${error}`); + this.log.error(`Error on creating a new case: ${error}`); + throw error; + } + } + + public async bulkCreateCases({ + cases, + refresh, + }: BulkCreateCasesArgs): Promise> { + try { + this.log.debug(`Attempting to bulk create cases`); + + const bulkCreateRequest = cases.map(({ id, ...attributes }) => { + const decodedAttributes = decodeOrThrow(CaseTransformedAttributesRt)(attributes); + + const { attributes: transformedAttributes, referenceHandler } = + transformAttributesToESModel(decodedAttributes); + + transformedAttributes.total_alerts = -1; + transformedAttributes.total_comments = -1; + + return { + type: CASE_SAVED_OBJECT, + id, + attributes: transformedAttributes, + references: referenceHandler.build(), + }; + }); + + const bulkCreateResponse = + await this.unsecuredSavedObjectsClient.bulkCreate( + bulkCreateRequest, + { + refresh, + } + ); + + const res = bulkCreateResponse.saved_objects.map((theCase) => { + if (isSOError(theCase)) { + return theCase; + } + + const transformedCase = transformSavedObjectToExternalModel(theCase); + const decodedRes = decodeOrThrow(CaseTransformedAttributesRt)(transformedCase.attributes); + + return { ...transformedCase, attributes: decodedRes }; + }); + + return { saved_objects: res }; + } catch (error) { + this.log.error(`Case Service: Error on bulk creating cases: ${error}`); throw error; } } diff --git a/x-pack/plugins/cases/server/services/cases/types.ts b/x-pack/plugins/cases/server/services/cases/types.ts index 14f210134e48d4..478525f32d6fa3 100644 --- a/x-pack/plugins/cases/server/services/cases/types.ts +++ b/x-pack/plugins/cases/server/services/cases/types.ts @@ -46,11 +46,15 @@ export interface FindCaseCommentsArgs { options?: SavedObjectFindOptionsKueryNode; } -export interface PostCaseArgs extends IndexRefresh { +export interface CreateCaseArgs extends IndexRefresh { attributes: CaseTransformedAttributes; id: string; } +export interface BulkCreateCasesArgs extends IndexRefresh { + cases: Array<{ id: string } & CaseTransformedAttributes>; +} + export interface PatchCase extends IndexRefresh { caseId: string; updatedAttributes: Partial; diff --git a/x-pack/plugins/cases/server/services/mocks.ts b/x-pack/plugins/cases/server/services/mocks.ts index 7a9507be8b080c..7e2636ffbd6899 100644 --- a/x-pack/plugins/cases/server/services/mocks.ts +++ b/x-pack/plugins/cases/server/services/mocks.ts @@ -55,7 +55,8 @@ export const createCaseServiceMock = (): CaseServiceMock => { getResolveCase: jest.fn(), getTags: jest.fn(), getReporters: jest.fn(), - postNewCase: jest.fn(), + createCase: jest.fn(), + bulkCreateCases: jest.fn(), patchCase: jest.fn(), patchCases: jest.fn(), findCasesGroupedByID: jest.fn(), @@ -101,6 +102,7 @@ const createUserActionPersisterServiceMock = (): CaseUserActionPersisterServiceM bulkCreateAttachmentDeletion: jest.fn(), bulkCreateAttachmentCreation: jest.fn(), createUserAction: jest.fn(), + bulkCreateUserAction: jest.fn(), }; return service as unknown as CaseUserActionPersisterServiceMock; diff --git a/x-pack/plugins/cases/server/services/notifications/email_notification_service.ts b/x-pack/plugins/cases/server/services/notifications/email_notification_service.ts index 73ae62d5822891..e17eb2f22f7bcb 100644 --- a/x-pack/plugins/cases/server/services/notifications/email_notification_service.ts +++ b/x-pack/plugins/cases/server/services/notifications/email_notification_service.ts @@ -13,7 +13,7 @@ import type { UserProfileUserInfo } from '@kbn/user-profile-components'; import { CASE_SAVED_OBJECT, MAX_CONCURRENT_SEARCHES } from '../../../common/constants'; import type { CaseSavedObjectTransformed } from '../../common/types/case'; import { getCaseViewPath } from '../../common/utils'; -import type { NotificationService, NotifyArgs } from './types'; +import type { NotificationService, NotifyAssigneesArgs } from './types'; import { assigneesTemplateRenderer } from './templates/assignees/renderer'; type WithRequiredProperty = T & Required>; @@ -86,7 +86,7 @@ export class EmailNotificationService implements NotificationService { return assigneesTemplateRenderer(theCase, caseUrl); } - public async notifyAssignees({ assignees, theCase }: NotifyArgs) { + public async notifyAssignees({ assignees, theCase }: NotifyAssigneesArgs) { try { if (!this.notifications.isEmailServiceAvailable()) { this.logger.warn('Could not notifying assignees. Email service is not available.'); @@ -139,14 +139,14 @@ export class EmailNotificationService implements NotificationService { } } - public async bulkNotifyAssignees(casesAndAssigneesToNotifyForAssignment: NotifyArgs[]) { + public async bulkNotifyAssignees(casesAndAssigneesToNotifyForAssignment: NotifyAssigneesArgs[]) { if (casesAndAssigneesToNotifyForAssignment.length === 0) { return; } await pMap( casesAndAssigneesToNotifyForAssignment, - (args: NotifyArgs) => this.notifyAssignees(args), + (args: NotifyAssigneesArgs) => this.notifyAssignees(args), { concurrency: MAX_CONCURRENT_SEARCHES, } diff --git a/x-pack/plugins/cases/server/services/notifications/types.ts b/x-pack/plugins/cases/server/services/notifications/types.ts index d89184f03b01c9..7cbbc33cf808e1 100644 --- a/x-pack/plugins/cases/server/services/notifications/types.ts +++ b/x-pack/plugins/cases/server/services/notifications/types.ts @@ -8,12 +8,12 @@ import type { CaseAssignees } from '../../../common/types/domain'; import type { CaseSavedObjectTransformed } from '../../common/types/case'; -export interface NotifyArgs { +export interface NotifyAssigneesArgs { assignees: CaseAssignees; theCase: CaseSavedObjectTransformed; } export interface NotificationService { - notifyAssignees: (args: NotifyArgs) => Promise; - bulkNotifyAssignees: (args: NotifyArgs[]) => Promise; + notifyAssignees: (args: NotifyAssigneesArgs) => Promise; + bulkNotifyAssignees: (args: NotifyAssigneesArgs[]) => Promise; } diff --git a/x-pack/plugins/cases/server/services/test_utils.ts b/x-pack/plugins/cases/server/services/test_utils.ts index 022a868ee0d9b6..cd0f16d66e4cbc 100644 --- a/x-pack/plugins/cases/server/services/test_utils.ts +++ b/x-pack/plugins/cases/server/services/test_utils.ts @@ -26,6 +26,7 @@ import type { ConnectorPersistedFields } from '../common/types/connectors'; import type { CasePersistedAttributes } from '../common/types/case'; import { CasePersistedSeverity, CasePersistedStatus } from '../common/types/case'; import type { ExternalServicePersisted } from '../common/types/external_service'; +import type { SOWithErrors } from '../common/types'; /** * This is only a utility interface to help with constructing test cases. After the migration, the ES format will no longer @@ -271,3 +272,14 @@ export const mockPointInTimeFinder = }, }); }; + +export const createErrorSO = (type: string): SOWithErrors => ({ + id: '1', + type, + error: { + error: 'error', + message: 'message', + statusCode: 500, + }, + references: [], +}); diff --git a/x-pack/plugins/cases/server/services/user_actions/index.test.ts b/x-pack/plugins/cases/server/services/user_actions/index.test.ts index 905e94bc11d8a4..20c06f2701fedc 100644 --- a/x-pack/plugins/cases/server/services/user_actions/index.test.ts +++ b/x-pack/plugins/cases/server/services/user_actions/index.test.ts @@ -107,9 +107,11 @@ describe('CaseUserActionService', () => { describe('create case', () => { it('creates a create case user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: casePayload, - type: UserActionTypes.create_case, + userAction: { + ...commonArgs, + payload: casePayload, + type: UserActionTypes.create_case, + }, }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( @@ -159,9 +161,11 @@ describe('CaseUserActionService', () => { it('logs a create case user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: casePayload, - type: UserActionTypes.create_case, + userAction: { + ...commonArgs, + payload: casePayload, + type: UserActionTypes.create_case, + }, }); expect(mockAuditLogger.log).toBeCalledTimes(1); @@ -193,9 +197,11 @@ describe('CaseUserActionService', () => { describe('status', () => { it('creates an update status user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: { status: CaseStatuses.closed }, - type: UserActionTypes.status, + userAction: { + ...commonArgs, + payload: { status: CaseStatuses.closed }, + type: UserActionTypes.status, + }, }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( @@ -218,9 +224,11 @@ describe('CaseUserActionService', () => { it('logs an update status user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: { status: CaseStatuses.closed }, - type: UserActionTypes.status, + userAction: { + ...commonArgs, + payload: { status: CaseStatuses.closed }, + type: UserActionTypes.status, + }, }); expect(mockAuditLogger.log).toBeCalledTimes(1); @@ -253,9 +261,11 @@ describe('CaseUserActionService', () => { describe('severity', () => { it('creates an update severity user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: { severity: CaseSeverity.MEDIUM }, - type: UserActionTypes.severity, + userAction: { + ...commonArgs, + payload: { severity: CaseSeverity.MEDIUM }, + type: UserActionTypes.severity, + }, }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( @@ -278,9 +288,11 @@ describe('CaseUserActionService', () => { it('logs an update severity user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: { severity: CaseSeverity.MEDIUM }, - type: UserActionTypes.severity, + userAction: { + ...commonArgs, + payload: { severity: CaseSeverity.MEDIUM }, + type: UserActionTypes.severity, + }, }); expect(mockAuditLogger.log).toBeCalledTimes(1); @@ -313,9 +325,11 @@ describe('CaseUserActionService', () => { describe('push', () => { it('creates a push user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: { externalService }, - type: UserActionTypes.pushed, + userAction: { + ...commonArgs, + payload: { externalService }, + type: UserActionTypes.pushed, + }, }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( @@ -357,9 +371,11 @@ describe('CaseUserActionService', () => { it('logs a push user action', async () => { await service.creator.createUserAction({ - ...commonArgs, - payload: { externalService }, - type: UserActionTypes.pushed, + userAction: { + ...commonArgs, + payload: { externalService }, + type: UserActionTypes.pushed, + }, }); expect(mockAuditLogger.log).toBeCalledTimes(1); @@ -396,11 +412,13 @@ describe('CaseUserActionService', () => { [UserActionActions.update], ])('creates a comment user action of action: %s', async (action) => { await service.creator.createUserAction({ - ...commonArgs, - type: UserActionTypes.comment, - action, - attachmentId: 'test-id', - payload: { attachment: comment }, + userAction: { + ...commonArgs, + type: UserActionTypes.comment, + action, + attachmentId: 'test-id', + payload: { attachment: comment }, + }, }); expect(unsecuredSavedObjectsClient.create).toHaveBeenCalledWith( @@ -438,11 +456,13 @@ describe('CaseUserActionService', () => { [UserActionActions.update], ])('logs a comment user action of action: %s', async (action) => { await service.creator.createUserAction({ - ...commonArgs, - type: UserActionTypes.comment, - action, - attachmentId: 'test-id', - payload: { attachment: comment }, + userAction: { + ...commonArgs, + type: UserActionTypes.comment, + action, + attachmentId: 'test-id', + payload: { attachment: comment }, + }, }); expect(mockAuditLogger.log).toBeCalledTimes(1); diff --git a/x-pack/plugins/cases/server/services/user_actions/operations/create.test.ts b/x-pack/plugins/cases/server/services/user_actions/operations/create.test.ts index 853c4969237f5c..833e8676a26195 100644 --- a/x-pack/plugins/cases/server/services/user_actions/operations/create.test.ts +++ b/x-pack/plugins/cases/server/services/user_actions/operations/create.test.ts @@ -15,7 +15,11 @@ import { set, unset } from 'lodash'; import { createConnectorObject } from '../../test_utils'; import { UserActionPersister } from './create'; import { createUserActionSO } from '../test_utils'; -import type { BulkCreateAttachmentUserAction, CreateUserActionClient } from '../types'; +import type { + BuilderParameters, + BulkCreateAttachmentUserAction, + CreateUserActionArgs, +} from '../types'; import type { UserActionPersistedAttributes } from '../../../common/types/user_actions'; import { getAssigneesAddedRemovedUserActions, @@ -65,16 +69,19 @@ describe('UserActionPersister', () => { jest.useRealTimers(); }); - const getRequest = () => + const getRequest = (overrides = {}) => ({ - action: 'update' as const, - type: 'connector' as const, - caseId: 'test', - payload: { connector: createConnectorObject().connector }, - connectorId: '1', - owner: 'cases', - user: { email: '', full_name: '', username: '' }, - } as CreateUserActionClient<'connector'>); + userAction: { + action: 'update' as const, + type: 'connector' as const, + caseId: 'test', + payload: { connector: createConnectorObject().connector }, + connectorId: '1', + owner: 'cases', + user: { email: '', full_name: '', username: '' }, + ...overrides, + }, + } as CreateUserActionArgs); const getBulkCreateAttachmentRequest = (): BulkCreateAttachmentUserAction => ({ caseId: 'test', @@ -107,7 +114,7 @@ describe('UserActionPersister', () => { it('throws if fields is omitted', async () => { const req = getRequest(); - unset(req, 'payload.connector.fields'); + unset(req, 'userAction.payload.connector.fields'); await expect(persister.createUserAction(req)).rejects.toThrow( 'Invalid value "undefined" supplied to "payload,connector,fields"' @@ -141,6 +148,56 @@ describe('UserActionPersister', () => { }); }); + it('decodes correctly the requested attributes', async () => { + await expect( + persister.bulkCreateUserAction({ + userActions: [getRequest().userAction], + }) + ).resolves.not.toThrow(); + }); + + it('throws if owner is omitted', async () => { + const req = getRequest().userAction; + unset(req, 'owner'); + + await expect( + persister.bulkCreateUserAction({ + userActions: [req], + }) + ).rejects.toThrow('Invalid value "undefined" supplied to "owner"'); + }); + + it('strips out excess attributes', async () => { + const req = getRequest().userAction; + set(req, 'payload.foo', 'bar'); + + await expect( + persister.bulkCreateUserAction({ + userActions: [req], + }) + ).resolves.not.toThrow(); + + const persistedAttributes = unsecuredSavedObjectsClient.bulkCreate.mock.calls[0][0][0] + .attributes as UserActionPersistedAttributes; + + expect(persistedAttributes.payload).not.toHaveProperty('foo'); + }); + }); + + describe('bulkCreateUserAction', () => { + beforeEach(() => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [ + { + attributes: createUserActionSO(), + id: '1', + type: CASE_USER_ACTION_SAVED_OBJECT, + references: [], + }, + ], + }); + }); + it('decodes correctly the requested attributes', async () => { await expect( persister.bulkCreateAttachmentCreation(getBulkCreateAttachmentRequest()) @@ -524,4 +581,103 @@ describe('UserActionPersister', () => { }); }); }); + + describe('bulkCreateUserAction', () => { + beforeEach(() => { + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValue({ + saved_objects: [ + { + attributes: [createUserActionSO()], + id: '1', + type: CASE_USER_ACTION_SAVED_OBJECT, + references: [], + }, + { + attributes: [createUserActionSO()], + id: '2', + type: CASE_USER_ACTION_SAVED_OBJECT, + references: [], + }, + ], + }); + }); + + it('bulk creates the user actions correctly', async () => { + const connectorUserAction = getRequest().userAction; + const titleUserAction = getRequest<'title'>({ + type: 'title', + payload: { title: 'my title' }, + }).userAction; + + await persister.bulkCreateUserAction({ + userActions: [connectorUserAction, titleUserAction], + }); + + expect(unsecuredSavedObjectsClient.bulkCreate.mock.calls[0][0]).toMatchInlineSnapshot(` + Array [ + Object { + "attributes": Object { + "action": "update", + "created_at": "2022-01-09T22:00:00.000Z", + "created_by": Object { + "email": "", + "full_name": "", + "username": "", + }, + "owner": "cases", + "payload": Object { + "connector": Object { + "fields": Object { + "issueType": "bug", + "parent": "2", + "priority": "high", + }, + "name": ".jira", + "type": ".jira", + }, + }, + "type": "connector", + }, + "references": Array [ + Object { + "id": "test", + "name": "associated-cases", + "type": "cases", + }, + Object { + "id": "1", + "name": "connectorId", + "type": "action", + }, + ], + "type": "cases-user-actions", + }, + Object { + "attributes": Object { + "action": "update", + "created_at": "2022-01-09T22:00:00.000Z", + "created_by": Object { + "email": "", + "full_name": "", + "username": "", + }, + "owner": "cases", + "payload": Object { + "title": "my title", + }, + "type": "title", + }, + "references": Array [ + Object { + "id": "test", + "name": "associated-cases", + "type": "cases", + }, + ], + "type": "cases-user-actions", + }, + ] + `); + }); + }); }); diff --git a/x-pack/plugins/cases/server/services/user_actions/operations/create.ts b/x-pack/plugins/cases/server/services/user_actions/operations/create.ts index 1f2ba3dc8f0463..e66f375e391081 100644 --- a/x-pack/plugins/cases/server/services/user_actions/operations/create.ts +++ b/x-pack/plugins/cases/server/services/user_actions/operations/create.ts @@ -30,7 +30,6 @@ import type { BulkCreateBulkUpdateCaseUserActions, CommonUserActionArgs, CreatePayloadFunction, - CreateUserActionClient, CreateUserActionES, GetUserActionItemByDifference, PostCaseUserActionArgs, @@ -38,6 +37,8 @@ import type { TypedUserActionDiffedItems, UserActionEvent, UserActionsDict, + CreateUserActionArgs, + BulkCreateUserActionArgs, } from '../types'; import { isAssigneesArray, isCustomFieldsArray, isStringArray } from '../type_guards'; import type { IndexRefresh } from '../../types'; @@ -375,21 +376,16 @@ export class UserActionPersister { } public async createUserAction({ - action, - type, - caseId, - user, - owner, - payload, - connectorId, - attachmentId, + userAction, refresh, - }: CreateUserActionClient): Promise { + }: CreateUserActionArgs): Promise { + const { action, type, caseId, user, owner, payload, connectorId, attachmentId } = userAction; + try { this.context.log.debug(`Attempting to create a user action of type: ${type}`); const userActionBuilder = this.builderFactory.getBuilder(type); - const userAction = userActionBuilder?.build({ + const userActionPayload = userActionBuilder?.build({ action, caseId, user, @@ -399,8 +395,8 @@ export class UserActionPersister { payload, }); - if (userAction) { - await this.createAndLog({ userAction, refresh }); + if (userActionPayload) { + await this.createAndLog({ userAction: userActionPayload, refresh }); } } catch (error) { this.context.log.error(`Error on creating user action of type: ${type}. Error: ${error}`); @@ -408,6 +404,45 @@ export class UserActionPersister { } } + public async bulkCreateUserAction({ + userActions, + refresh, + }: BulkCreateUserActionArgs): Promise { + try { + this.context.log.debug(`Attempting to bulk create a user actions`); + + if (userActions.length <= 0) { + return; + } + + const userActionsPayload = userActions + .map(({ action, type, caseId, user, owner, payload, connectorId, attachmentId }) => { + const userActionBuilder = this.builderFactory.getBuilder(type); + const userAction = userActionBuilder?.build({ + action, + caseId, + user, + owner, + connectorId, + attachmentId, + payload, + }); + + if (userAction == null) { + return null; + } + + return userAction; + }) + .filter(Boolean) as UserActionEvent[]; + + await this.bulkCreateAndLog({ userActions: userActionsPayload, refresh }); + } catch (error) { + this.context.log.error(`Error on bulk creating user actions. Error: ${error}`); + throw error; + } + } + private async createAndLog({ userAction, refresh, diff --git a/x-pack/plugins/cases/server/services/user_actions/types.ts b/x-pack/plugins/cases/server/services/user_actions/types.ts index 42ebb7e4135821..4b99651300852c 100644 --- a/x-pack/plugins/cases/server/services/user_actions/types.ts +++ b/x-pack/plugins/cases/server/services/user_actions/types.ts @@ -312,9 +312,13 @@ export interface BulkCreateAttachmentUserAction attachments: Array<{ id: string; owner: string; attachment: AttachmentRequest }>; } -export type CreateUserActionClient = CreateUserAction & - CommonUserActionArgs & - IndexRefresh; +export type CreateUserActionArgs = { + userAction: CreateUserAction & CommonUserActionArgs; +} & IndexRefresh; + +export type BulkCreateUserActionArgs = { + userActions: Array & CommonUserActionArgs>; +} & IndexRefresh; export interface CreateUserActionES extends IndexRefresh { attributes: T; diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts index fda0b54995233a..def4eca415d428 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.test.ts @@ -4,8 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ -import { executeAction } from './executor'; +import { executeAction, Props } from './executor'; +import { PassThrough } from 'stream'; import { KibanaRequest } from '@kbn/core-http-server'; import { RequestBody } from './langchain/types'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; @@ -19,27 +26,86 @@ describe('executeAction', () => { beforeEach(() => { jest.clearAllMocks(); }); - it('should execute an action with the provided connectorId and request body params', async () => { + it('should execute an action and return a StaticResponse when the response from the actions framework is a string', async () => { const actions = { getActionsClientWithRequest: jest.fn().mockResolvedValue({ execute: jest.fn().mockResolvedValue({ - status: 'ok', data: { message: 'Test message', }, }), }), - } as unknown as ActionsPluginStart; + }; + const connectorId = 'testConnectorId'; - const connectorId = '12345'; + const result = await executeAction({ actions, request, connectorId } as unknown as Props); + + expect(result).toEqual({ + connector_id: connectorId, + data: 'Test message', + status: 'ok', + }); + }); + + it('should execute an action and return a Readable object when the response from the actions framework is a stream', async () => { + const readableStream = new PassThrough(); + const actions = { + getActionsClientWithRequest: jest.fn().mockResolvedValue({ + execute: jest.fn().mockResolvedValue({ + data: readableStream, + }), + }), + }; + const connectorId = 'testConnectorId'; + + const result = await executeAction({ actions, request, connectorId } as unknown as Props); - const response = await executeAction({ actions, request, connectorId }); + expect(JSON.stringify(result)).toStrictEqual( + JSON.stringify(readableStream.pipe(new PassThrough())) + ); + }); + + it('should throw an error if the actions plugin fails to retrieve the actions client', async () => { + const actions = { + getActionsClientWithRequest: jest + .fn() + .mockRejectedValue(new Error('Failed to retrieve actions client')), + }; + const connectorId = 'testConnectorId'; + + await expect( + executeAction({ actions, request, connectorId } as unknown as Props) + ).rejects.toThrowError('Failed to retrieve actions client'); + }); + + it('should throw an error if the actions client fails to execute the action', async () => { + const actions = { + getActionsClientWithRequest: jest.fn().mockResolvedValue({ + execute: jest.fn().mockRejectedValue(new Error('Failed to execute action')), + }), + }; + const connectorId = 'testConnectorId'; + + await expect( + executeAction({ actions, request, connectorId } as unknown as Props) + ).rejects.toThrowError('Failed to execute action'); + }); + + it('should throw an error when the response from the actions framework is null or undefined', async () => { + const actions = { + getActionsClientWithRequest: jest.fn().mockResolvedValue({ + execute: jest.fn().mockResolvedValue({ + data: null, + }), + }), + }; + const connectorId = 'testConnectorId'; - expect(actions.getActionsClientWithRequest).toHaveBeenCalledWith(request); - expect(actions.getActionsClientWithRequest).toHaveBeenCalledTimes(1); - expect(response.connector_id).toBe(connectorId); - expect(response.data).toBe('Test message'); - expect(response.status).toBe('ok'); + try { + await executeAction({ actions, request, connectorId } as unknown as Props); + } catch (e) { + expect(e.message).toBe('Action result status is error: result is not streamable'); + } }); it('should throw an error if action result status is "error"', async () => { @@ -59,7 +125,7 @@ describe('executeAction', () => { ); }); - it('should throw an error if content of response data is not a string', async () => { + it('should throw an error if content of response data is not a string or streamable', async () => { const actions = { getActionsClientWithRequest: jest.fn().mockResolvedValue({ execute: jest.fn().mockResolvedValue({ @@ -73,7 +139,7 @@ describe('executeAction', () => { const connectorId = '12345'; await expect(executeAction({ actions, request, connectorId })).rejects.toThrowError( - 'Action result status is error: content should be a string, but it had an unexpected type: number' + 'Action result status is error: result is not streamable' ); }); }); diff --git a/x-pack/plugins/elastic_assistant/server/lib/executor.ts b/x-pack/plugins/elastic_assistant/server/lib/executor.ts index 21c0962e1c370c..88266914f36edf 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/executor.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/executor.ts @@ -8,9 +8,10 @@ import { get } from 'lodash/fp'; import { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; import { KibanaRequest } from '@kbn/core-http-server'; +import { PassThrough, Readable } from 'stream'; import { RequestBody } from './langchain/types'; -interface Props { +export interface Props { actions: ActionsPluginStart; connectorId: string; request: KibanaRequest; @@ -25,11 +26,20 @@ export const executeAction = async ({ actions, request, connectorId, -}: Props): Promise => { +}: Props): Promise => { const actionsClient = await actions.getActionsClientWithRequest(request); + const actionResult = await actionsClient.execute({ actionId: connectorId, - params: request.body.params, + params: { + ...request.body.params, + subActionParams: + // TODO: Remove in part 2 of streaming work for security solution + // tracked here: https://github.com/elastic/security-team/issues/7363 + request.body.params.subAction === 'invokeAI' + ? request.body.params.subActionParams + : { body: JSON.stringify(request.body.params.subActionParams), stream: true }, + }, }); if (actionResult.status === 'error') { @@ -38,14 +48,18 @@ export const executeAction = async ({ ); } const content = get('data.message', actionResult); - if (typeof content !== 'string') { - throw new Error( - `Action result status is error: content should be a string, but it had an unexpected type: ${typeof content}` - ); + if (typeof content === 'string') { + return { + connector_id: connectorId, + data: content, // the response from the actions framework + status: 'ok', + }; } - return { - connector_id: connectorId, - data: content, // the response from the actions framework - status: 'ok', - }; + const readable = get('data', actionResult) as Readable; + + if (typeof readable?.read !== 'function') { + throw new Error('Action result status is error: result is not streamable'); + } + + return readable.pipe(new PassThrough()); }; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts index c13977ddb1e7d0..5f21ef9707d44e 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/helpers.ts @@ -13,13 +13,13 @@ export const getLangChainMessage = ( ): BaseMessage => { switch (assistantMessage.role) { case 'system': - return new SystemMessage(assistantMessage.content); + return new SystemMessage(assistantMessage.content ?? ''); case 'user': - return new HumanMessage(assistantMessage.content); + return new HumanMessage(assistantMessage.content ?? ''); case 'assistant': - return new AIMessage(assistantMessage.content); + return new AIMessage(assistantMessage.content ?? ''); default: - return new HumanMessage(assistantMessage.content); + return new HumanMessage(assistantMessage.content ?? ''); } }; diff --git a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_clone.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_clone.test.tsx index b81c4ce59ea3c6..ecd3e617635f14 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_clone.test.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_clone.test.tsx @@ -40,6 +40,7 @@ const templateToClone = getComposableTemplate({ template: { mappings: MAPPINGS, }, + allowAutoCreate: true, }); describe('', () => { @@ -97,7 +98,7 @@ describe('', () => { actions.clickNextButton(); }); - const { priority, version, _kbnMeta } = templateToClone; + const { priority, version, _kbnMeta, allowAutoCreate } = templateToClone; expect(httpSetup.post).toHaveBeenLastCalledWith( `${API_BASE_PATH}/index_templates`, expect.objectContaining({ @@ -106,6 +107,7 @@ describe('', () => { indexPatterns: DEFAULT_INDEX_PATTERNS, priority, version, + allowAutoCreate, _kbnMeta, }), }) diff --git a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx index 8fb60ec708fd0e..7558ae7ce272a3 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx @@ -532,6 +532,7 @@ describe('', () => { await actions.completeStepOne({ name: TEMPLATE_NAME, indexPatterns: DEFAULT_INDEX_PATTERNS, + allowAutoCreate: true, }); // Component templates await actions.completeStepTwo('test_component_template_1'); @@ -558,6 +559,7 @@ describe('', () => { body: JSON.stringify({ name: TEMPLATE_NAME, indexPatterns: DEFAULT_INDEX_PATTERNS, + allowAutoCreate: true, _kbnMeta: { type: 'default', hasDatastream: false, @@ -631,6 +633,7 @@ describe('', () => { body: JSON.stringify({ index_patterns: DEFAULT_INDEX_PATTERNS, data_stream: {}, + allow_auto_create: false, }), }) ); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_edit.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_edit.test.tsx index 359f1472aec62b..157cec2f91cd4c 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_edit.test.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_edit.test.tsx @@ -140,6 +140,7 @@ describe('', () => { name: 'test', indexPatterns: ['myPattern*'], version: 1, + allowAutoCreate: false, dataStream: { hidden: true, anyUnknownKey: 'should_be_kept', @@ -198,6 +199,7 @@ describe('', () => { await actions.completeStepOne({ indexPatterns: UPDATED_INDEX_PATTERN, priority: 3, + allowAutoCreate: true, }); // Component templates await actions.completeStepTwo(); @@ -252,6 +254,7 @@ describe('', () => { indexPatterns: UPDATED_INDEX_PATTERN, priority: 3, version: templateToEdit.version, + allowAutoCreate: true, _kbnMeta: { type: 'default', hasDatastream: false, diff --git a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts index bf16e8e5e803d5..0974ee79346ed5 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_form.helpers.ts @@ -146,6 +146,7 @@ export const formSetup = async (initTestBed: SetupFunc) => { priority, version, dataStream, + allowAutoCreate, }: Partial = {}) => { const { component, form, find } = testBed; @@ -184,6 +185,10 @@ export const formSetup = async (initTestBed: SetupFunc) => { form.setInputValue('versionField.input', JSON.stringify(version)); } + if (allowAutoCreate) { + form.toggleEuiSwitch('allowAutoCreateField.input'); + } + clickNextButton(); }); @@ -332,6 +337,7 @@ export type TestSubjects = | 'orderField.input' | 'priorityField.input' | 'dataStreamField.input' + | 'allowAutoCreateField.input' | 'pageTitle' | 'previewTab' | 'removeFieldButton' diff --git a/x-pack/plugins/index_management/common/lib/template_serialization.ts b/x-pack/plugins/index_management/common/lib/template_serialization.ts index ce49c32cd82715..7c75ec3c56f972 100644 --- a/x-pack/plugins/index_management/common/lib/template_serialization.ts +++ b/x-pack/plugins/index_management/common/lib/template_serialization.ts @@ -16,8 +16,16 @@ import { const hasEntries = (data: object = {}) => Object.entries(data).length > 0; export function serializeTemplate(templateDeserialized: TemplateDeserialized): TemplateSerialized { - const { version, priority, indexPatterns, template, composedOf, dataStream, _meta } = - templateDeserialized; + const { + version, + priority, + indexPatterns, + template, + composedOf, + dataStream, + _meta, + allowAutoCreate, + } = templateDeserialized; return { version, @@ -26,6 +34,7 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T index_patterns: indexPatterns, data_stream: dataStream, composed_of: composedOf, + allow_auto_create: allowAutoCreate, _meta, }; } @@ -43,6 +52,7 @@ export function deserializeTemplate( _meta, composed_of: composedOf, data_stream: dataStream, + allow_auto_create: allowAutoCreate, } = templateEs; const { settings } = template; @@ -64,6 +74,7 @@ export function deserializeTemplate( ilmPolicy: settings?.index?.lifecycle, composedOf, dataStream, + allowAutoCreate, _meta, _kbnMeta: { type, diff --git a/x-pack/plugins/index_management/common/types/templates.ts b/x-pack/plugins/index_management/common/types/templates.ts index ca1433053cb9d7..53e120c88c96ff 100644 --- a/x-pack/plugins/index_management/common/types/templates.ts +++ b/x-pack/plugins/index_management/common/types/templates.ts @@ -24,6 +24,7 @@ export interface TemplateSerialized { priority?: number; _meta?: { [key: string]: any }; data_stream?: {}; + allow_auto_create?: boolean; } /** @@ -42,6 +43,7 @@ export interface TemplateDeserialized { composedOf?: string[]; // Composable template only version?: number; priority?: number; // Composable template only + allowAutoCreate?: boolean; order?: number; // Legacy template only ilmPolicy?: { name: string; diff --git a/x-pack/plugins/index_management/public/application/components/template_form/steps/step_logistics.tsx b/x-pack/plugins/index_management/public/application/components/template_form/steps/step_logistics.tsx index f818446b32ebe3..6dd062ba426745 100644 --- a/x-pack/plugins/index_management/public/application/components/template_form/steps/step_logistics.tsx +++ b/x-pack/plugins/index_management/public/application/components/template_form/steps/step_logistics.tsx @@ -112,6 +112,19 @@ function getFieldsMeta(esDocsBase: string) { }), testSubject: 'versionField', }, + allowAutoCreate: { + title: i18n.translate('xpack.idxMgmt.templateForm.stepLogistics.allowAutoCreateTitle', { + defaultMessage: 'Allow auto create', + }), + description: i18n.translate( + 'xpack.idxMgmt.templateForm.stepLogistics.allowAutoCreateDescription', + { + defaultMessage: + 'Indices can be automatically created even if auto-creation of indices is disabled via actions.auto_create_index.', + } + ), + testSubject: 'allowAutoCreateField', + }, }; } @@ -185,9 +198,8 @@ export const StepLogistics: React.FunctionComponent = React.memo( }); }, [onChange, isFormValid, validate, getFormData]); - const { name, indexPatterns, createDataStream, order, priority, version } = getFieldsMeta( - documentationService.getEsDocsBase() - ); + const { name, indexPatterns, createDataStream, order, priority, version, allowAutoCreate } = + getFieldsMeta(documentationService.getEsDocsBase()); return ( <> @@ -294,6 +306,16 @@ export const StepLogistics: React.FunctionComponent = React.memo( /> + {/* Allow auto create */} + {isLegacy === false && ( + + + + )} + {/* _meta */} {isLegacy === false && ( ( /> ); -const getDescriptionText = (data: any) => { - const hasEntries = data && Object.entries(data).length > 0; +const getDescriptionText = (data: Aliases | boolean | undefined) => { + const hasEntries = typeof data === 'boolean' ? data : data && Object.entries(data).length > 0; return hasEntries ? ( = React.memo( version, order, priority, + allowAutoCreate, composedOf, _meta, _kbnMeta: { isLegacy }, @@ -186,6 +187,21 @@ export const StepReview: React.FunctionComponent = React.memo( {version ? version : } + {/* Allow auto create */} + {isLegacy !== true && ( + <> + + + + + {getDescriptionText(allowAutoCreate)} + + + )} + {/* components */} {isLegacy !== true && ( <> diff --git a/x-pack/plugins/index_management/public/application/components/template_form/template_form_schemas.tsx b/x-pack/plugins/index_management/public/application/components/template_form/template_form_schemas.tsx index 77cbca87c7655b..454a0ca4b5b0ae 100644 --- a/x-pack/plugins/index_management/public/application/components/template_form/template_form_schemas.tsx +++ b/x-pack/plugins/index_management/public/application/components/template_form/template_form_schemas.tsx @@ -158,6 +158,13 @@ export const schemas: Record = { }), formatters: [toInt], }, + allowAutoCreate: { + type: FIELD_TYPES.TOGGLE, + label: i18n.translate('xpack.idxMgmt.templateForm.stepLogistics.fieldAllowAutoCreateLabel', { + defaultMessage: 'Allow auto create (optional)', + }), + defaultValue: false, + }, _meta: { label: i18n.translate('xpack.idxMgmt.templateForm.stepLogistics.metaFieldEditorLabel', { defaultMessage: '_meta field data (optional)', diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx index 20a7c172d674b2..5e90a97fbcdb07 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx @@ -50,6 +50,7 @@ export const TabSummary: React.FunctionComponent = ({ templateDetails }) ilmPolicy, _meta, _kbnMeta: { isLegacy, hasDatastream }, + allowAutoCreate, } = templateDetails; const numIndexPatterns = indexPatterns.length; @@ -192,6 +193,21 @@ export const TabSummary: React.FunctionComponent = ({ templateDetails }) {version || version === 0 ? version : i18nTexts.none} + + {/* Allow auto create */} + {isLegacy !== true && ( + <> + + + + + {allowAutoCreate ? i18nTexts.yes : i18nTexts.no} + + + )} diff --git a/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts b/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts index 247eda7206fb87..d4236fb9051c46 100644 --- a/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts +++ b/x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts @@ -13,6 +13,7 @@ export const templateSchema = schema.object({ version: schema.maybe(schema.number()), order: schema.maybe(schema.number()), priority: schema.maybe(schema.number()), + allowAutoCreate: schema.maybe(schema.boolean()), template: schema.maybe( schema.object({ settings: schema.maybe(schema.object({}, { unknowns: 'allow' })), diff --git a/x-pack/plugins/index_management/test/fixtures/template.ts b/x-pack/plugins/index_management/test/fixtures/template.ts index ae9be960157350..19a3b30c84c6bb 100644 --- a/x-pack/plugins/index_management/test/fixtures/template.ts +++ b/x-pack/plugins/index_management/test/fixtures/template.ts @@ -21,6 +21,7 @@ export const getComposableTemplate = ({ hasDatastream = false, isLegacy = false, type = 'default', + allowAutoCreate = false, }: Partial< TemplateDeserialized & { isLegacy?: boolean; @@ -33,6 +34,7 @@ export const getComposableTemplate = ({ version, priority, indexPatterns, + allowAutoCreate, template: { aliases, mappings, @@ -58,6 +60,7 @@ export const getTemplate = ({ hasDatastream = false, isLegacy = false, type = 'default', + allowAutoCreate = false, }: Partial< TemplateDeserialized & { isLegacy?: boolean; @@ -70,6 +73,7 @@ export const getTemplate = ({ version, order, indexPatterns, + allowAutoCreate, template: { aliases, mappings, diff --git a/x-pack/plugins/osquery/cypress/e2e/all/add_integration.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/add_integration.cy.ts index 882bfcba65f435..6eae4381cf34be 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/add_integration.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/add_integration.cy.ts @@ -124,12 +124,13 @@ describe('ALL - Add Integration', { tags: ['@ess', '@serverless'] }, () => { cy.getBySel('epmList.searchBar').type('osquery'); cy.getBySel('integration-card:epr:osquery_manager').click(); cy.getBySel('addIntegrationPolicyButton').click(); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.getBySel('agentPolicySelect').within(() => { cy.contains(policyName); }); - cy.getBySel('packagePolicyNameInput') - .wait(500) - .type(`{selectall}{backspace}${integrationName}`); + cy.getBySel('packagePolicyNameInput').clear().wait(500); + cy.getBySel('packagePolicyNameInput').type(`${integrationName}`); cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click(); cy.getBySel('confirmModalCancelButton').click(); cy.get(`[title="${integrationName}"]`).should('exist'); diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_linked_apps.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_linked_apps.cy.ts index b5f5886da2e352..1c932435d81736 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/alerts_linked_apps.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/alerts_linked_apps.cy.ts @@ -18,8 +18,7 @@ import { import { closeModalIfVisible, closeToastIfVisible } from '../../tasks/integrations'; import { RESULTS_TABLE, RESULTS_TABLE_BUTTON } from '../../screens/live_query'; -// FLAKY: https://github.com/elastic/kibana/issues/170521 -describe.skip( +describe( 'Alert Event Details', { tags: ['@ess', '@serverless'], @@ -46,9 +45,10 @@ describe.skip( cy.getBySel('editRuleSettingsLink').click(); cy.getBySel('globalLoadingIndicator').should('not.exist'); cy.getBySel('edit-rule-actions-tab').click(); - cy.getBySel('osquery-investigation-guide-text').should('exist'); - cy.getBySel('osqueryAddInvestigationGuideQueries').should('not.be.disabled'); + cy.getBySel('globalLoadingIndicator').should('not.exist'); + cy.contains('Loading connectors...').should('not.exist'); + cy.getBySel('osqueryAddInvestigationGuideQueries').click(); cy.getBySel('osquery-investigation-guide-text').should('not.exist'); @@ -60,6 +60,7 @@ describe.skip( cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => { cy.contains('select * from users'); }); + cy.contains('Save changes').click(); cy.contains(`${ruleName} was saved`).should('exist'); closeToastIfVisible(); diff --git a/x-pack/plugins/osquery/cypress/tasks/integrations.ts b/x-pack/plugins/osquery/cypress/tasks/integrations.ts index 760f9da287dba2..3a20080a6705e7 100644 --- a/x-pack/plugins/osquery/cypress/tasks/integrations.ts +++ b/x-pack/plugins/osquery/cypress/tasks/integrations.ts @@ -34,8 +34,10 @@ export const addIntegration = (agentPolicy = DEFAULT_POLICY) => { export const addCustomIntegration = (integrationName: string, policyName: string) => { cy.getBySel(ADD_POLICY_BTN).click(); cy.getBySel(DATA_COLLECTION_SETUP_STEP).find('.euiLoadingSpinner').should('not.exist'); - cy.getBySel('packagePolicyNameInput').type(`{selectall}{backspace}${integrationName}`); - cy.getBySel('createAgentPolicyNameField').type(`{selectall}{backspace}${policyName}`); + cy.getBySel('packagePolicyNameInput').clear(); + cy.getBySel('packagePolicyNameInput').type(`${integrationName}`); + cy.getBySel('createAgentPolicyNameField').clear(); + cy.getBySel('createAgentPolicyNameField').type(`${policyName}`); cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click(); // No agent is enrolled with this policy, close "Add agent" modal cy.getBySel('confirmModalCancelButton').click(); diff --git a/x-pack/plugins/security_solution/public/assistant/comment_actions/index.tsx b/x-pack/plugins/security_solution/public/assistant/comment_actions/index.tsx index b7cac488319dc2..5a3e4a3d1c2101 100644 --- a/x-pack/plugins/security_solution/public/assistant/comment_actions/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/comment_actions/index.tsx @@ -41,17 +41,19 @@ const CommentActionsComponent: React.FC = ({ message }) => { [dispatch] ); + const content = message.content ?? ''; + const onAddNoteToTimeline = useCallback(() => { updateAndAssociateNode({ associateNote, - newNote: message.content, + newNote: content, updateNewNote: () => {}, updateNote, user: '', // TODO: attribute assistant messages }); toasts.addSuccess(i18n.ADDED_NOTE_TO_TIMELINE); - }, [associateNote, message.content, toasts, updateNote]); + }, [associateNote, content, toasts, updateNote]); // Attach to case support const selectCaseModal = cases.hooks.useCasesAddToExistingCaseModal({ @@ -65,13 +67,13 @@ const CommentActionsComponent: React.FC = ({ message }) => { selectCaseModal.open({ getAttachments: () => [ { - comment: message.content, + comment: content, type: AttachmentType.user, owner: i18n.ELASTIC_AI_ASSISTANT, }, ], }); - }, [message.content, selectCaseModal, showAssistantOverlay]); + }, [content, selectCaseModal, showAssistantOverlay]); return ( @@ -99,7 +101,7 @@ const CommentActionsComponent: React.FC = ({ message }) => { - + {(copy) => ( { it('Does not add error state message has no error', () => { - const currentConversation = { - apiConfig: {}, - id: '1', - messages: [ - { - role: user, - content: 'Hello {name}', - timestamp: '2022-01-01', - isError: false, - }, - ], - }; - const lastCommentRef = { current: null }; - const showAnonymizedValues = false; - - const result = getComments({ currentConversation, lastCommentRef, showAnonymizedValues }); + const result = getComments(testProps); expect(result[0].eventColor).toEqual(undefined); }); it('Adds error state when message has error', () => { - const currentConversation = { - apiConfig: {}, - id: '1', - messages: [ - { - role: user, - content: 'Hello {name}', - timestamp: '2022-01-01', - isError: true, - }, - ], - }; - const lastCommentRef = { current: null }; - const showAnonymizedValues = false; - - const result = getComments({ currentConversation, lastCommentRef, showAnonymizedValues }); + const result = getComments({ + ...testProps, + currentConversation: { + apiConfig: {}, + id: '1', + messages: [ + { + role: user, + content: 'Hello {name}', + timestamp: '2022-01-01', + isError: true, + }, + ], + }, + }); expect(result[0].eventColor).toEqual('danger'); }); }); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx index e75117d0c71b68..9c547b3033112b 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx @@ -6,103 +6,152 @@ */ import type { EuiCommentProps } from '@elastic/eui'; -import type { Conversation } from '@kbn/elastic-assistant'; -import { - EuiAvatar, - EuiMarkdownFormat, - EuiSpacer, - EuiText, - getDefaultEuiMarkdownParsingPlugins, - getDefaultEuiMarkdownProcessingPlugins, -} from '@elastic/eui'; +import type { Conversation, Message } from '@kbn/elastic-assistant'; +import { EuiAvatar, EuiLoadingSpinner } from '@elastic/eui'; import React from 'react'; import { AssistantAvatar } from '@kbn/elastic-assistant'; +import { getMessageContentWithReplacements } from '../helpers'; +import { StreamComment } from './stream'; import { CommentActions } from '../comment_actions'; import * as i18n from './translations'; -import { customCodeBlockLanguagePlugin } from './custom_codeblock/custom_codeblock_markdown_plugin'; -import { CustomCodeBlock } from './custom_codeblock/custom_code_block'; + +export interface ContentMessage extends Message { + content: string; +} +const transformMessageWithReplacements = ({ + message, + content, + showAnonymizedValues, + replacements, +}: { + message: Message; + content: string; + showAnonymizedValues: boolean; + replacements?: Record; +}): ContentMessage => { + return { + ...message, + content: + showAnonymizedValues || !replacements + ? content + : getMessageContentWithReplacements({ + messageContent: content, + replacements, + }), + }; +}; export const getComments = ({ + amendMessage, currentConversation, - lastCommentRef, + isFetchingResponse, + regenerateMessage, showAnonymizedValues, }: { + amendMessage: ({ conversationId, content }: { conversationId: string; content: string }) => void; currentConversation: Conversation; - lastCommentRef: React.MutableRefObject; + isFetchingResponse: boolean; + regenerateMessage: (conversationId: string) => void; showAnonymizedValues: boolean; }): EuiCommentProps[] => { - const parsingPlugins = getDefaultEuiMarkdownParsingPlugins(); - const processingPlugins = getDefaultEuiMarkdownProcessingPlugins(); - - const { components } = processingPlugins[1][1]; + const amendMessageOfConversation = (content: string) => { + amendMessage({ + conversationId: currentConversation.id, + content, + }); + }; - processingPlugins[1][1].components = { - ...components, - customCodeBlock: (props) => { - return ( - <> - - - - ); - }, + const regenerateMessageOfConversation = () => { + regenerateMessage(currentConversation.id); }; - // Fun fact: must spread existing parsingPlugins last - const parsingPluginList = [customCodeBlockLanguagePlugin, ...parsingPlugins]; - const processingPluginList = processingPlugins; + const extraLoadingComment = isFetchingResponse + ? [ + { + username: i18n.ASSISTANT, + timelineAvatar: , + timestamp: '...', + children: ( + ({ content: '' } as unknown as ContentMessage)} + isFetching + // we never need to append to a code block in the loading comment, which is what this index is used for + index={999} + /> + ), + }, + ] + : []; - return currentConversation.messages.map((message, index) => { - const isUser = message.role === 'user'; - const replacements = currentConversation.replacements; - const messageContentWithReplacements = - replacements != null - ? Object.keys(replacements).reduce( - (acc, replacement) => acc.replaceAll(replacement, replacements[replacement]), - message.content - ) - : message.content; - const transformedMessage = { - ...message, - content: messageContentWithReplacements, - }; + return [ + ...currentConversation.messages.map((message, index) => { + const isLastComment = index === currentConversation.messages.length - 1; + const isUser = message.role === 'user'; + const replacements = currentConversation.replacements; - return { - actions: , - children: - index !== currentConversation.messages.length - 1 ? ( - - - {showAnonymizedValues ? message.content : transformedMessage.content} - - + const messageProps = { + timelineAvatar: isUser ? ( + ) : ( - - - {showAnonymizedValues ? message.content : transformedMessage.content} - - - + + ), + timestamp: i18n.AT( + message.timestamp.length === 0 ? new Date().toLocaleString() : message.timestamp + ), + username: isUser ? i18n.YOU : i18n.ASSISTANT, + eventColor: message.isError ? 'danger' : undefined, + }; + + const transformMessage = (content: string) => + transformMessageWithReplacements({ + message, + content, + showAnonymizedValues, + replacements, + }); + + // message still needs to stream, no actions returned and replacements handled by streamer + if (!(message.content && message.content.length)) { + return { + ...messageProps, + children: ( + + ), + }; + } + + // transform message here so we can send correct message to CommentActions + const transformedMessage = transformMessage(message.content ?? ''); + + return { + ...messageProps, + actions: , + children: ( + ), - timelineAvatar: isUser ? ( - - ) : ( - - ), - timestamp: i18n.AT( - message.timestamp.length === 0 ? new Date().toLocaleString() : message.timestamp - ), - username: isUser ? i18n.YOU : i18n.ASSISTANT, - eventColor: message.isError ? 'danger' : undefined, - }; - }); + }; + }), + ...extraLoadingComment, + ]; }; diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/buttons/regenerate_response_button.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/buttons/regenerate_response_button.tsx new file mode 100644 index 00000000000000..e9121a9ed9f20f --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/buttons/regenerate_response_button.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiButtonEmptyProps } from '@elastic/eui'; +import { EuiButtonEmpty } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +export function RegenerateResponseButton(props: Partial) { + return ( + + {i18n.translate('xpack.securitySolution.aiAssistant.regenerateResponseButtonLabel', { + defaultMessage: 'Regenerate', + })} + + ); +} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/buttons/stop_generating_button.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/buttons/stop_generating_button.tsx new file mode 100644 index 00000000000000..5144e82b1125f5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/buttons/stop_generating_button.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiButtonEmptyProps } from '@elastic/eui'; +import { EuiButtonEmpty } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +export function StopGeneratingButton(props: Partial) { + return ( + + {i18n.translate('xpack.securitySolution.aiAssistant.stopGeneratingButtonLabel', { + defaultMessage: 'Stop generating', + })} + + ); +} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/failed_to_load_response.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/failed_to_load_response.tsx new file mode 100644 index 00000000000000..5161f5a2b0298e --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/failed_to_load_response.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +export function FailedToLoadResponse() { + return ( + + + + + + + {i18n.translate('xpack.securitySolution.aiAssistant.failedLoadingResponseText', { + defaultMessage: 'Failed to load response', + })} + + + + ); +} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx new file mode 100644 index 00000000000000..7813e45829d1c8 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.test.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { StreamComment } from '.'; +import { useStream } from './use_stream'; +const mockSetComplete = jest.fn(); + +jest.mock('./use_stream'); + +const content = 'Test Content'; +const testProps = { + amendMessage: jest.fn(), + content, + index: 1, + isLastComment: true, + regenerateMessage: jest.fn(), + transformMessage: jest.fn(), +}; + +const mockReader = jest.fn() as unknown as ReadableStreamDefaultReader; + +describe('StreamComment', () => { + beforeEach(() => { + jest.clearAllMocks(); + (useStream as jest.Mock).mockReturnValue({ + error: null, + isLoading: false, + isStreaming: false, + pendingMessage: 'Test Message', + setComplete: mockSetComplete, + }); + }); + it('renders content correctly', () => { + render(); + + expect(screen.getByText(content)).toBeInTheDocument(); + }); + + it('renders cursor when content is loading', () => { + render(); + expect(screen.getByTestId('cursor')).toBeInTheDocument(); + expect(screen.queryByTestId('stopGeneratingButton')).not.toBeInTheDocument(); + }); + + it('renders cursor and stopGeneratingButton when reader is loading', () => { + render(); + expect(screen.getByTestId('stopGeneratingButton')).toBeInTheDocument(); + expect(screen.getByTestId('cursor')).toBeInTheDocument(); + }); + + it('renders controls correctly when not loading', () => { + render(); + + expect(screen.getByTestId('regenerateResponseButton')).toBeInTheDocument(); + }); + + it('calls setComplete when StopGeneratingButton is clicked', () => { + render(); + + fireEvent.click(screen.getByTestId('stopGeneratingButton')); + + expect(mockSetComplete).toHaveBeenCalled(); + }); + + it('displays an error message correctly', () => { + (useStream as jest.Mock).mockReturnValue({ + error: 'Test Error Message', + isLoading: false, + isStreaming: false, + pendingMessage: 'Test Message', + setComplete: mockSetComplete, + }); + render(); + + expect(screen.getByTestId('messsage-error')).toBeInTheDocument(); + }); +}); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx new file mode 100644 index 00000000000000..db394f39bfa326 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useMemo, useRef } from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import type { ContentMessage } from '..'; +import { useStream } from './use_stream'; +import { StopGeneratingButton } from './buttons/stop_generating_button'; +import { RegenerateResponseButton } from './buttons/regenerate_response_button'; +import { MessagePanel } from './message_panel'; +import { MessageText } from './message_text'; + +interface Props { + amendMessage: (message: string) => void; + content?: string; + isFetching?: boolean; + isLastComment: boolean; + index: number; + reader?: ReadableStreamDefaultReader; + regenerateMessage: () => void; + transformMessage: (message: string) => ContentMessage; +} + +export const StreamComment = ({ + amendMessage, + content, + index, + isFetching = false, + isLastComment, + reader, + regenerateMessage, + transformMessage, +}: Props) => { + const { error, isLoading, isStreaming, pendingMessage, setComplete } = useStream({ + amendMessage, + content, + reader, + }); + + const currentState = useRef({ isStreaming, pendingMessage, amendMessage }); + + useEffect(() => { + currentState.current = { isStreaming, pendingMessage, amendMessage }; + }, [amendMessage, isStreaming, pendingMessage]); + + useEffect( + () => () => { + // if the component is unmounted while streaming, amend the message with the pending message + if (currentState.current.isStreaming && currentState.current.pendingMessage.length > 0) { + currentState.current.amendMessage(currentState.current.pendingMessage ?? ''); + } + }, + // store values in currentState to detect true unmount + [] + ); + + const message = useMemo( + // only transform streaming message, transform happens upstream for content message + () => content ?? transformMessage(pendingMessage).content, + [content, transformMessage, pendingMessage] + ); + const isAnythingLoading = useMemo( + () => isFetching || isLoading || isStreaming, + [isFetching, isLoading, isStreaming] + ); + const controls = useMemo(() => { + if (reader == null || !isLastComment) { + return; + } + if (isAnythingLoading) { + return ( + { + setComplete(true); + }} + /> + ); + } + return ( + + + + + + ); + }, [isAnythingLoading, isLastComment, reader, regenerateMessage, setComplete]); + + return ( + } + error={error ? new Error(error) : undefined} + controls={controls} + /> + ); +}; diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_panel.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_panel.tsx new file mode 100644 index 00000000000000..91c7f6ea4247d6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_panel.tsx @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiHorizontalRule, EuiSpacer } from '@elastic/eui'; +import React from 'react'; +import { FailedToLoadResponse } from './failed_to_load_response'; + +interface Props { + error?: Error; + body?: React.ReactNode; + controls?: React.ReactNode; +} + +export function MessagePanel(props: Props) { + return ( + <> + {props.body} + {props.error ? ( + + {props.body ? : null} + + + ) : null} + {props.controls ? ( + <> + + + + {props.controls} + + ) : null} + + ); +} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx new file mode 100644 index 00000000000000..415800a04609d8 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiMarkdownFormat, + EuiSpacer, + EuiText, + getDefaultEuiMarkdownParsingPlugins, + getDefaultEuiMarkdownProcessingPlugins, + transparentize, +} from '@elastic/eui'; +import { css } from '@emotion/css'; +import classNames from 'classnames'; +import type { Code, InlineCode, Parent, Text } from 'mdast'; +import React from 'react'; +import { euiThemeVars } from '@kbn/ui-theme'; + +import type { Node } from 'unist'; +import { customCodeBlockLanguagePlugin } from '../custom_codeblock/custom_codeblock_markdown_plugin'; +import { CustomCodeBlock } from '../custom_codeblock/custom_code_block'; + +interface Props { + content: string; + index: number; + loading: boolean; +} + +const ANIMATION_TIME = 1; + +const cursorCss = css` + @keyframes blink { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0; + } + } + + animation: blink ${ANIMATION_TIME}s infinite; + width: 10px; + height: 16px; + vertical-align: middle; + display: inline-block; + background: ${transparentize(euiThemeVars.euiColorDarkShade, 0.25)}; +`; + +const Cursor = () => ( + +); + +// a weird combination of different whitespace chars to make sure it stays +// invisible even when we cannot properly parse the text while still being +// unique +const CURSOR = ` ᠎  `; + +const loadingCursorPlugin = () => { + const visitor = (node: Node, parent?: Parent) => { + if ('children' in node) { + const nodeAsParent = node as Parent; + nodeAsParent.children.forEach((child) => { + visitor(child, nodeAsParent); + }); + } + + if (node.type !== 'text' && node.type !== 'inlineCode' && node.type !== 'code') { + return; + } + + const textNode = node as Text | InlineCode | Code; + + const indexOfCursor = textNode.value.indexOf(CURSOR); + if (indexOfCursor === -1) { + return; + } + + textNode.value = textNode.value.replace(CURSOR, ''); + + const indexOfNode = parent?.children.indexOf(textNode) ?? 0; + parent?.children.splice(indexOfNode + 1, 0, { + type: 'cursor' as Text['type'], + value: CURSOR, + }); + }; + + return (tree: Node) => { + visitor(tree); + }; +}; + +const getPluginDependencies = () => { + const parsingPlugins = getDefaultEuiMarkdownParsingPlugins(); + + const processingPlugins = getDefaultEuiMarkdownProcessingPlugins(); + + const { components } = processingPlugins[1][1]; + + processingPlugins[1][1].components = { + ...components, + cursor: Cursor, + customCodeBlock: (props) => { + return ( + <> + + + + ); + }, + table: (props) => ( + <> +
+ {' '} + + + + + ), + th: (props) => { + const { children, ...rest } = props; + return ( + + ); + }, + tr: (props) => , + td: (props) => { + const { children, ...rest } = props; + return ( + + ); + }, + }; + + return { + parsingPluginList: [loadingCursorPlugin, customCodeBlockLanguagePlugin, ...parsingPlugins], + processingPluginList: processingPlugins, + }; +}; + +export function MessageText({ loading, content, index }: Props) { + const containerClassName = css` + overflow-wrap: break-word; + `; + + const { parsingPluginList, processingPluginList } = getPluginDependencies(); + + return ( + + + {`${content}${loading ? CURSOR : ''}`} + + + ); +} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/stream_observable.test.ts b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/stream_observable.test.ts new file mode 100644 index 00000000000000..9a63621021cc3e --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/stream_observable.test.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getStreamObservable } from './stream_observable'; +// import { getReaderValue, mockUint8Arrays } from './mock'; +import type { PromptObservableState } from './types'; +import { Subject } from 'rxjs'; +describe('getStreamObservable', () => { + const mockReader = { + read: jest.fn(), + cancel: jest.fn(), + }; + + const typedReader = mockReader as unknown as ReadableStreamDefaultReader; + + const setLoading = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should emit loading state and chunks', (done) => { + const completeSubject = new Subject(); + const expectedStates: PromptObservableState[] = [ + { chunks: [], loading: true }, + { + chunks: [ + { + id: '1', + object: 'chunk', + created: 1635633600000, + model: 'model-1', + choices: [ + { + index: 0, + delta: { role: 'role-1', content: 'content-1' }, + finish_reason: null, + }, + ], + }, + ], + message: 'content-1', + loading: true, + }, + { + chunks: [ + { + id: '1', + object: 'chunk', + created: 1635633600000, + model: 'model-1', + choices: [ + { + index: 0, + delta: { role: 'role-1', content: 'content-1' }, + finish_reason: null, + }, + ], + }, + ], + message: 'content-1', + loading: false, + }, + ]; + + mockReader.read + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array( + new TextEncoder().encode(`data: ${JSON.stringify(expectedStates[1].chunks[0])}`) + ), + }) + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array(new TextEncoder().encode(``)), + }) + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array(new TextEncoder().encode('data: [DONE]\n')), + }) + .mockResolvedValue({ + done: true, + }); + + const source = getStreamObservable(typedReader, setLoading); + const emittedStates: PromptObservableState[] = []; + + source.subscribe({ + next: (state) => emittedStates.push(state), + complete: () => { + expect(emittedStates).toEqual(expectedStates); + done(); + + completeSubject.subscribe({ + next: () => { + expect(setLoading).toHaveBeenCalledWith(false); + expect(typedReader.cancel).toHaveBeenCalled(); + done(); + }, + }); + }, + error: (err) => done(err), + }); + }); + + it('should handle errors', (done) => { + const completeSubject = new Subject(); + const error = new Error('Test Error'); + // Simulate an error + mockReader.read.mockRejectedValue(error); + const source = getStreamObservable(typedReader, setLoading); + + source.subscribe({ + next: (state) => {}, + complete: () => done(new Error('Should not complete')), + error: (err) => { + expect(err).toEqual(error); + done(); + completeSubject.subscribe({ + next: () => { + expect(setLoading).toHaveBeenCalledWith(false); + expect(typedReader.cancel).toHaveBeenCalled(); + done(); + }, + }); + }, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/stream_observable.ts b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/stream_observable.ts new file mode 100644 index 00000000000000..83f9b4cf8ead35 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/stream_observable.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { concatMap, delay, finalize, Observable, of, scan, shareReplay, timestamp } from 'rxjs'; +import type { Dispatch, SetStateAction } from 'react'; +import type { PromptObservableState, Chunk } from './types'; + +const MIN_DELAY = 35; +/** + * Returns an Observable that reads data from a ReadableStream and emits values representing the state of the data processing. + * + * @param reader - The ReadableStreamDefaultReader used to read data from the stream. + * @param setLoading - A function to update the loading state. + * @returns {Observable} An Observable that emits PromptObservableState + */ +export const getStreamObservable = ( + reader: ReadableStreamDefaultReader, + setLoading: Dispatch> +): Observable => + new Observable((observer) => { + observer.next({ chunks: [], loading: true }); + const decoder = new TextDecoder(); + const chunks: Chunk[] = []; + function read() { + reader + .read() + .then(({ done, value }: { done: boolean; value?: Uint8Array }) => { + try { + if (done) { + observer.next({ + chunks, + message: getMessageFromChunks(chunks), + loading: false, + }); + observer.complete(); + return; + } + + const nextChunks: Chunk[] = decoder + .decode(value) + .split('\n') + // every line starts with "data: ", we remove it and are left with stringified JSON or the string "[DONE]" + .map((str) => str.substring(6)) + // filter out empty lines and the "[DONE]" string + .filter((str) => !!str && str !== '[DONE]') + .map((line) => JSON.parse(line)); + + nextChunks.forEach((chunk) => { + chunks.push(chunk); + observer.next({ + chunks, + message: getMessageFromChunks(chunks), + loading: true, + }); + }); + } catch (err) { + observer.error(err); + return; + } + read(); + }) + .catch((err) => { + observer.error(err); + }); + } + read(); + return () => { + reader.cancel(); + }; + }).pipe( + // make sure the request is only triggered once, + // even with multiple subscribers + shareReplay(1), + // append a timestamp of when each value was emitted + timestamp(), + // use the previous timestamp to calculate a target + // timestamp for emitting the next value + scan((acc, value) => { + const lastTimestamp = acc.timestamp || 0; + const emitAt = Math.max(lastTimestamp + MIN_DELAY, value.timestamp); + return { + timestamp: emitAt, + value: value.value, + }; + }), + // add the delay based on the elapsed time + // using concatMap(of(value).pipe(delay(50)) + // leads to browser issues because timers + // are throttled when the tab is not active + concatMap((value) => { + const now = Date.now(); + const delayFor = value.timestamp - now; + + if (delayFor <= 0) { + return of(value.value); + } + + return of(value.value).pipe(delay(delayFor)); + }), + // set loading to false when the observable completes or errors out + finalize(() => setLoading(false)) + ); + +function getMessageFromChunks(chunks: Chunk[]) { + return chunks.map((chunk) => chunk.choices[0]?.delta.content ?? '').join(''); +} + +export const getPlaceholderObservable = () => new Observable(); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/types.ts b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/types.ts new file mode 100644 index 00000000000000..3cf45852ddb116 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface PromptObservableState { + chunks: Chunk[]; + message?: string; + error?: string; + loading: boolean; +} +export interface ChunkChoice { + index: 0; + delta: { role: string; content: string }; + finish_reason: null | string; +} +export interface Chunk { + id: string; + object: string; + created: number; + model: string; + choices: ChunkChoice[]; +} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.test.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.test.tsx new file mode 100644 index 00000000000000..4fbecfac870e18 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.test.tsx @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { useStream } from './use_stream'; + +const amendMessage = jest.fn(); +const reader = jest.fn(); +const cancel = jest.fn(); +const exampleChunk = { + id: '1', + object: 'chunk', + created: 1635633600000, + model: 'model-1', + choices: [ + { + index: 0, + delta: { role: 'role-1', content: 'content-1' }, + finish_reason: null, + }, + ], +}; +const readerComplete = { + read: reader + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array(new TextEncoder().encode(`data: ${JSON.stringify(exampleChunk)}`)), + }) + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array(new TextEncoder().encode(``)), + }) + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array(new TextEncoder().encode('data: [DONE]\n')), + }) + .mockResolvedValue({ + done: true, + }), + cancel, + releaseLock: jest.fn(), + closed: jest.fn().mockResolvedValue(true), +} as unknown as ReadableStreamDefaultReader; + +const defaultProps = { amendMessage, reader: readerComplete }; +describe('useStream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('Should stream response. isLoading/isStreaming are true while streaming, isLoading/isStreaming are false when streaming completes', async () => { + const { result, waitFor } = renderHook(() => useStream(defaultProps)); + expect(reader).toHaveBeenCalledTimes(1); + await waitFor(() => { + expect(result.current).toEqual({ + error: undefined, + isLoading: true, + isStreaming: false, + pendingMessage: '', + setComplete: expect.any(Function), + }); + }); + await waitFor(() => { + expect(result.current).toEqual({ + error: undefined, + isLoading: true, + isStreaming: true, + pendingMessage: 'content-1', + setComplete: expect.any(Function), + }); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + error: undefined, + isLoading: false, + isStreaming: false, + pendingMessage: 'content-1', + setComplete: expect.any(Function), + }); + }); + + expect(reader).toHaveBeenCalledTimes(4); + }); + + it('should not call observable when content is provided', () => { + renderHook(() => + useStream({ + ...defaultProps, + content: 'test content', + }) + ); + expect(reader).not.toHaveBeenCalled(); + }); + + it('should handle a stream error and update UseStream object accordingly', async () => { + const errorMessage = 'Test error message'; + const errorReader = { + read: jest + .fn() + .mockResolvedValueOnce({ + done: false, + value: new Uint8Array(new TextEncoder().encode(`data: ${JSON.stringify(exampleChunk)}`)), + }) + .mockRejectedValue(new Error(errorMessage)), + cancel, + releaseLock: jest.fn(), + closed: jest.fn().mockResolvedValue(true), + } as unknown as ReadableStreamDefaultReader; + const { result, waitForNextUpdate } = renderHook(() => + useStream({ + amendMessage, + reader: errorReader, + }) + ); + expect(result.current.error).toBeUndefined(); + + await waitForNextUpdate(); + + expect(result.current.error).toBe(errorMessage); + expect(result.current.isLoading).toBe(false); + expect(result.current.pendingMessage).toBe(''); + expect(cancel).toHaveBeenCalled(); + }); + + it('should handle an empty content and reader object and return an empty observable', () => { + const { result } = renderHook(() => + useStream({ + ...defaultProps, + content: '', + reader: undefined, + }) + ); + + expect(result.current).toEqual({ + error: undefined, + isLoading: false, + isStreaming: false, + pendingMessage: '', + setComplete: expect.any(Function), + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx new file mode 100644 index 00000000000000..148338f2afafa1 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/use_stream.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useMemo, useState } from 'react'; +import type { Subscription } from 'rxjs'; +import { share } from 'rxjs'; +import { getPlaceholderObservable, getStreamObservable } from './stream_observable'; + +interface UseStreamProps { + amendMessage: (message: string) => void; + content?: string; + reader?: ReadableStreamDefaultReader; +} +interface UseStream { + // The error message, if an error occurs during streaming. + error: string | undefined; + // Indicates whether the streaming is in progress or not + isLoading: boolean; + // Indicates whether the streaming is in progress and there is a pending message. + isStreaming: boolean; + // The pending message from the streaming data. + pendingMessage: string; + // A function to mark the streaming as complete + setComplete: (complete: boolean) => void; +} +/** + * A hook that takes a ReadableStreamDefaultReader and returns an object with properties and functions + * that can be used to handle streaming data from a readable stream + * @param amendMessage - handles the amended message + * @param content - the content of the message. If provided, the function will not use the reader to stream data. + * @param reader - The readable stream reader used to stream data. If provided, the function will use this reader to stream data. + */ +export const useStream = ({ amendMessage, content, reader }: UseStreamProps): UseStream => { + const [pendingMessage, setPendingMessage] = useState(); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(); + const [subscription, setSubscription] = useState(); + const observer$ = useMemo( + () => + content == null && reader != null + ? getStreamObservable(reader, setLoading) + : getPlaceholderObservable(), + [content, reader] + ); + const onCompleteStream = useCallback(() => { + subscription?.unsubscribe(); + setLoading(false); + amendMessage(pendingMessage ?? ''); + }, [amendMessage, pendingMessage, subscription]); + const [complete, setComplete] = useState(false); + useEffect(() => { + if (complete) { + setComplete(false); + onCompleteStream(); + } + }, [complete, onCompleteStream]); + useEffect(() => { + const newSubscription = observer$.pipe(share()).subscribe({ + next: ({ message, loading: isLoading }) => { + setLoading(isLoading); + setPendingMessage(message); + }, + complete: () => { + setComplete(true); + }, + error: (err) => { + setError(err.message); + }, + }); + setSubscription(newSubscription); + }, [observer$]); + return { + error, + isLoading: loading, + isStreaming: loading && pendingMessage != null, + pendingMessage: pendingMessage ?? '', + setComplete, + }; +}; diff --git a/x-pack/plugins/security_solution/public/assistant/helpers.tsx b/x-pack/plugins/security_solution/public/assistant/helpers.tsx index 04bba339edd5da..beca67f55eeb4a 100644 --- a/x-pack/plugins/security_solution/public/assistant/helpers.tsx +++ b/x-pack/plugins/security_solution/public/assistant/helpers.tsx @@ -90,7 +90,7 @@ export const augmentMessageCodeBlocks = ( const cbd = currentConversation.messages.map(({ content }) => analyzeMarkdown( getMessageContentWithReplacements({ - messageContent: content, + messageContent: content ?? '', replacements: currentConversation.replacements, }) ) diff --git a/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.test.tsx b/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.test.tsx index 3cbc54cf24f126..c49e1cf133dd4d 100644 --- a/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.test.tsx @@ -19,6 +19,12 @@ import { useUiSetting$ } from '../../../lib/kibana'; jest.mock('./use_set_alert_tags'); jest.mock('../../../lib/kibana'); +jest.mock( + '../../../../detections/containers/detection_engine/alerts/use_alerts_privileges', + () => ({ + useAlertsPrivileges: jest.fn().mockReturnValue({ hasIndexWrite: true }), + }) +); const defaultProps: UseBulkAlertTagsItemsProps = { refetch: () => {}, diff --git a/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.tsx b/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.tsx index df4d5c37aeeaf5..977cb0bf8b315a 100644 --- a/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.tsx +++ b/x-pack/plugins/security_solution/public/common/components/toolbar/bulk_actions/use_bulk_alert_tags_items.tsx @@ -8,6 +8,7 @@ import { EuiFlexGroup, EuiIconTip, EuiFlexItem } from '@elastic/eui'; import type { RenderContentPanelProps } from '@kbn/triggers-actions-ui-plugin/public/types'; import React, { useCallback, useMemo } from 'react'; +import { useAlertsPrivileges } from '../../../../detections/containers/detection_engine/alerts/use_alerts_privileges'; import { BulkAlertTagsPanel } from './alert_bulk_tags'; import * as i18n from './translations'; import { useSetAlertTags } from './use_set_alert_tags'; @@ -24,6 +25,7 @@ export interface UseBulkAlertTagsPanel { } export const useBulkAlertTagsItems = ({ refetch }: UseBulkAlertTagsItemsProps) => { + const { hasIndexWrite } = useAlertsPrivileges(); const setAlertTags = useSetAlertTags(); const handleOnAlertTagsSubmit = useCallback( async (tags, ids, onSuccess, setIsLoading) => { @@ -34,16 +36,22 @@ export const useBulkAlertTagsItems = ({ refetch }: UseBulkAlertTagsItemsProps) = [setAlertTags] ); - const alertTagsItems = [ - { - key: 'manage-alert-tags', - 'data-test-subj': 'alert-tags-context-menu-item', - name: i18n.ALERT_TAGS_CONTEXT_MENU_ITEM_TITLE, - panel: 1, - label: i18n.ALERT_TAGS_CONTEXT_MENU_ITEM_TITLE, - disableOnQuery: true, - }, - ]; + const alertTagsItems = useMemo( + () => + hasIndexWrite + ? [ + { + key: 'manage-alert-tags', + 'data-test-subj': 'alert-tags-context-menu-item', + name: i18n.ALERT_TAGS_CONTEXT_MENU_ITEM_TITLE, + panel: 1, + label: i18n.ALERT_TAGS_CONTEXT_MENU_ITEM_TITLE, + disableOnQuery: true, + }, + ] + : [], + [hasIndexWrite] + ); const TitleContent = useMemo( () => ( @@ -79,15 +87,18 @@ export const useBulkAlertTagsItems = ({ refetch }: UseBulkAlertTagsItemsProps) = ); const alertTagsPanels: UseBulkAlertTagsPanel[] = useMemo( - () => [ - { - id: 1, - title: TitleContent, - 'data-test-subj': 'alert-tags-context-menu-panel', - renderContent, - }, - ], - [TitleContent, renderContent] + () => + hasIndexWrite + ? [ + { + id: 1, + title: TitleContent, + 'data-test-subj': 'alert-tags-context-menu-panel', + renderContent, + }, + ] + : [], + [TitleContent, hasIndexWrite, renderContent] ); return { diff --git a/x-pack/plugins/security_solution/public/detections/hooks/trigger_actions_alert_table/use_alert_actions.tsx b/x-pack/plugins/security_solution/public/detections/hooks/trigger_actions_alert_table/use_alert_actions.tsx index 5ac7628f5376fb..decc2cb159b5cb 100644 --- a/x-pack/plugins/security_solution/public/detections/hooks/trigger_actions_alert_table/use_alert_actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/hooks/trigger_actions_alert_table/use_alert_actions.tsx @@ -19,6 +19,7 @@ import type { AlertWorkflowStatus } from '../../../common/types'; import { FILTER_CLOSED, FILTER_OPEN, FILTER_ACKNOWLEDGED } from '../../../../common/types'; import * as i18n from '../translations'; import { buildTimeRangeFilter } from '../../components/alerts_table/helpers'; +import { useAlertsPrivileges } from '../../containers/detection_engine/alerts/use_alerts_privileges'; interface UseBulkAlertActionItemsArgs { /* Table ID for which this hook is being used */ @@ -41,6 +42,7 @@ export const useBulkAlertActionItems = ({ to, refetch: refetchProp, }: UseBulkAlertActionItemsArgs) => { + const { hasIndexWrite } = useAlertsPrivileges(); const { startTransaction } = useStartTransaction(); const { addSuccess, addError, addWarning } = useAppToasts(); @@ -172,7 +174,9 @@ export const useBulkAlertActionItems = ({ [getOnAction] ); - return [FILTER_OPEN, FILTER_CLOSED, FILTER_ACKNOWLEDGED].map((status) => - getUpdateAlertStatusAction(status as AlertWorkflowStatus) - ); + return hasIndexWrite + ? [FILTER_OPEN, FILTER_CLOSED, FILTER_ACKNOWLEDGED].map((status) => + getUpdateAlertStatusAction(status as AlertWorkflowStatus) + ) + : []; }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx index 04580b1546b4cd..b62e36e20b9384 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx @@ -108,6 +108,7 @@ const AssistantTab: React.FC<{ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index ec1db8812966ed..9620998722600e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -507,7 +507,9 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = } else { await ruleExecutionLogger.logStatusChange({ newStatus: RuleExecutionStatusEnum.failed, - message: `Bulk Indexing of alerts failed: ${truncateList(result.errors).join()}`, + message: `An error occurred during rule execution: message: "${truncateList( + result.errors + ).join()}"`, metrics: { searchDurations: result.searchAfterTimes, indexingDurations: result.bulkCreateTimes, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md index 70c00d6203c8bb..863a2370f74239 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md @@ -12,7 +12,7 @@ Each page is evaluated in 3 phases. Phase 1: Collect "recent" terms - terms that have appeared in the last rule interval, without regard to whether or not they have appeared in historical data. This is done using a composite aggregation to ensure we can iterate over every term. Phase 2: Check if the page of terms contains any new terms. This uses a regular terms agg with the include parameter - every term is added to the array of include values, so the terms agg is limited to only aggregating on the terms of interest from phase 1. This avoids issues with the terms agg providing approximate results due to getting different terms from different shards. -For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword value. Fields values encoded in base64 and joined with configured a delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /, =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1. +For multiple new terms fields(['source.host', 'source.ip']), in composite aggregation uses pagination through phase 1 aggregation results. It is done, by splitting page results(10,000 buckets) into chunks(500 size of a chunk). Each chunk then gets converted into a DSL query as a filter and applied in a single request. Phase 3: Any new terms from phase 2 are processed and the first document to contain that term is retrieved. The document becomes the basis of the generated alert. This is done with an aggregation query that is very similar to the agg used in phase 2, except it also includes a top_hits agg. top_hits is moved to a separate, later phase for efficiency - top_hits is slow and most terms will not be new in phase 2. This means we only execute the top_hits agg on the terms that are actually new which is faster. @@ -27,7 +27,3 @@ The new terms rule type reuses the singleSearchAfter function which implements t ## Limitations and future enhancements - Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions. -- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields. - Following edge cases possible: - - false negatives (alert is not generated) if too many fields were emitted and actual new values are not getting evaluated if it happened in document in rule run window. - - false positives (wrong alert generated) if too many fields were emitted in historical document and some old terms are not getting evaluated against values in new documents. diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts index 93ba7064c6ce33..d628d8dbf2d903 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts @@ -8,6 +8,8 @@ import type { Moment } from 'moment'; import type { ESSearchResponse } from '@kbn/es-types'; import type { SignalSource } from '../types'; +import type { GenericBulkCreateResponse } from '../factories/bulk_create_factory'; +import type { NewTermsFieldsLatest } from '../../../../../common/api/detection_engine/model/alerts'; export type RecentTermsAggResult = ESSearchResponse< SignalSource, @@ -19,23 +21,41 @@ export type NewTermsAggResult = ESSearchResponse< { body: { aggregations: ReturnType } } >; +export type CompositeDocFetchAggResult = ESSearchResponse< + SignalSource, + { body: { aggregations: ReturnType } } +>; + +export type CompositeNewTermsAggResult = ESSearchResponse< + SignalSource, + { body: { aggregations: ReturnType } } +>; + export type DocFetchAggResult = ESSearchResponse< SignalSource, { body: { aggregations: ReturnType } } >; +export type CreateAlertsHook = ( + aggResult: CompositeDocFetchAggResult | DocFetchAggResult +) => Promise>; + const PAGE_SIZE = 10000; /** * Creates an aggregation that pages through all terms. Used to find the terms that have appeared recently, * without regard to whether or not they're actually new. + * @param param.pageSize - defines size of composite aggregation results. default value is 10,000, arguments values used ofr cases when composite aggregations calls split in batches + * refer to multiTermsCompositeNonRetryable method to more details */ export const buildRecentTermsAgg = ({ fields, after, + pageSize, }: { fields: string[]; after: Record | undefined; + pageSize?: number; }) => { const sources = fields.map((field) => ({ [field]: { @@ -49,7 +69,7 @@ export const buildRecentTermsAgg = ({ new_terms: { composite: { sources, - size: PAGE_SIZE, + size: pageSize ?? PAGE_SIZE, after, }, }, @@ -138,3 +158,99 @@ export const buildDocFetchAgg = ({ }, }; }; + +/** + * Creates an aggregation that returns a bucket for each term + */ +export const buildCompositeNewTermsAgg = ({ + newValueWindowStart, + timestampField, + fields, + after, + pageSize, +}: { + newValueWindowStart: Moment; + timestampField: string; + fields: string[]; + after: Record | undefined; + pageSize?: number; +}) => { + return { + new_terms: { + ...buildRecentTermsAgg({ fields, after, pageSize }).new_terms, + aggs: { + first_seen: { + min: { + field: timestampField, + }, + }, + filtering_agg: { + bucket_selector: { + buckets_path: { + first_seen_value: 'first_seen', + }, + script: { + params: { + start_time: newValueWindowStart.valueOf(), + }, + source: 'params.first_seen_value > params.start_time', + }, + }, + }, + }, + }, + }; +}; + +/** + * Creates an aggregation that fetches the oldest document for each term. + */ +export const buildCompositeDocFetchAgg = ({ + fields, + timestampField, + after, + newValueWindowStart, + pageSize, +}: { + newValueWindowStart: Moment; + fields: string[]; + timestampField: string; + after: Record | undefined; + pageSize?: number; +}) => { + return { + new_terms: { + ...buildRecentTermsAgg({ fields, after, pageSize }).new_terms, + aggs: { + first_seen: { + min: { + field: timestampField, + }, + }, + filtering_agg: { + bucket_selector: { + buckets_path: { + first_seen_value: 'first_seen', + }, + script: { + params: { + start_time: newValueWindowStart.valueOf(), + }, + source: 'params.first_seen_value > params.start_time', + }, + }, + }, + docs: { + top_hits: { + size: 1, + sort: [ + { + [timestampField]: 'asc' as const, + }, + ], + }, + }, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts index 88212b42fe7137..57a1ecfe38109e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { isObject } from 'lodash'; + import { NEW_TERMS_RULE_TYPE_ID } from '@kbn/securitysolution-rules'; import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common'; import { SERVER_APP_ID } from '../../../../../common/constants'; @@ -16,31 +18,26 @@ import { getFilter } from '../utils/get_filter'; import { wrapNewTermsAlerts } from './wrap_new_terms_alerts'; import type { EventsAndTerms } from './wrap_new_terms_alerts'; import type { - DocFetchAggResult, RecentTermsAggResult, + DocFetchAggResult, NewTermsAggResult, + CreateAlertsHook, } from './build_new_terms_aggregation'; import { - buildDocFetchAgg, buildRecentTermsAgg, buildNewTermsAgg, + buildDocFetchAgg, } from './build_new_terms_aggregation'; import { validateIndexPatterns } from '../utils'; -import { - parseDateString, - validateHistoryWindowStart, - transformBucketsToValues, - getNewTermsRuntimeMappings, - getAggregationField, - decodeMatchedValues, -} from './utils'; +import { parseDateString, validateHistoryWindowStart, transformBucketsToValues } from './utils'; import { addToSearchAfterReturn, createSearchAfterReturnType, - getMaxSignalsWarning, getUnprocessedExceptionsWarnings, + getMaxSignalsWarning, } from '../utils/utils'; import { createEnrichEventsFunction } from '../utils/enrichments'; +import { multiTermsComposite } from './multi_terms_composite'; export const createNewTermsAlertType = ( createOptions: CreateRuleOptions @@ -118,7 +115,7 @@ export const createNewTermsAlertType = ( from: params.from, }); - const esFilter = await getFilter({ + const filterArgs = { filters: params.filters, index: inputIndex, language: params.language, @@ -128,7 +125,8 @@ export const createNewTermsAlertType = ( query: params.query, exceptionFilter, fields: inputIndexFields, - }); + }; + const esFilter = await getFilter(filterArgs); const parsedHistoryWindowSize = parseDateString({ date: params.historyWindowStart, @@ -182,88 +180,87 @@ export const createNewTermsAlertType = ( result.searchAfterTimes.push(searchDuration); result.errors.push(...searchErrors); - afterKey = searchResultWithAggs.aggregations.new_terms.after_key; - // If the aggregation returns no after_key it signals that we've paged through all results // and the current page is empty so we can immediately break. - if (afterKey == null) { + if (searchResultWithAggs.aggregations.new_terms.after_key == null) { break; } const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets; - const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); - const newTermsRuntimeMappings = getNewTermsRuntimeMappings( - params.newTermsFields, - bucketsForField - ); - // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. - // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the - // response correspond to each new term. - const { - searchResult: pageSearchResult, - searchDuration: pageSearchDuration, - searchErrors: pageSearchErrors, - } = await singleSearchAfter({ - aggregations: buildNewTermsAgg({ - newValueWindowStart: tuple.from, - timestampField: aggregatableTimestampField, - field: getAggregationField(params.newTermsFields), - include: includeValues, - }), - runtimeMappings: { - ...runtimeMappings, - ...newTermsRuntimeMappings, - }, - searchAfterSortIds: undefined, - index: inputIndex, - // For Phase 2, we expand the time range to aggregate over the history window - // in addition to the rule interval - from: parsedHistoryWindowSize.toISOString(), - to: tuple.to.toISOString(), - services, - ruleExecutionLogger, - filter: esFilter, - pageSize: 0, - primaryTimestamp, - secondaryTimestamp, - }); - result.searchAfterTimes.push(pageSearchDuration); - result.errors.push(...pageSearchErrors); - - logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`); + const createAlertsHook: CreateAlertsHook = async (aggResult) => { + const eventsAndTerms: EventsAndTerms[] = ( + aggResult?.aggregations?.new_terms.buckets ?? [] + ).map((bucket) => { + const newTerms = isObject(bucket.key) ? Object.values(bucket.key) : [bucket.key]; + return { + event: bucket.docs.hits.hits[0], + newTerms, + }; + }); - const pageSearchResultWithAggs = pageSearchResult as NewTermsAggResult; - if (!pageSearchResultWithAggs.aggregations) { - throw new Error('Aggregations were missing on new terms search result'); - } + const wrappedAlerts = wrapNewTermsAlerts({ + eventsAndTerms, + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: inputIndex, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + }); - // PHASE 3: For each term that is not in the history window, fetch the oldest document in - // the rule interval for that term. This is the first document to contain the new term, and will - // become the basis of the resulting alert. - // One document could become multiple alerts if the document contains an array with multiple new terms. - if (pageSearchResultWithAggs.aggregations.new_terms.buckets.length > 0) { - const actualNewTerms = pageSearchResultWithAggs.aggregations.new_terms.buckets.map( - (bucket) => bucket.key + const bulkCreateResult = await bulkCreate( + wrappedAlerts, + params.maxSignals - result.createdSignalsCount, + createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }) ); + addToSearchAfterReturn({ current: result, next: bulkCreateResult }); + + return bulkCreateResult; + }; + + // separate route for multiple new terms + // it uses paging through composite aggregation + if (params.newTermsFields.length > 1) { + await multiTermsComposite({ + filterArgs, + buckets: bucketsForField, + params, + aggregatableTimestampField, + parsedHistoryWindowSize, + services, + result, + logger, + runOpts: execOptions.runOpts, + afterKey, + createAlertsHook, + }); + } else { + // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. + // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the + // response correspond to each new term. + const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField); const { - searchResult: docFetchSearchResult, - searchDuration: docFetchSearchDuration, - searchErrors: docFetchSearchErrors, + searchResult: pageSearchResult, + searchDuration: pageSearchDuration, + searchErrors: pageSearchErrors, } = await singleSearchAfter({ - aggregations: buildDocFetchAgg({ + aggregations: buildNewTermsAgg({ + newValueWindowStart: tuple.from, timestampField: aggregatableTimestampField, - field: getAggregationField(params.newTermsFields), - include: actualNewTerms, + field: params.newTermsFields[0], + include: includeValues, }), - runtimeMappings: { - ...runtimeMappings, - ...newTermsRuntimeMappings, - }, + runtimeMappings, searchAfterSortIds: undefined, index: inputIndex, - // For phase 3, we go back to aggregating only over the rule interval - excluding the history window - from: tuple.from.toISOString(), + // For Phase 2, we expand the time range to aggregate over the history window + // in addition to the rule interval + from: parsedHistoryWindowSize.toISOString(), to: tuple.to.toISOString(), services, ruleExecutionLogger, @@ -272,51 +269,67 @@ export const createNewTermsAlertType = ( primaryTimestamp, secondaryTimestamp, }); - result.searchAfterTimes.push(docFetchSearchDuration); - result.errors.push(...docFetchSearchErrors); + result.searchAfterTimes.push(pageSearchDuration); + result.errors.push(...pageSearchErrors); - const docFetchResultWithAggs = docFetchSearchResult as DocFetchAggResult; + logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`); - if (!docFetchResultWithAggs.aggregations) { - throw new Error('Aggregations were missing on document fetch search result'); + const pageSearchResultWithAggs = pageSearchResult as NewTermsAggResult; + if (!pageSearchResultWithAggs.aggregations) { + throw new Error('Aggregations were missing on new terms search result'); } - const eventsAndTerms: EventsAndTerms[] = - docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => { - const newTerms = decodeMatchedValues(params.newTermsFields, bucket.key); - return { - event: bucket.docs.hits.hits[0], - newTerms, - }; + // PHASE 3: For each term that is not in the history window, fetch the oldest document in + // the rule interval for that term. This is the first document to contain the new term, and will + // become the basis of the resulting alert. + // One document could become multiple alerts if the document contains an array with multiple new terms. + if (pageSearchResultWithAggs.aggregations.new_terms.buckets.length > 0) { + const actualNewTerms = pageSearchResultWithAggs.aggregations.new_terms.buckets.map( + (bucket) => bucket.key + ); + + const { + searchResult: docFetchSearchResult, + searchDuration: docFetchSearchDuration, + searchErrors: docFetchSearchErrors, + } = await singleSearchAfter({ + aggregations: buildDocFetchAgg({ + timestampField: aggregatableTimestampField, + field: params.newTermsFields[0], + include: actualNewTerms, + }), + runtimeMappings, + searchAfterSortIds: undefined, + index: inputIndex, + // For phase 3, we go back to aggregating only over the rule interval - excluding the history window + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + ruleExecutionLogger, + filter: esFilter, + pageSize: 0, + primaryTimestamp, + secondaryTimestamp, }); + result.searchAfterTimes.push(docFetchSearchDuration); + result.errors.push(...docFetchSearchErrors); - const wrappedAlerts = wrapNewTermsAlerts({ - eventsAndTerms, - spaceId, - completeRule, - mergeStrategy, - indicesToQuery: inputIndex, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - }); + const docFetchResultWithAggs = docFetchSearchResult as DocFetchAggResult; - const bulkCreateResult = await bulkCreate( - wrappedAlerts, - params.maxSignals - result.createdSignalsCount, - createEnrichEventsFunction({ - services, - logger: ruleExecutionLogger, - }) - ); + if (!docFetchResultWithAggs.aggregations) { + throw new Error('Aggregations were missing on document fetch search result'); + } - addToSearchAfterReturn({ current: result, next: bulkCreateResult }); + const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs); - if (bulkCreateResult.alertsWereTruncated) { - result.warningMessages.push(getMaxSignalsWarning()); - break; + if (bulkCreateResult.alertsWereTruncated) { + result.warningMessages.push(getMaxSignalsWarning()); + break; + } } } + + afterKey = searchResultWithAggs.aggregations.new_terms.after_key; } return { ...result, state }; }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts new file mode 100644 index 00000000000000..dc33626bbd5352 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/multi_terms_composite.ts @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import pRetry from 'p-retry'; +import type { Moment } from 'moment'; +import type { Logger } from '@kbn/logging'; +import type { NewTermsRuleParams } from '../../rule_schema'; +import type { GetFilterArgs } from '../utils/get_filter'; +import { getFilter } from '../utils/get_filter'; +import { singleSearchAfter } from '../utils/single_search_after'; +import { + buildCompositeNewTermsAgg, + buildCompositeDocFetchAgg, +} from './build_new_terms_aggregation'; + +import type { + CompositeDocFetchAggResult, + CompositeNewTermsAggResult, + CreateAlertsHook, +} from './build_new_terms_aggregation'; + +import { getMaxSignalsWarning } from '../utils/utils'; + +import type { RuleServices, SearchAfterAndBulkCreateReturnType, RunOpts } from '../types'; + +/** + * composite aggregation page batch size set to 500 as it shows th best performance(refer https://github.com/elastic/kibana/pull/157413) and + * allows to be scaled down below when max_clause_count error is encountered + */ +const BATCH_SIZE = 500; + +interface MultiTermsCompositeArgsBase { + filterArgs: GetFilterArgs; + buckets: Array<{ + doc_count: number; + key: Record; + }>; + params: NewTermsRuleParams; + aggregatableTimestampField: string; + parsedHistoryWindowSize: Moment; + services: RuleServices; + result: SearchAfterAndBulkCreateReturnType; + logger: Logger; + runOpts: RunOpts; + afterKey: Record | undefined; + createAlertsHook: CreateAlertsHook; +} + +interface MultiTermsCompositeArgs extends MultiTermsCompositeArgsBase { + batchSize: number; +} + +/** + * This helper does phase2/phase3(look README) got multiple new terms + * It takes full page of results from phase 1 (10,000) + * Splits it in chunks (starts from 500) and applies it as a filter in new composite aggregation request + * It pages through though all 10,000 results from phase1 until maxSize alerts found + */ +const multiTermsCompositeNonRetryable = async ({ + filterArgs, + buckets, + params, + aggregatableTimestampField, + parsedHistoryWindowSize, + services, + result, + logger, + runOpts, + afterKey, + createAlertsHook, + batchSize, +}: MultiTermsCompositeArgs) => { + const { + ruleExecutionLogger, + tuple, + inputIndex, + runtimeMappings, + primaryTimestamp, + secondaryTimestamp, + } = runOpts; + + let internalAfterKey = afterKey ?? undefined; + + let i = 0; + + while (i < buckets.length) { + const batch = buckets.slice(i, i + batchSize); + i += batchSize; + const batchFilters = batch.map((b) => { + const must = Object.keys(b.key).map((key) => ({ match: { [key]: b.key[key] } })); + + return { bool: { must } }; + }); + + const esFilterForBatch = await getFilter({ + ...filterArgs, + filters: [ + ...(Array.isArray(filterArgs.filters) ? filterArgs.filters : []), + { bool: { should: batchFilters } }, + ], + }); + + // PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window. + // The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the + // response correspond to each new term. + const { + searchResult: pageSearchResult, + searchDuration: pageSearchDuration, + searchErrors: pageSearchErrors, + } = await singleSearchAfter({ + aggregations: buildCompositeNewTermsAgg({ + newValueWindowStart: tuple.from, + timestampField: aggregatableTimestampField, + fields: params.newTermsFields, + after: internalAfterKey, + pageSize: batchSize, + }), + runtimeMappings, + searchAfterSortIds: undefined, + index: inputIndex, + // For Phase 2, we expand the time range to aggregate over the history window + // in addition to the rule interval + from: parsedHistoryWindowSize.toISOString(), + to: tuple.to.toISOString(), + services, + ruleExecutionLogger, + filter: esFilterForBatch, + pageSize: 0, + primaryTimestamp, + secondaryTimestamp, + }); + + result.searchAfterTimes.push(pageSearchDuration); + result.errors.push(...pageSearchErrors); + logger.debug(`Time spent on phase 2 terms agg: ${pageSearchDuration}`); + + const pageSearchResultWithAggs = pageSearchResult as CompositeNewTermsAggResult; + if (!pageSearchResultWithAggs.aggregations) { + throw new Error('Aggregations were missing on new terms search result'); + } + + // PHASE 3: For each term that is not in the history window, fetch the oldest document in + // the rule interval for that term. This is the first document to contain the new term, and will + // become the basis of the resulting alert. + // One document could become multiple alerts if the document contains an array with multiple new terms. + if (pageSearchResultWithAggs.aggregations.new_terms.buckets.length > 0) { + const { + searchResult: docFetchSearchResult, + searchDuration: docFetchSearchDuration, + searchErrors: docFetchSearchErrors, + } = await singleSearchAfter({ + aggregations: buildCompositeDocFetchAgg({ + newValueWindowStart: tuple.from, + timestampField: aggregatableTimestampField, + fields: params.newTermsFields, + after: internalAfterKey, + pageSize: batchSize, + }), + runtimeMappings, + searchAfterSortIds: undefined, + index: inputIndex, + from: parsedHistoryWindowSize.toISOString(), + to: tuple.to.toISOString(), + services, + ruleExecutionLogger, + filter: esFilterForBatch, + pageSize: 0, + primaryTimestamp, + secondaryTimestamp, + }); + result.searchAfterTimes.push(docFetchSearchDuration); + result.errors.push(...docFetchSearchErrors); + + const docFetchResultWithAggs = docFetchSearchResult as CompositeDocFetchAggResult; + + if (!docFetchResultWithAggs.aggregations) { + throw new Error('Aggregations were missing on document fetch search result'); + } + + const bulkCreateResult = await createAlertsHook(docFetchResultWithAggs); + + if (bulkCreateResult.alertsWereTruncated) { + result.warningMessages.push(getMaxSignalsWarning()); + return result; + } + } + + internalAfterKey = batch[batch.length - 1]?.key; + } + + return result; +}; + +/** + * If request fails with batch size of BATCH_SIZE + * We will try to reduce it in twice per each request, three times, up until 125 + * Per ES documentation, max_clause_count min value is 1,000 - so with 125 we should be able execute query below max_clause_count value + */ +export const multiTermsComposite = async (args: MultiTermsCompositeArgsBase): Promise => { + let retryBatchSize = BATCH_SIZE; + const ruleExecutionLogger = args.runOpts.ruleExecutionLogger; + await pRetry( + async (retryCount) => { + try { + const res = await multiTermsCompositeNonRetryable({ ...args, batchSize: retryBatchSize }); + return res; + } catch (e) { + // do not retry if error not related to too many clauses + // if user's configured rule somehow has filter itself greater than max_clause_count, we won't get to this place anyway, + // as rule would fail on phase 1 + if ( + ![ + 'query_shard_exception: failed to create query', + 'Query contains too many nested clauses;', + ].some((errMessage) => e.message.includes(errMessage)) + ) { + args.result.errors.push(e.message); + return args.result; + } + + retryBatchSize = retryBatchSize / 2; + ruleExecutionLogger.warn( + `New terms query for multiple fields failed due to too many clauses in query: ${e.message}. Retrying #${retryCount} with ${retryBatchSize} for composite aggregation` + ); + throw e; + } + }, + { + retries: 2, + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts index 9d9993d471e303..8ede65f0adcb51 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts @@ -5,16 +5,7 @@ * 2.0. */ -import { - parseDateString, - validateHistoryWindowStart, - transformBucketsToValues, - getAggregationField, - decodeMatchedValues, - getNewTermsRuntimeMappings, - createFieldValuesMap, - AGG_FIELD_NAME, -} from './utils'; +import { parseDateString, validateHistoryWindowStart, transformBucketsToValues } from './utils'; describe('new terms utils', () => { describe('parseDateString', () => { @@ -119,257 +110,6 @@ describe('new terms utils', () => { ).toEqual(['host-0']); }); - it('should return correct value for multiple new terms fields', () => { - expect( - transformBucketsToValues( - ['source.host', 'source.ip'], - [ - { - key: { - 'source.host': 'host-0', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - { - key: { - 'source.host': 'host-1', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - ] - ) - ).toEqual(['aG9zdC0w_MTI3LjAuMC4x', 'aG9zdC0x_MTI3LjAuMC4x']); - }); - - it('should filter null values for multiple new terms fields', () => { - expect( - transformBucketsToValues( - ['source.host', 'source.ip'], - [ - { - key: { - 'source.host': 'host-0', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - { - key: { - 'source.host': 'host-1', - 'source.ip': null, - }, - doc_count: 1, - }, - ] - ) - ).toEqual(['aG9zdC0w_MTI3LjAuMC4x']); - }); - }); - - describe('getAggregationField', () => { - it('should return correct value for a single new terms field', () => { - expect(getAggregationField(['source.ip'])).toBe('source.ip'); - }); - it('should return correct value for multiple new terms fields', () => { - expect(getAggregationField(['source.host', 'source.ip'])).toBe(AGG_FIELD_NAME); - }); - }); - - describe('decodeMatchedValues', () => { - it('should return correct value for a single new terms field', () => { - expect(decodeMatchedValues(['source.ip'], '127.0.0.1')).toEqual(['127.0.0.1']); - }); - it('should return correct value for multiple new terms fields', () => { - expect(decodeMatchedValues(['source.host', 'source.ip'], 'aG9zdC0w_MTI3LjAuMC4x')).toEqual([ - 'host-0', - '127.0.0.1', - ]); - }); - }); - - describe('getNewTermsRuntimeMappings', () => { - it('should not return runtime field if new terms fields is empty', () => { - expect(getNewTermsRuntimeMappings([], [])).toBeUndefined(); - }); - it('should not return runtime field if new terms fields has only one field', () => { - expect(getNewTermsRuntimeMappings(['host.name'], [])).toBeUndefined(); - }); - - it('should return runtime field if new terms fields has more than one field', () => { - const runtimeMappings = getNewTermsRuntimeMappings( - ['source.host', 'source.ip'], - [ - { - key: { - 'source.host': 'host-0', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - { - key: { - 'source.host': 'host-1', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - ] - ); - - expect(runtimeMappings?.[AGG_FIELD_NAME]).toMatchObject({ - type: 'keyword', - script: { - params: { - fields: ['source.host', 'source.ip'], - values: { - 'source.host': { - 'host-0': true, - 'host-1': true, - }, - 'source.ip': { - '127.0.0.1': true, - }, - }, - }, - source: expect.any(String), - }, - }); - }); - }); -}); - -describe('createFieldValuesMap', () => { - it('should return undefined if new terms fields has only one field', () => { - expect( - createFieldValuesMap( - ['host.name'], - [ - { - key: { - 'source.host': 'host-0', - }, - doc_count: 1, - }, - { - key: { - 'source.host': 'host-1', - }, - doc_count: 3, - }, - ] - ) - ).toBeUndefined(); - }); - - it('should return values map if new terms fields has more than one field', () => { - expect( - createFieldValuesMap( - ['source.host', 'source.ip'], - [ - { - key: { - 'source.host': 'host-0', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - { - key: { - 'source.host': 'host-1', - 'source.ip': '127.0.0.1', - }, - doc_count: 1, - }, - ] - ) - ).toEqual({ - 'source.host': { - 'host-0': true, - 'host-1': true, - }, - 'source.ip': { - '127.0.0.1': true, - }, - }); - }); - - it('should not put value in map if it is null', () => { - expect( - createFieldValuesMap( - ['source.host', 'source.ip'], - [ - { - key: { - 'source.host': 'host-1', - 'source.ip': null, - }, - doc_count: 1, - }, - ] - ) - ).toEqual({ - 'source.host': { - 'host-1': true, - }, - 'source.ip': {}, - }); - }); - - it('should put value in map if it is a number', () => { - expect( - createFieldValuesMap( - ['source.host', 'source.id'], - [ - { - key: { - 'source.host': 'host-1', - 'source.id': 100, - }, - doc_count: 1, - }, - ] - ) - ).toEqual({ - 'source.host': { - 'host-1': true, - }, - 'source.id': { - '100': true, - }, - }); - }); - - it('should put value in map if it is a boolean', () => { - expect( - createFieldValuesMap( - ['source.host', 'user.enabled'], - [ - { - key: { - 'source.host': 'host-1', - 'user.enabled': true, - }, - doc_count: 1, - }, - { - key: { - 'source.host': 'host-1', - 'user.enabled': false, - }, - doc_count: 1, - }, - ] - ) - ).toEqual({ - 'source.host': { - 'host-1': true, - }, - 'user.enabled': { - true: true, - false: true, - }, - }); + // TODO: write test for multiple fields? }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts index de5822d29b1b57..30faf1f64fa963 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts @@ -9,9 +9,6 @@ import dateMath from '@elastic/datemath'; import moment from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -export const AGG_FIELD_NAME = 'new_terms_values'; -const DELIMITER = '_'; - export const parseDateString = ({ date, forceNow, @@ -54,143 +51,18 @@ export const validateHistoryWindowStart = ({ /** * Takes a list of buckets and creates value from them to be used in 'include' clause of terms aggregation. * For a single new terms field, value equals to bucket name - * For multiple new terms fields and buckets, value equals to concatenated base64 encoded bucket names - * @returns for buckets('host-0', 'test'), resulted value equals to: 'aG9zdC0w_dGVzdA==' + * Otherwise throws error + * @returns */ export const transformBucketsToValues = ( newTermsFields: string[], buckets: estypes.AggregationsCompositeBucket[] ): Array => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together + // if new terms include only one field we don't use runtime mappings and don't stitch fields buckets together if (newTermsFields.length === 1) { return buckets .map((bucket) => Object.values(bucket.key)[0]) .filter((value): value is string | number => value != null); } - - return buckets - .map((bucket) => Object.values(bucket.key)) - .filter((values) => !values.some((value) => value == null)) - .map((values) => - values - .map((value) => - Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64') - ) - .join(DELIMITER) - ); -}; - -/** - * transforms arrays of new terms fields and its values in object - * [new_terms_field]: { [value1]: true, [value1]: true } - * It's needed to have constant time complexity of accessing whether value is present in new terms - * It will be passed to Painless script used in runtime field - */ -export const createFieldValuesMap = ( - newTermsFields: string[], - buckets: estypes.AggregationsCompositeBucket[] -) => { - if (newTermsFields.length === 1) { - return undefined; - } - - const valuesMap = newTermsFields.reduce>>( - (acc, field) => ({ ...acc, [field]: {} }), - {} - ); - - buckets - .map((bucket) => bucket.key) - .forEach((bucket) => { - Object.entries(bucket).forEach(([key, value]) => { - if (value == null) { - return; - } - const strValue = typeof value !== 'string' ? value.toString() : value; - valuesMap[key][strValue] = true; - }); - }); - - return valuesMap; -}; - -export const getNewTermsRuntimeMappings = ( - newTermsFields: string[], - buckets: estypes.AggregationsCompositeBucket[] -): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length <= 1) { - return undefined; - } - - const values = createFieldValuesMap(newTermsFields, buckets); - return { - [AGG_FIELD_NAME]: { - type: 'keyword', - script: { - params: { fields: newTermsFields, values }, - source: ` - def stack = new Stack(); - // ES has limit in 100 values for runtime field, after this query will fail - int emitLimit = 100; - stack.add([0, '']); - - while (stack.length > 0) { - if (emitLimit == 0) { - break; - } - def tuple = stack.pop(); - def index = tuple[0]; - def line = tuple[1]; - if (index === params['fields'].length) { - emit(line); - emitLimit = emitLimit - 1; - } else { - def fieldName = params['fields'][index]; - for (field in doc[fieldName]) { - def fieldStr = String.valueOf(field); - if (!params['values'][fieldName].containsKey(fieldStr)) { - continue; - } - def delimiter = index === 0 ? '' : '${DELIMITER}'; - def nextLine = line + delimiter + fieldStr.encodeBase64(); - - stack.add([index + 1, nextLine]) - } - } - } - `, - }, - }, - }; -}; - -/** - * For a single new terms field, aggregation field equals to new terms field - * For multiple new terms fields, aggregation field equals to defined AGG_FIELD_NAME, which is runtime field - */ -export const getAggregationField = (newTermsFields: string[]): string => { - // if new terms include only one field we don't use runtime mappings and don't stich fields buckets together - if (newTermsFields.length === 1) { - return newTermsFields[0]; - } - - return AGG_FIELD_NAME; -}; - -const decodeBucketKey = (bucketKey: string): string[] => { - return bucketKey - .split(DELIMITER) - .map((encodedValue) => Buffer.from(encodedValue, 'base64').toString()); -}; - -/** - * decodes matched values(bucket keys) from terms aggregation and returns fields as array - * @returns 'aG9zdC0w_dGVzdA==' bucket key will result in ['host-0', 'test'] - */ -export const decodeMatchedValues = (newTermsFields: string[], bucketKey: string | number) => { - // if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here - const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string); - - return values; + throw Error('Can be used for single new terms field only'); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts index 1d8b09053b4667..7d7492bd17e2b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts @@ -28,7 +28,7 @@ import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { ESBoolQuery } from '../../../../../common/typed_json'; import { getQueryFilter } from './get_query_filter'; -interface GetFilterArgs { +export interface GetFilterArgs { type: Type; filters: unknown | undefined; language: LanguageOrUndefined; diff --git a/x-pack/plugins/serverless_search/public/application/components/connectors/empty_connectors_prompt.tsx b/x-pack/plugins/serverless_search/public/application/components/connectors/empty_connectors_prompt.tsx index 6496b6c357150e..a997c8086225f3 100644 --- a/x-pack/plugins/serverless_search/public/application/components/connectors/empty_connectors_prompt.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/connectors/empty_connectors_prompt.tsx @@ -50,7 +50,7 @@ export const EmptyConnectorsPrompt: React.FC = () => {

{i18n.translate('xpack.serverlessSearch.connectorsEmpty.description', { defaultMessage: - "To set up and deploy a connector you'll be working between the third-party data source, your terminal, and the Kibana UI. The high level process looks like this:", + "To set up and deploy a connector you'll be working between the third-party data source, your terminal, and the Elasticsearch serverless UI. The high level process looks like this:", })}

diff --git a/x-pack/plugins/serverless_search/public/application/components/connectors_overview.tsx b/x-pack/plugins/serverless_search/public/application/components/connectors_overview.tsx index 1c766020b7b7c9..f79dce303617b0 100644 --- a/x-pack/plugins/serverless_search/public/application/components/connectors_overview.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/connectors_overview.tsx @@ -82,9 +82,9 @@ export const ConnectorsOverview = () => { - + {i18n.translate('xpack.serverlessSearch.connectorsPythonLink', { - defaultMessage: 'connectors-python', + defaultMessage: 'elastic/connectors', })} diff --git a/x-pack/test/cases_api_integration/common/plugins/cases/server/routes.ts b/x-pack/test/cases_api_integration/common/plugins/cases/server/routes.ts index d68b87a2b132d5..30dc4da6bb3243 100644 --- a/x-pack/test/cases_api_integration/common/plugins/cases/server/routes.ts +++ b/x-pack/test/cases_api_integration/common/plugins/cases/server/routes.ts @@ -5,6 +5,7 @@ * 2.0. */ +import Boom from '@hapi/boom'; import { createHash } from 'crypto'; import { schema } from '@kbn/config-schema'; import type { CoreSetup, Logger } from '@kbn/core/server'; @@ -12,7 +13,7 @@ import type { ExternalReferenceAttachmentType, PersistableStateAttachmentTypeSetup, } from '@kbn/cases-plugin/server/attachment_framework/types'; -import { CasesPatchRequest } from '@kbn/cases-plugin/common/types/api'; +import { BulkCreateCasesRequest, CasesPatchRequest } from '@kbn/cases-plugin/common/types/api'; import type { FixtureStartDeps } from './plugin'; const hashParts = (parts: string[]): string => { @@ -106,4 +107,35 @@ export const registerRoutes = (core: CoreSetup, logger: Logger } } ); + + router.post( + { + path: '/api/cases_fixture/cases:bulkCreate', + validate: { + body: schema.object({}, { unknowns: 'allow' }), + }, + }, + async (context, request, response) => { + try { + const [_, { cases }] = await core.getStartServices(); + const client = await cases.getCasesClientWithRequest(request); + + return response.ok({ + body: await client.cases.bulkCreate(request.body as BulkCreateCasesRequest), + }); + } catch (error) { + logger.error(`Error : ${error}`); + + const boom = new Boom.Boom(error.message, { + statusCode: error.wrappedError.output.statusCode, + }); + + return response.customError({ + body: boom, + headers: boom.output.headers as { [key: string]: string }, + statusCode: boom.output.statusCode, + }); + } + } + ); }; diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/bulk_create_cases.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/bulk_create_cases.ts new file mode 100644 index 00000000000000..8177649f22ab3c --- /dev/null +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/bulk_create_cases.ts @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; +import expect from '@kbn/expect'; +import { BulkCreateCasesResponse } from '@kbn/cases-plugin/common/types/api'; +import { CaseSeverity } from '@kbn/cases-plugin/common'; +import { CaseStatuses, CustomFieldTypes } from '@kbn/cases-plugin/common/types/domain'; +import { User } from '../../../../common/lib/authentication/types'; +import { defaultUser, getPostCaseRequest, postCaseResp } from '../../../../common/lib/mock'; +import { + deleteAllCaseItems, + getCaseUserActions, + getSpaceUrlPrefix, + removeServerGeneratedPropertiesFromCase, + removeServerGeneratedPropertiesFromUserAction, +} from '../../../../common/lib/api'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import { + secOnly, + secOnlyRead, + globalRead, + obsOnlyRead, + obsSecRead, + noKibanaPrivileges, + testDisabled, + superUser, +} from '../../../../common/lib/authentication/users'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const supertest = getService('supertest'); + const es = getService('es'); + + /** + * There is no official route that supports + * bulk creating cases. The purpose of this test + * is to test the bulkCreate method of the cases client in + * x-pack/plugins/cases/server/client/cases/bulk_create.ts + * + * The test route is configured here x-pack/test/cases_api_integration/common/plugins/cases/server/routes.ts + */ + describe('bulk_create_cases', () => { + const bulkCreateCases = async ({ + superTestService = supertest, + data, + expectedHttpCode = 200, + auth = { user: superUser, space: null }, + }: { + superTestService?: SuperTest.SuperTest; + data: object; + expectedHttpCode?: number; + auth?: { user: User; space: string | null }; + }): Promise => { + return superTestService + .post(`${getSpaceUrlPrefix(auth?.space)}/api/cases_fixture/cases:bulkCreate`) + .set('kbn-xsrf', 'foo') + .set('x-elastic-internal-origin', 'foo') + .auth(auth.user.username, auth.user.password) + .send(data) + .expect(expectedHttpCode) + .then((response) => response.body); + }; + + afterEach(async () => { + await deleteAllCaseItems(es); + }); + + it('should bulk create cases', async () => { + const createdCases = await bulkCreateCases({ + data: { + cases: [getPostCaseRequest(), getPostCaseRequest({ severity: CaseSeverity.MEDIUM })], + }, + }); + + expect(createdCases.cases.length === 2); + + const firstCase = removeServerGeneratedPropertiesFromCase(createdCases.cases[0]); + const secondCase = removeServerGeneratedPropertiesFromCase(createdCases.cases[1]); + + expect(firstCase).to.eql(postCaseResp(null, getPostCaseRequest())); + expect(secondCase).to.eql( + postCaseResp(null, getPostCaseRequest({ severity: CaseSeverity.MEDIUM })) + ); + }); + + it('should bulk create cases with different owners', async () => { + const createdCases = await bulkCreateCases({ + data: { + cases: [getPostCaseRequest(), getPostCaseRequest({ owner: 'observabilityFixture' })], + }, + }); + + expect(createdCases.cases.length === 2); + + const firstCase = removeServerGeneratedPropertiesFromCase(createdCases.cases[0]); + const secondCase = removeServerGeneratedPropertiesFromCase(createdCases.cases[1]); + + expect(firstCase).to.eql(postCaseResp(null, getPostCaseRequest())); + expect(secondCase).to.eql( + postCaseResp(null, getPostCaseRequest({ owner: 'observabilityFixture' })) + ); + }); + + it('should allow creating a case with custom ID', async () => { + const createdCases = await bulkCreateCases({ + data: { + cases: [{ id: 'test-case', ...getPostCaseRequest() }], + }, + }); + + expect(createdCases.cases.length === 1); + + const firstCase = createdCases.cases[0]; + + expect(firstCase.id).to.eql('test-case'); + }); + + it('should validate custom fields correctly', async () => { + await bulkCreateCases({ + data: { + cases: [ + getPostCaseRequest({ + customFields: [ + { + key: 'duplicated_key', + type: CustomFieldTypes.TEXT, + value: 'this is a text field value', + }, + { + key: 'duplicated_key', + type: CustomFieldTypes.TEXT, + value: 'this is a text field value', + }, + ], + }), + ], + }, + expectedHttpCode: 400, + }); + }); + + it('should throw an error correctly', async () => { + await bulkCreateCases({ + data: { + cases: [ + // two cases with the same ID will result to a conflict error + { id: 'test-case', ...getPostCaseRequest() }, + { id: 'test-case', ...getPostCaseRequest() }, + ], + }, + expectedHttpCode: 409, + }); + }); + + it('should create user actions correctly', async () => { + const createdCases = await bulkCreateCases({ + data: { + cases: [getPostCaseRequest(), getPostCaseRequest({ severity: CaseSeverity.MEDIUM })], + }, + }); + + const firstCase = createdCases.cases[0]; + const secondCase = createdCases.cases[1]; + + const firstCaseUserActions = await getCaseUserActions({ + supertest, + caseID: firstCase.id, + }); + + const secondCaseUserActions = await getCaseUserActions({ + supertest, + caseID: secondCase.id, + }); + + expect(firstCaseUserActions.length).to.eql(1); + expect(secondCaseUserActions.length).to.eql(1); + + const firstCaseCreationUserAction = removeServerGeneratedPropertiesFromUserAction( + firstCaseUserActions[0] + ); + + const secondCaseCreationUserAction = removeServerGeneratedPropertiesFromUserAction( + secondCaseUserActions[0] + ); + + expect(firstCaseCreationUserAction).to.eql({ + action: 'create', + type: 'create_case', + created_by: defaultUser, + case_id: firstCase.id, + comment_id: null, + owner: 'securitySolutionFixture', + payload: { + description: firstCase.description, + title: firstCase.title, + tags: firstCase.tags, + connector: firstCase.connector, + settings: firstCase.settings, + owner: firstCase.owner, + status: CaseStatuses.open, + severity: CaseSeverity.LOW, + assignees: [], + category: null, + customFields: [], + }, + }); + + expect(secondCaseCreationUserAction).to.eql({ + action: 'create', + type: 'create_case', + created_by: defaultUser, + case_id: secondCase.id, + comment_id: null, + owner: 'securitySolutionFixture', + payload: { + description: secondCase.description, + title: secondCase.title, + tags: secondCase.tags, + connector: secondCase.connector, + settings: secondCase.settings, + owner: secondCase.owner, + status: CaseStatuses.open, + severity: CaseSeverity.MEDIUM, + assignees: [], + category: null, + customFields: [], + }, + }); + }); + + describe('rbac', () => { + it('returns a 403 when attempting to create a case with an owner that was from a disabled feature in the space', async () => { + const theCase = (await bulkCreateCases({ + superTestService: supertestWithoutAuth, + data: { cases: [getPostCaseRequest({ owner: 'testDisabledFixture' })] }, + expectedHttpCode: 403, + auth: { + user: testDisabled, + space: 'space1', + }, + })) as unknown as { message: string }; + + expect(theCase.message).to.eql( + 'Failed to bulk create cases: Error: Unauthorized to create case with owners: "testDisabledFixture"' + ); + }); + + it('User: security solution only - should create a case', async () => { + const cases = await bulkCreateCases({ + superTestService: supertestWithoutAuth, + data: { cases: [getPostCaseRequest({ owner: 'securitySolutionFixture' })] }, + expectedHttpCode: 200, + auth: { + user: secOnly, + space: 'space1', + }, + }); + + expect(cases.cases[0].owner).to.eql('securitySolutionFixture'); + }); + + it('User: security solution only - should NOT create a case of different owner', async () => { + await bulkCreateCases({ + superTestService: supertestWithoutAuth, + data: { cases: [getPostCaseRequest({ owner: 'observabilityFixture' })] }, + expectedHttpCode: 403, + auth: { + user: secOnly, + space: 'space1', + }, + }); + }); + + for (const user of [globalRead, secOnlyRead, obsOnlyRead, obsSecRead, noKibanaPrivileges]) { + it(`User ${ + user.username + } with role(s) ${user.roles.join()} - should NOT create a case`, async () => { + await bulkCreateCases({ + superTestService: supertestWithoutAuth, + data: { cases: [getPostCaseRequest({ owner: 'securitySolutionFixture' })] }, + expectedHttpCode: 403, + auth: { + user, + space: 'space1', + }, + }); + }); + } + + it('should NOT create a case in a space with no permissions', async () => { + await bulkCreateCases({ + superTestService: supertestWithoutAuth, + data: { cases: [getPostCaseRequest({ owner: 'securitySolutionFixture' })] }, + expectedHttpCode: 403, + auth: { + user: secOnly, + space: 'space2', + }, + }); + }); + }); + }); +}; diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/index.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/index.ts index 72d6c093f4bfb4..a0aee09f47ed94 100644 --- a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/index.ts +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/index.ts @@ -10,6 +10,9 @@ import { FtrProviderContext } from '../../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default ({ loadTestFile }: FtrProviderContext): void => { describe('Common', function () { + /** + * Public routes + */ loadTestFile(require.resolve('./client/update_alert_status')); loadTestFile(require.resolve('./comments/delete_comment')); loadTestFile(require.resolve('./comments/delete_comments')); @@ -46,7 +49,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => { /** * Internal routes */ - loadTestFile(require.resolve('./internal/bulk_create_attachments')); loadTestFile(require.resolve('./internal/bulk_get_cases')); loadTestFile(require.resolve('./internal/bulk_get_attachments')); @@ -61,6 +63,11 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./attachments_framework/external_references.ts')); loadTestFile(require.resolve('./attachments_framework/persistable_state.ts')); + /** + * Cases client + */ + loadTestFile(require.resolve('./cases/bulk_create_cases')); + // NOTE: Migrations are not included because they can inadvertently remove the .kibana indices which removes the users and spaces // which causes errors in any tests after them that relies on those }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts index 0ac86c991015d8..144d5e9bf51bd2 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts @@ -11,10 +11,7 @@ import { v4 as uuidv4 } from 'uuid'; import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { orderBy } from 'lodash'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; -import { - getNewTermsRuntimeMappings, - AGG_FIELD_NAME, -} from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/new_terms/utils'; + import { getMaxSignalsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; import { createRule, @@ -23,14 +20,12 @@ import { getOpenSignals, getPreviewAlerts, previewRule, - performSearchQuery, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries'; import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { dataGeneratorFactory } from '../../utils/data_generator'; -import { largeArraysBuckets } from './mocks/new_terms'; import { removeRandomValuedProperties } from './utils'; const historicalWindowStart = '2022-10-13T05:00:04.000Z'; @@ -468,7 +463,7 @@ export default ({ getService }: FtrProviderContext) => { const previewAlerts = await getPreviewAlerts({ es, previewId }); expect(previewAlerts.length).eql(1); - expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', 'false']); + expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', false]); }); it('should generate alerts for every term when history window is small', async () => { @@ -621,6 +616,293 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).eql(100); }); + + it('should not miss alerts if rule execution value combinations number is greater than 100', async () => { + // historical window documents + // 100 combinations for 127.0.0.1 x host-0, host-1, ..., host-100 + const historicalDocuments = [ + { + host: { + name: Array.from(Array(100)).map((_, i) => `host-${100 + i}`), + ip: ['127.0.0.1'], + }, + }, + ]; + + // rule execution documents + // 100 old combinations for 127.0.0.1 x host-0, host-1, ..., host-99 + // 10 new combinations 127.0.0.1 x a-0, a-1, ..., a-9 + const ruleExecutionDocuments = [ + { + host: { + name: [ + ...Array.from(Array(100)).map((_, i) => `host-${100 + i}`), + ...Array.from(Array(10)).map((_, i) => `a-${i}`), + ], + ip: ['127.0.0.1'], + }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.ip'], + from: ruleExecutionStart, + history_window_start: historicalWindowStart, + query: `id: "${testId}"`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + // 10 alerts (with host.names a-[0-9]) should be generated + expect(previewAlerts.length).eql(10); + }); + + it('should not miss alerts for high cardinality values in arrays, over 10.000 composite page size', async () => { + // historical window documents + // number of combinations is 50,000 + const historicalDocuments = [ + { + host: { + name: Array.from(Array(100)).map((_, i) => `host-${100 + i}`), + domain: Array.from(Array(100)).map((_, i) => `domain-${100 + i}`), + }, + user: { + name: Array.from(Array(5)).map((_, i) => `user-${100 + i}`), + }, + }, + ]; + + // rule execution documents + // number of combinations is 50,000 + new one + const ruleExecutionDocuments = [ + { + host: { + name: Array.from(Array(100)).map((_, i) => `host-${100 + i}`), + domain: Array.from(Array(100)).map((_, i) => `domain-${100 + i}`), + }, + user: { + name: Array.from(Array(5)).map((_, i) => `user-${100 + i}`), + }, + }, + { + host: { + name: 'host-140', + domain: 'domain-9999', + }, + user: { + name: 'user-9999', + }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.domain', 'user.name'], + from: ruleExecutionStart, + history_window_start: historicalWindowStart, + query: `id: "${testId}"`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + // only 1 alert should be generated + expect(previewAlerts.length).eql(1); + }); + + it('should not miss alerts for high cardinality values in arrays, over 10.000 composite page size spread over multiple pages', async () => { + // historical window documents + // number of combinations is 50,000 + const historicalDocuments = [ + { + host: { + name: Array.from(Array(100)).map((_, i) => `host-${100 + i}`), + domain: Array.from(Array(100)).map((_, i) => `domain-${100 + i}`), + }, + user: { + name: Array.from(Array(5)).map((_, i) => `user-${100 + i}`), + }, + }, + ]; + + // rule execution documents + // number of combinations is 50,000 + 4 new ones + const ruleExecutionDocuments = [ + { + host: { + name: Array.from(Array(100)).map((_, i) => `host-${100 + i}`), + domain: Array.from(Array(100)).map((_, i) => `domain-${100 + i}`), + }, + user: { + name: Array.from(Array(5)).map((_, i) => `user-${100 + i}`), + }, + }, + { + host: { + name: 'host-102', + domain: 'domain-9999', + }, + user: { + name: 'user-9999', + }, + }, + { + host: { + name: 'host-140', + domain: 'domain-9999', + }, + user: { + name: 'user-9999', + }, + }, + { + host: { + name: 'host-133', + domain: 'domain-9999', + }, + user: { + name: 'user-9999', + }, + }, + { + host: { + name: 'host-132', + domain: 'domain-9999', + }, + user: { + name: 'user-9999', + }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.domain', 'user.name'], + from: ruleExecutionStart, + history_window_start: historicalWindowStart, + query: `id: "${testId}"`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + // only 4 alerts should be generated + expect(previewAlerts.length).eql(4); + }); + + it('should not generate false positive alerts if rule historical window combinations overlap execution ones, which have more than 100', async () => { + // historical window documents + // number of combinations 400: [a, b] x domain-100, domain-101, ..., domain-299 + const historicalDocuments = [ + { + host: { + name: ['a', 'b'], + domain: Array.from(Array(200)).map((_, i) => `domain-${100 + i}`), + }, + }, + ]; + + // rule execution documents + // number of combinations 101: [a] x domain-100, domain-101, ..., domain-199 + b x domain-201 + // no new combination of values emitted + const ruleExecutionDocuments = [ + { + host: { + name: 'a', + domain: Array.from(Array(100)).map((_, i) => `domain-${100 + i}`), + }, + }, + { + host: { + name: 'b', + domain: 'domain-201', + }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.domain'], + from: ruleExecutionStart, + history_window_start: historicalWindowStart, + query: `id: "${testId}"`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + expect(previewAlerts.length).eql(0); + }); + + it('should not generate false positive alerts if rule historical window combinations overlap execution ones, which have precisely 100', async () => { + // historical window documents + // number of combinations 400: [a, b] x domain-100, domain-101, ..., domain-299 + const historicalDocuments = [ + { + host: { + name: ['a', 'b'], + domain: Array.from(Array(200)).map((_, i) => `domain-${100 + i}`), + }, + }, + ]; + + // rule execution documents + // number of combinations 100: [a] x domain-100, domain-101, ..., domain-199 + // no new combination of values emitted + const ruleExecutionDocuments = [ + { + host: { + name: 'a', + domain: Array.from(Array(100)).map((_, i) => `domain-${100 + i}`), + }, + }, + ]; + + const testId = await newTermsTestExecutionSetup({ + historicalDocuments, + ruleExecutionDocuments, + }); + + const rule: NewTermsRuleCreateProps = { + ...getCreateNewTermsRulesSchemaMock('rule-1', true), + index: ['new_terms'], + new_terms_fields: ['host.name', 'host.domain'], + from: ruleExecutionStart, + history_window_start: historicalWindowStart, + query: `id: "${testId}"`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 }); + + expect(previewAlerts.length).eql(0); + }); }); describe('timestamp override and fallback', () => { @@ -753,299 +1035,5 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23); }); }); - - describe('runtime field', () => { - it('should return runtime field created from 2 single values', async () => { - // encoded base64 values of "host-0" and "127.0.0.1" joined with underscore - const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x']; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'first_doc' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['host.name', 'host.ip'], - [ - { - key: { - 'host.name': 'host-0', - 'host.ip': '127.0.0.1', - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should not return runtime field created from 2 single values if its value is not in buckets', async () => { - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'first_doc' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['host.name', 'host.ip'], - [ - { - key: { - 'host.name': 'host-0', - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); - }); - - it('should return runtime field created from 2 single values, including number value', async () => { - // encoded base64 values of "user-0" and 0 joined with underscore - const expectedEncodedValues = ['dXNlci0w_MA==']; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'first_doc' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['user.name', 'user.id'], - [ - { - key: { - 'user.name': 'user-0', - 'user.id': 0, - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should return runtime field created from 2 single values, including boolean value', async () => { - // encoded base64 values of "user-0" and true joined with underscore - const expectedEncodedValues = ['dXNlci0w_dHJ1ZQ==']; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'first_doc' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['user.name', 'user.enabled'], - [ - { - key: { - 'user.name': 'user-0', - 'user.enabled': true, - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should return runtime field created from 3 single values', async () => { - // encoded base64 values of "host-0" and "127.0.0.1" and "user-0" joined with underscore - const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x_dXNlci0w']; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'first_doc' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['host.name', 'host.ip', 'user.name'], - [ - { - key: { - 'host.name': 'host-0', - 'host.ip': '127.0.0.1', - 'user.name': 'user-0', - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should return runtime field created from fields of arrays', async () => { - // encoded base64 values of all combinations of ["192.168.1.1", "192.168.1.2"] - // and ["tag-new-1", "tag-2", "tag-new-3"] joined with underscore - const expectedEncodedValues = [ - 'MTkyLjE2OC4xLjE=_dGFnLTI=', - 'MTkyLjE2OC4xLjE=_dGFnLW5ldy0x', - 'MTkyLjE2OC4xLjE=_dGFnLW5ldy0z', - 'MTkyLjE2OC4xLjI=_dGFnLTI=', - 'MTkyLjE2OC4xLjI=_dGFnLW5ldy0x', - 'MTkyLjE2OC4xLjI=_dGFnLW5ldy0z', - ]; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'doc_with_source_ip_as_array' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['source.ip', 'tags'], - [ - { - key: { - tags: 'tag-new-1', - 'source.ip': '192.168.1.1', - }, - doc_count: 1, - }, - { - key: { - tags: 'tag-2', - 'source.ip': '192.168.1.1', - }, - doc_count: 1, - }, - { - key: { - tags: 'tag-new-3', - 'source.ip': '192.168.1.1', - }, - doc_count: 1, - }, - { - key: { - tags: 'tag-new-1', - 'source.ip': '192.168.1.2', - }, - doc_count: 1, - }, - { - key: { - tags: 'tag-2', - 'source.ip': '192.168.1.2', - }, - doc_count: 1, - }, - { - key: { - tags: 'tag-new-3', - 'source.ip': '192.168.1.2', - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should return runtime field without duplicated values', async () => { - // encoded base64 values of "host-0" and ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"] - // joined with underscore, without duplicates in tags - const expectedEncodedValues = ['aG9zdC0w_dGFnLTE=', 'aG9zdC0w_dGFnLTI=']; - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'doc_with_duplicated_tags' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['host.name', 'tags'], - [ - { - key: { - tags: 'tag-1', - 'host.name': 'host-0', - }, - doc_count: 1, - }, - { - key: { - tags: 'tag-2', - 'host.name': 'host-0', - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues); - }); - - it('should not return runtime field if one of fields is null', async () => { - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'doc_with_null_field' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'], - runtimeMappings: getNewTermsRuntimeMappings( - ['host.name', 'possibly_null_field'], - [ - { - key: { - 'host.name': 'host-0', - possibly_null_field: null, - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits.length).to.be(1); - expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined); - expect(hits.hits[0].fields?.possibly_null_field).to.be(undefined); - expect(hits.hits[0].fields?.['host.name']).to.eql(['host-0']); - }); - - it('should not return runtime field if one of fields is not defined in a document', async () => { - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'doc_without_large_arrays' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['host.name', 'large_array_5'], - [ - { - key: { - 'host.name': 'host-0', - }, - doc_count: 1, - }, - ] - ), - }); - - expect(hits.hits.length).to.be(1); - expect(hits.hits[0].fields).to.be(undefined); - }); - - // There is a limit in ES for a number of emitted values in runtime field (100) - // This test makes sure runtime script doesn't cause query failure and returns first 100 results - it('should return runtime field if number of emitted values greater than 100', async () => { - const { hits } = await performSearchQuery({ - es, - query: { match: { id: 'first_doc' } }, - index: 'new_terms', - fields: [AGG_FIELD_NAME], - runtimeMappings: getNewTermsRuntimeMappings( - ['large_array_20', 'large_array_10'], - largeArraysBuckets - ), - }); - - // runtime field should have 100 values, as large_array_20 and large_array_10 - // give in total 200 combinations - expect(hits.hits[0].fields?.[AGG_FIELD_NAME].length).to.be(100); - }); - }); }); }; diff --git a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json index 2f156ddedf5802..ef1868f329e4c4 100644 --- a/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json @@ -12,6 +12,9 @@ }, "host": { "properties": { + "domain": { + "type": "keyword" + }, "name": { "type": "keyword" }, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/open_close_alerts.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/open_close_alerts.ts index 801791ccc660d1..87d9293cb1bd02 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/open_close_alerts.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/open_close_alerts.ts @@ -47,7 +47,8 @@ export default ({ getService }: FtrProviderContext) => { const dataPathBuilder = new EsArchivePathBuilder(isServerless); const path = dataPathBuilder.getPath('auditbeat/hosts'); - describe('@ess @serverless open_close_alerts', () => { + // Failing: See https://github.com/elastic/kibana/issues/170753 + describe.skip('@ess @serverless open_close_alerts', () => { describe('validation checks', () => { describe('update by ids', () => { it('should not give errors when querying and the alerts index does not exist yet', async () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/alert_status.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/alert_status.cy.ts index ca90e9b72efd1b..c55c8f62dc4a9c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/alert_status.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_alerts/alert_status.cy.ts @@ -5,8 +5,16 @@ * 2.0. */ +import { ROLES } from '@kbn/security-solution-plugin/common/test'; import { getNewRule } from '../../../objects/rule'; -import { ALERTS_COUNT, SELECTED_ALERTS } from '../../../screens/alerts'; +import { + ALERTS_COUNT, + CLOSE_SELECTED_ALERTS_BTN, + MARK_ALERT_ACKNOWLEDGED_BTN, + SELECTED_ALERTS, + TAKE_ACTION_POPOVER_BTN, + TIMELINE_CONTEXT_MENU_BTN, +} from '../../../screens/alerts'; import { selectNumberOfAlerts, @@ -30,12 +38,12 @@ import { visit } from '../../../tasks/navigation'; import { ALERTS_URL } from '../../../urls/navigation'; // FLAKY: https://github.com/elastic/kibana/issues/169091 -describe('Changing alert status', { tags: ['@ess', '@serverless'] }, () => { +describe('Changing alert status', () => { before(() => { cy.task('esArchiverLoad', { archiveName: 'auditbeat_big' }); }); - context('Opening alerts', () => { + context('Opening alerts', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); deleteAlertsAndRules(); @@ -116,7 +124,7 @@ describe('Changing alert status', { tags: ['@ess', '@serverless'] }, () => { }); }); - context('Marking alerts as acknowledged', () => { + context('Marking alerts as acknowledged', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); deleteAlertsAndRules(); @@ -167,7 +175,7 @@ describe('Changing alert status', { tags: ['@ess', '@serverless'] }, () => { }); }); - context('Closing alerts', () => { + context('Closing alerts', { tags: ['@ess', '@serverless'] }, () => { beforeEach(() => { login(); deleteAlertsAndRules(); @@ -228,4 +236,32 @@ describe('Changing alert status', { tags: ['@ess', '@serverless'] }, () => { }); }); }); + + // This test is unable to be run in serverless as `reader` is not available and viewer is currently reserved + // https://github.com/elastic/kibana/pull/169723#issuecomment-1793191007 + // https://github.com/elastic/kibana/issues/170583 + context('User is readonly', { tags: ['@ess', '@brokenInServerless'] }, () => { + beforeEach(() => { + login(); + visit(ALERTS_URL); + deleteAlertsAndRules(); + createRule(getNewRule()); + login(ROLES.reader); + visit(ALERTS_URL, { role: ROLES.reader }); + waitForAlertsToPopulate(); + }); + it('should not allow users to change a single alert status', () => { + // This is due to the reader role which makes everything in security 'read only' + cy.get(TIMELINE_CONTEXT_MENU_BTN).should('not.exist'); + }); + + it('should not allow users to bulk change the alert status', () => { + selectNumberOfAlerts(2); + cy.get(TAKE_ACTION_POPOVER_BTN).first().click(); + cy.get(TAKE_ACTION_POPOVER_BTN).should('be.visible'); + + cy.get(CLOSE_SELECTED_ALERTS_BTN).should('not.exist'); + cy.get(MARK_ALERT_ACKNOWLEDGED_BTN).should('not.exist'); + }); + }); });
+ + + {children} + + +
+
+ + {children} + +
+