From fb0e5ceb48a653047ec0f7b7c0fb7a09092b0100 Mon Sep 17 00:00:00 2001 From: Bryan Clement Date: Tue, 20 Jul 2021 04:21:39 -0700 Subject: [PATCH] [Osquery] 7.14 bug squash (#105387) --- .../action_results/action_results_summary.tsx | 12 ++++++ .../osquery/public/agents/use_all_agents.ts | 3 +- .../routes/saved_queries/list/index.tsx | 4 +- .../form/use_saved_query_form.tsx | 29 ++++++++++++-- .../saved_queries/use_create_saved_query.ts | 10 +++++ .../saved_queries/use_update_saved_query.ts | 11 +++++ .../form/queries_field.tsx | 16 ++++++++ .../queries/query_flyout.tsx | 5 ++- .../scheduled_query_groups/queries/schema.tsx | 12 ++++-- .../use_scheduled_query_group_query_form.tsx | 27 ++++++++++--- .../queries/validations.ts | 17 +++++++- .../use_scheduled_query_group.ts | 2 +- .../server/routes/usage/recorder.test.ts | 23 ++++------- .../osquery/server/routes/usage/recorder.ts | 40 +++++++++---------- .../plugins/osquery/server/usage/collector.ts | 6 +-- .../plugins/osquery/server/usage/fetchers.ts | 8 +++- 16 files changed, 162 insertions(+), 63 deletions(-) diff --git a/x-pack/plugins/osquery/public/action_results/action_results_summary.tsx b/x-pack/plugins/osquery/public/action_results/action_results_summary.tsx index bf4c97d63d74c..75277059bbf97 100644 --- a/x-pack/plugins/osquery/public/action_results/action_results_summary.tsx +++ b/x-pack/plugins/osquery/public/action_results/action_results_summary.tsx @@ -53,6 +53,18 @@ const ActionResultsSummaryComponent: React.FC = ({ sortField: '@timestamp', isLive, }); + if (expired) { + // @ts-expect-error update types + edges.forEach((edge) => { + if (!edge.fields.completed_at) { + edge.fields['error.keyword'] = edge.fields.error = [ + i18n.translate('xpack.osquery.liveQueryActionResults.table.expiredErrorText', { + defaultMessage: 'The action request timed out.', + }), + ]; + } + }); + } const { data: logsResults } = useAllResults({ actionId, diff --git a/x-pack/plugins/osquery/public/agents/use_all_agents.ts b/x-pack/plugins/osquery/public/agents/use_all_agents.ts index cda15cc805437..fac43eaa7ffc3 100644 --- a/x-pack/plugins/osquery/public/agents/use_all_agents.ts +++ b/x-pack/plugins/osquery/public/agents/use_all_agents.ts @@ -34,8 +34,7 @@ export const useAllAgents = ( const { isLoading: agentsLoading, data: agentData } = useQuery( ['agents', osqueryPolicies, searchValue, perPage], () => { - const policyFragment = osqueryPolicies.map((p) => `policy_id:${p}`).join(' or '); - let kuery = `last_checkin_status: online and (${policyFragment})`; + let kuery = `${osqueryPolicies.map((p) => `policy_id:${p}`).join(' or ')}`; if (searchValue) { kuery += ` and (local_metadata.host.hostname:*${searchValue}* or local_metadata.elastic.agent.id:*${searchValue}*)`; diff --git a/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx b/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx index 8738c06d06597..0c04e816dae7a 100644 --- a/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx +++ b/x-pack/plugins/osquery/public/routes/saved_queries/list/index.tsx @@ -55,8 +55,8 @@ const SavedQueriesPageComponent = () => { const { push } = useHistory(); const newQueryLinkProps = useRouterNavigate('saved_queries/new'); const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState('updated_at'); + const [pageSize, setPageSize] = useState(20); + const [sortField, setSortField] = useState('attributes.updated_at'); const [sortDirection, setSortDirection] = useState('desc'); const { data } = useSavedQueries({ isLive: true }); diff --git a/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx b/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx index 6417b40747e0f..8cfceec643bac 100644 --- a/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx +++ b/x-pack/plugins/osquery/public/saved_queries/form/use_saved_query_form.tsx @@ -9,9 +9,11 @@ import { isArray } from 'lodash'; import uuid from 'uuid'; import { produce } from 'immer'; +import { useMemo } from 'react'; import { useForm } from '../../shared_imports'; -import { formSchema } from '../../scheduled_query_groups/queries/schema'; +import { createFormSchema } from '../../scheduled_query_groups/queries/schema'; import { ScheduledQueryGroupFormData } from '../../scheduled_query_groups/queries/use_scheduled_query_group_query_form'; +import { useSavedQueries } from '../use_saved_queries'; const SAVED_QUERY_FORM_ID = 'savedQueryForm'; @@ -20,11 +22,29 @@ interface UseSavedQueryFormProps { handleSubmit: (payload: unknown) => Promise; } -export const useSavedQueryForm = ({ defaultValue, handleSubmit }: UseSavedQueryFormProps) => - useForm({ +export const useSavedQueryForm = ({ defaultValue, handleSubmit }: UseSavedQueryFormProps) => { + const { data } = useSavedQueries({}); + const ids: string[] = useMemo( + () => data?.savedObjects.map((obj) => obj.attributes.id) ?? [], + [data] + ); + const idSet = useMemo>(() => { + const res = new Set(ids); + // @ts-expect-error update types + if (defaultValue && defaultValue.id) res.delete(defaultValue.id); + return res; + }, [ids, defaultValue]); + const formSchema = useMemo>(() => createFormSchema(idSet), [ + idSet, + ]); + return useForm({ id: SAVED_QUERY_FORM_ID + uuid.v4(), schema: formSchema, - onSubmit: handleSubmit, + onSubmit: async (formData, isValid) => { + if (isValid) { + return handleSubmit(formData); + } + }, options: { stripEmptyFields: false, }, @@ -62,3 +82,4 @@ export const useSavedQueryForm = ({ defaultValue, handleSubmit }: UseSavedQueryF }; }, }); +}; diff --git a/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts index cc5c33c6e4280..1d10d80bd6fbf 100644 --- a/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts +++ b/x-pack/plugins/osquery/public/saved_queries/use_create_saved_query.ts @@ -37,6 +37,16 @@ export const useCreateSavedQuery = ({ withRedirect }: UseCreateSavedQueryProps) throw new Error('CurrentUser is missing'); } + const conflictingEntries = await savedObjects.client.find({ + type: savedQuerySavedObjectType, + // @ts-expect-error update types + search: payload.id, + searchFields: ['id'], + }); + if (conflictingEntries.savedObjects.length) { + // @ts-expect-error update types + throw new Error(`Saved query with id ${payload.id} already exists.`); + } return savedObjects.client.create(savedQuerySavedObjectType, { // @ts-expect-error update types ...payload, diff --git a/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts b/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts index 6f4aa51710811..fe0d38648b23c 100644 --- a/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts +++ b/x-pack/plugins/osquery/public/saved_queries/use_update_saved_query.ts @@ -37,6 +37,17 @@ export const useUpdateSavedQuery = ({ savedQueryId }: UseUpdateSavedQueryProps) throw new Error('CurrentUser is missing'); } + const conflictingEntries = await savedObjects.client.find({ + type: savedQuerySavedObjectType, + // @ts-expect-error update types + search: payload.id, + searchFields: ['id'], + }); + if (conflictingEntries.savedObjects.length) { + // @ts-expect-error update types + throw new Error(`Saved query with id ${payload.id} already exists.`); + } + return savedObjects.client.update(savedQuerySavedObjectType, savedQueryId, { // @ts-expect-error update types ...payload, diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/form/queries_field.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/form/queries_field.tsx index 0718ff028e002..46b4a9a72f7ee 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/form/queries_field.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/form/queries_field.tsx @@ -216,6 +216,20 @@ const QueriesFieldComponent: React.FC = ({ field.value, ]); + const uniqueQueryIds = useMemo( + () => + field.value && field.value[0].streams.length + ? field.value[0].streams.reduce((acc, stream) => { + if (stream.vars?.id.value) { + acc.push(stream.vars?.id.value); + } + + return acc; + }, [] as string[]) + : [], + [field.value] + ); + return ( <> @@ -256,6 +270,7 @@ const QueriesFieldComponent: React.FC = ({ {} {showAddQueryFlyout && ( = ({ )} {showEditQueryFlyout != null && showEditQueryFlyout >= 0 && ( Promise; @@ -47,6 +48,7 @@ interface QueryFlyoutProps { } const QueryFlyoutComponent: React.FC = ({ + uniqueQueryIds, defaultValue, integrationPackageVersion, onSave, @@ -54,6 +56,7 @@ const QueryFlyoutComponent: React.FC = ({ }) => { const [isEditMode] = useState(!!defaultValue); const { form } = useScheduledQueryGroupQueryForm({ + uniqueQueryIds, defaultValue, handleSubmit: (payload, isValid) => new Promise((resolve) => { @@ -65,7 +68,7 @@ const QueryFlyoutComponent: React.FC = ({ }), }); - /* Platform and version fields are supported since osquer_manger@0.3.0 */ + /* Platform and version fields are supported since osquery_manager@0.3.0 */ const isFieldSupported = useMemo( () => (integrationPackageVersion ? satisfies(integrationPackageVersion, '>=0.3.0') : false), [integrationPackageVersion] diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx index 0b23ce924f930..3eb299cf5fa15 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/schema.tsx @@ -12,15 +12,19 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { FIELD_TYPES } from '../../shared_imports'; -import { idFieldValidations, intervalFieldValidation, queryFieldValidation } from './validations'; +import { + createIdFieldValidations, + intervalFieldValidation, + queryFieldValidation, +} from './validations'; -export const formSchema = { +export const createFormSchema = (ids: Set) => ({ id: { type: FIELD_TYPES.TEXT, label: i18n.translate('xpack.osquery.scheduledQueryGroup.queryFlyoutForm.idFieldLabel', { defaultMessage: 'ID', }), - validations: idFieldValidations.map((validator) => ({ validator })), + validations: createIdFieldValidations(ids).map((validator) => ({ validator })), }, description: { type: FIELD_TYPES.TEXT, @@ -69,4 +73,4 @@ export const formSchema = { ) as unknown) as string, validations: [], }, -}; +}); diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx index fdf781c6d6f7a..67361e612b094 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/use_scheduled_query_group_query_form.tsx @@ -5,17 +5,19 @@ * 2.0. */ -import { isArray } from 'lodash'; +import { isArray, xor } from 'lodash'; import uuid from 'uuid'; import { produce } from 'immer'; +import { useMemo } from 'react'; import { FormConfig, useForm } from '../../shared_imports'; import { OsqueryManagerPackagePolicyConfigRecord } from '../../../common/types'; -import { formSchema } from './schema'; +import { createFormSchema } from './schema'; const FORM_ID = 'editQueryFlyoutForm'; export interface UseScheduledQueryGroupQueryFormProps { + uniqueQueryIds: string[]; defaultValue?: OsqueryManagerPackagePolicyConfigRecord | undefined; handleSubmit: FormConfig< OsqueryManagerPackagePolicyConfigRecord, @@ -32,12 +34,26 @@ export interface ScheduledQueryGroupFormData { } export const useScheduledQueryGroupQueryForm = ({ + uniqueQueryIds, defaultValue, handleSubmit, -}: UseScheduledQueryGroupQueryFormProps) => - useForm({ +}: UseScheduledQueryGroupQueryFormProps) => { + const idSet = useMemo>( + () => + new Set(xor(uniqueQueryIds, defaultValue?.id.value ? [defaultValue.id.value] : [])), + [uniqueQueryIds, defaultValue] + ); + const formSchema = useMemo>(() => createFormSchema(idSet), [ + idSet, + ]); + + return useForm({ id: FORM_ID + uuid.v4(), - onSubmit: handleSubmit, + onSubmit: async (formData, isValid) => { + if (isValid && handleSubmit) { + return handleSubmit(formData, isValid); + } + }, options: { stripEmptyFields: false, }, @@ -75,3 +91,4 @@ export const useScheduledQueryGroupQueryForm = ({ }, schema: formSchema, }); +}; diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/validations.ts b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/validations.ts index 95e3000476a08..c9f128b8e5d79 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/queries/validations.ts +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/queries/validations.ts @@ -23,13 +23,28 @@ const idSchemaValidation: ValidationFunc = ({ value }) => { } }; -export const idFieldValidations = [ +const createUniqueIdValidation = (ids: Set) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const uniqueIdCheck: ValidationFunc = ({ value }) => { + if (ids.has(value)) { + return { + message: i18n.translate('xpack.osquery.scheduledQueryGroup.queryFlyoutForm.uniqueIdError', { + defaultMessage: 'ID must be unique', + }), + }; + } + }; + return uniqueIdCheck; +}; + +export const createIdFieldValidations = (ids: Set) => [ fieldValidators.emptyField( i18n.translate('xpack.osquery.scheduledQueryGroup.queryFlyoutForm.emptyIdError', { defaultMessage: 'ID is required', }) ), idSchemaValidation, + createUniqueIdValidation(ids), ]; export const intervalFieldValidation: ValidationFunc< diff --git a/x-pack/plugins/osquery/public/scheduled_query_groups/use_scheduled_query_group.ts b/x-pack/plugins/osquery/public/scheduled_query_groups/use_scheduled_query_group.ts index 93d552b3f71f3..30adbb6cfa4ee 100644 --- a/x-pack/plugins/osquery/public/scheduled_query_groups/use_scheduled_query_group.ts +++ b/x-pack/plugins/osquery/public/scheduled_query_groups/use_scheduled_query_group.ts @@ -31,7 +31,7 @@ export const useScheduledQueryGroup = ({ () => http.get(packagePolicyRouteService.getInfoPath(scheduledQueryGroupId)), { keepPreviousData: true, - enabled: !skip, + enabled: !skip || !scheduledQueryGroupId, select: (response) => response.item, } ); diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts index aa5f550234fcb..ae93f08d76bd5 100644 --- a/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts @@ -11,7 +11,7 @@ import { usageMetricSavedObjectType } from '../../../common/types'; import { CounterValue, - createMetricObjects, + getOrCreateMetricObject, getRouteMetric, incrementCount, RouteString, @@ -45,31 +45,22 @@ describe('Usage metric recorder', () => { get.mockClear(); create.mockClear(); }); - it('should seed route metrics objects', async () => { + it('should create metrics that do not exist', async () => { get.mockRejectedValueOnce('stub value'); create.mockReturnValueOnce('stub value'); - const result = await createMetricObjects(savedObjectsClient); + const result = await getOrCreateMetricObject(savedObjectsClient, 'live_query'); checkGetCalls(get.mock.calls); checkCreateCalls(create.mock.calls); - expect(result).toBe(true); + expect(result).toBe('stub value'); }); - it('should handle previously seeded objects properly', async () => { + it('should handle previously created objects properly', async () => { get.mockReturnValueOnce('stub value'); create.mockRejectedValueOnce('stub value'); - const result = await createMetricObjects(savedObjectsClient); + const result = await getOrCreateMetricObject(savedObjectsClient, 'live_query'); checkGetCalls(get.mock.calls); checkCreateCalls(create.mock.calls, []); - expect(result).toBe(true); - }); - - it('should report failure to create the metrics object', async () => { - get.mockRejectedValueOnce('stub value'); - create.mockRejectedValueOnce('stub value'); - const result = await createMetricObjects(savedObjectsClient); - checkGetCalls(get.mock.calls); - checkCreateCalls(create.mock.calls); - expect(result).toBe(false); + expect(result).toBe('stub value'); }); }); diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.ts index 9f5e7cd1d56e0..cd374b9020979 100644 --- a/x-pack/plugins/osquery/server/routes/usage/recorder.ts +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.ts @@ -18,30 +18,28 @@ export type RouteString = 'live_query'; export const routeStrings: RouteString[] = ['live_query']; -export async function createMetricObjects(soClient: SavedObjectsClientContract) { - const res = await Promise.allSettled( - routeStrings.map(async (route) => { - try { - await soClient.get(usageMetricSavedObjectType, route); - } catch (e) { - await soClient.create( - usageMetricSavedObjectType, - { - errors: 0, - count: 0, - }, - { - id: route, - } - ); +export async function getOrCreateMetricObject( + soClient: SavedObjectsClientContract, + route: string +) { + try { + return await soClient.get(usageMetricSavedObjectType, route); + } catch (e) { + return await soClient.create( + usageMetricSavedObjectType, + { + errors: 0, + count: 0, + }, + { + id: route, } - }) - ); - return !res.some((e) => e.status === 'rejected'); + ); + } } export async function getCount(soClient: SavedObjectsClientContract, route: RouteString) { - return await soClient.get(usageMetricSavedObjectType, route); + return await getOrCreateMetricObject(soClient, route); } export interface CounterValue { @@ -55,7 +53,7 @@ export async function incrementCount( key: keyof CounterValue = 'count', increment = 1 ) { - const metric = await soClient.get(usageMetricSavedObjectType, route); + const metric = await getOrCreateMetricObject(soClient, route); metric.attributes[key] += increment; await soClient.update(usageMetricSavedObjectType, route, metric.attributes); } diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts index 9b690be6df0f1..4432592a4e063 100644 --- a/x-pack/plugins/osquery/server/usage/collector.ts +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -7,7 +7,6 @@ import { CoreSetup, SavedObjectsClient } from '../../../../../src/core/server'; import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; -import { createMetricObjects } from '../routes/usage'; import { getBeatUsage, getLiveQueryUsage, getPolicyLevelUsage } from './fetchers'; import { CollectorDependencies, usageSchema, UsageData } from './types'; @@ -25,10 +24,7 @@ export const registerCollector: RegisterCollector = ({ core, osqueryContext, usa const collector = usageCollection.makeUsageCollector({ type: 'osquery', schema: usageSchema, - isReady: async () => { - const savedObjectsClient = new SavedObjectsClient(await getInternalSavedObjectsClient(core)); - return await createMetricObjects(savedObjectsClient); - }, + isReady: () => true, fetch: async ({ esClient }: CollectorFetchContext): Promise => { const savedObjectsClient = new SavedObjectsClient(await getInternalSavedObjectsClient(core)); return { diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index 3d5f3592101fd..3ac7d56acac4d 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -45,6 +45,11 @@ export async function getPolicyLevelUsage( const agentResponse = await esClient.search({ body: { size: 0, + query: { + match: { + active: true, + }, + }, aggs: { policied: { filter: { @@ -87,7 +92,8 @@ export function getScheduledQueryUsage(packagePolicies: ListResult { ++acc.queryGroups.total; - if (item.inputs.length === 0) { + const policyAgents = item.inputs.reduce((sum, input) => sum + input.streams.length, 0); + if (policyAgents === 0) { ++acc.queryGroups.empty; } return acc;